From eda961edc2cdc6cf2dd6f5e71693bcd24997e178 Mon Sep 17 00:00:00 2001 From: KUOKA Yusuke Date: Sat, 27 Jul 2019 22:57:54 +0900 Subject: [PATCH 01/18] fix: --state-values-set not setting more than first child in nested paths (#774) Fixes #773 --- pkg/maputil/maputil.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/maputil/maputil.go b/pkg/maputil/maputil.go index 8a1ef45c..76b2fe82 100644 --- a/pkg/maputil/maputil.go +++ b/pkg/maputil/maputil.go @@ -76,8 +76,13 @@ func Set(m map[string]interface{}, key []string, value string) map[string]interf nested, ok := m[k] if !ok { - new_m := map[string]interface{}{} - nested = Set(new_m, remain, value) + nested = map[string]interface{}{} + } + switch t := nested.(type) { + case map[string]interface{}: + nested = Set(t, remain, value) + default: + panic(fmt.Errorf("unexpected type: %v(%T)", t, t)) } m[k] = nested From fe5102e7773aa1ffe0c807ce6be30496d76ef481 Mon Sep 17 00:00:00 2001 From: bitsofinfo Date: Mon, 29 Jul 2019 19:42:47 -0600 Subject: [PATCH 02/18] users update (#750) --- USERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/USERS.md b/USERS.md index 89505904..a2d11516 100644 --- a/USERS.md +++ b/USERS.md @@ -16,4 +16,5 @@ information to this file. | [Vlocity](https://vlocity.com/) | proof-of-concept | | Melbourne, Australia | March 2019 | | [transit](https://transit.app/) | production | [Blog post](https://medium.com/@naseem_60378/helmfile-its-like-a-helm-for-your-helm-74a908581599). | Montreal, Canada | March 2019 | | [uniqkey](https://uniqkey.eu/) | production | [Wiki Page](https://ocd-scm.github.io/ocd-meta/) | Copenhagen, Denmark | April 2019 | +| [bitsofinfo](https://github.com/bitsofinfo/helmfile-deploy) | proof-of-concept, stage | Used with [helmfile-deploy](https://github.com/bitsofinfo/helmfile-deploy) to manage releases for dozens of apps | USA | July 2019 | From e5038fb04f5e1f134740ddc3a64b794313e85a9c Mon Sep 17 00:00:00 2001 From: a-hat <51818964+a-hat@users.noreply.github.com> Date: Tue, 30 Jul 2019 03:43:30 +0200 Subject: [PATCH 03/18] print yaml content on error in fromYaml (#765) --- pkg/tmpl/context_funcs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tmpl/context_funcs.go b/pkg/tmpl/context_funcs.go index 28a6f691..9b563348 100644 --- a/pkg/tmpl/context_funcs.go +++ b/pkg/tmpl/context_funcs.go @@ -149,7 +149,7 @@ func FromYaml(str string) (Values, error) { m := Values{} if err := yaml.Unmarshal([]byte(str), &m); err != nil { - return nil, err + return nil, fmt.Errorf("%s, offending yaml: %s", err, str) } return m, nil } From bce2f4728b71a7a173766f11e1018c200e4c602c Mon Sep 17 00:00:00 2001 From: OlivierB Date: Tue, 30 Jul 2019 03:44:42 +0200 Subject: [PATCH 04/18] fix: pass namespace to helm template command (#771) Resolves #770 --- pkg/app/app_test.go | 8 ++++---- pkg/state/state.go | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 1702b7e6..8c6ca93f 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -1906,8 +1906,8 @@ releases: var helm = &mockHelmExec{} var wantReleases = []mockTemplates{ - {[]string{"--name", "myrelease1", "--output-dir", "output/subdir/helmfile-[a-z0-9]{8}-myrelease1"}}, - {[]string{"--name", "myrelease2", "--output-dir", "output/subdir/helmfile-[a-z0-9]{8}-myrelease2"}}, + {[]string{"--name", "myrelease1", "--namespace", "testNamespace", "--output-dir", "output/subdir/helmfile-[a-z0-9]{8}-myrelease1"}}, + {[]string{"--name", "myrelease2", "--namespace", "testNamespace", "--output-dir", "output/subdir/helmfile-[a-z0-9]{8}-myrelease2"}}, } var buffer bytes.Buffer @@ -1920,12 +1920,13 @@ releases: Env: "default", Logger: logger, helmExecer: helm, + Namespace: "testNamespace", }, files) app.Template(configImpl{}) for i := range wantReleases { for j := range wantReleases[i].flags { - if j == 3 { + if j == 5 { matched, _ := regexp.Match(wantReleases[i].flags[j], []byte(helm.templated[i].flags[j])) if !matched { t.Errorf("HelmState.TemplateReleases() = [%v], want %v", helm.templated[i].flags[j], wantReleases[i].flags[j]) @@ -1936,5 +1937,4 @@ releases: } } - } diff --git a/pkg/state/state.go b/pkg/state/state.go index 99818cdd..a6ae762e 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -569,6 +569,8 @@ func (st *HelmState) TemplateReleases(helm helmexec.Interface, outputDir string, continue } + st.applyDefaultsTo(&release) + flags, err := st.flagsForTemplate(helm, &release, 0) if err != nil { errs = append(errs, err) From 6baad71b1f08bc7566cf29cb86e44af1e560a71c Mon Sep 17 00:00:00 2001 From: Travis Groth Date: Wed, 7 Aug 2019 10:00:19 -0400 Subject: [PATCH 05/18] Cache secrets and concurrent decryption (#790) Related to #782 and #444 - Allows concurrent decryption of different secrets files - Caches decrypted secrets by original file path and returns decrypted results from memory - Secrets being run through an instance of helmexec will be cached and run as fast as possible concurrently NB: This particular PR doesn't make _all_ calls to secrets cached and concurrent. Environment Secrets in particular seem to not be evaluated with a ScatterGather(), and doesn't use the same helmexec instance as other parts of the code, so it doesn't take advantage of these changes. Some reworking of the plumbing there would be needed. --- pkg/helmexec/exec.go | 113 ++++++++++++++++++++++---------------- pkg/helmexec/exec_test.go | 10 +++- 2 files changed, 74 insertions(+), 49 deletions(-) diff --git a/pkg/helmexec/exec.go b/pkg/helmexec/exec.go index 52a1d596..9a72f659 100644 --- a/pkg/helmexec/exec.go +++ b/pkg/helmexec/exec.go @@ -17,13 +17,19 @@ const ( command = "helm" ) +type decryptedSecret struct { + mutex sync.RWMutex + bytes []byte +} + type execer struct { - helmBinary string - runner Runner - logger *zap.SugaredLogger - kubeContext string - extra []string - decryptionMutex sync.Mutex + helmBinary string + runner Runner + logger *zap.SugaredLogger + kubeContext string + extra []string + decryptedSecretMutex sync.Mutex + decryptedSecrets map[string]*decryptedSecret } func NewLogger(writer io.Writer, logLevel string) *zap.SugaredLogger { @@ -46,10 +52,11 @@ func NewLogger(writer io.Writer, logLevel string) *zap.SugaredLogger { // New for running helm commands func New(logger *zap.SugaredLogger, kubeContext string, runner Runner) *execer { return &execer{ - helmBinary: command, - logger: logger, - kubeContext: kubeContext, - runner: runner, + helmBinary: command, + logger: logger, + kubeContext: kubeContext, + runner: runner, + decryptedSecrets: make(map[string]*decryptedSecret), } } @@ -125,55 +132,67 @@ func (helm *execer) List(context HelmContext, filter string, flags ...string) (s } func (helm *execer) DecryptSecret(context HelmContext, name string, flags ...string) (string, error) { - // Prevents https://github.com/roboll/helmfile/issues/258 - helm.decryptionMutex.Lock() - defer helm.decryptionMutex.Unlock() - absPath, err := filepath.Abs(name) if err != nil { return "", err } - helm.logger.Infof("Decrypting secret %v", absPath) - preArgs := context.GetTillerlessArgs(helm.helmBinary) - env := context.getTillerlessEnv() - out, err := helm.exec(append(append(preArgs, "secrets", "dec", absPath), flags...), env) - helm.info(out) - if err != nil { - return "", err - } - tmpFile, err := ioutil.TempFile("", "secret") - if err != nil { - return "", err - } - defer tmpFile.Close() + helm.logger.Debugf("Preparing to decrypt secret %v", absPath) + helm.decryptedSecretMutex.Lock() - // HELM_SECRETS_DEC_SUFFIX is used by the helm-secrets plugin to define the output file - decSuffix := os.Getenv("HELM_SECRETS_DEC_SUFFIX") - if len(decSuffix) == 0 { - decSuffix = ".yaml.dec" - } - decFilename := strings.Replace(absPath, ".yaml", decSuffix, 1) + secret, ok := helm.decryptedSecrets[absPath] - // os.Rename seems to results in "cross-device link` errors in some cases - // Instead of moving, copy it to the destination temp file as a work-around - // See https://github.com/roboll/helmfile/issues/251#issuecomment-417166296f - decFile, err := os.Open(decFilename) - if err != nil { - return "", err - } - defer decFile.Close() + // Cache miss + if !ok { - _, err = io.Copy(tmpFile, decFile) - if err != nil { - return "", err + secret = &decryptedSecret{} + helm.decryptedSecrets[absPath] = secret + + secret.mutex.Lock() + defer secret.mutex.Unlock() + helm.decryptedSecretMutex.Unlock() + + helm.logger.Infof("Decrypting secret %v", absPath) + preArgs := context.GetTillerlessArgs(helm.helmBinary) + env := context.getTillerlessEnv() + out, err := helm.exec(append(append(preArgs, "secrets", "dec", absPath), flags...), env) + helm.info(out) + if err != nil { + return "", err + } + + // HELM_SECRETS_DEC_SUFFIX is used by the helm-secrets plugin to define the output file + decSuffix := os.Getenv("HELM_SECRETS_DEC_SUFFIX") + if len(decSuffix) == 0 { + decSuffix = ".yaml.dec" + } + decFilename := strings.Replace(absPath, ".yaml", decSuffix, 1) + + secretBytes, err := ioutil.ReadFile(decFilename) + if err != nil { + return "", err + } + secret.bytes = secretBytes + + if err := os.Remove(decFilename); err != nil { + return "", err + } + + } else { + // Cache hit + helm.logger.Debugf("Found secret in cache %v", absPath) + + secret.mutex.RLock() + helm.decryptedSecretMutex.Unlock() + defer secret.mutex.RUnlock() } - if err := decFile.Close(); err != nil { + tmpFile, err := ioutil.TempFile("", "secret") + if err != nil { return "", err } - - if err := os.Remove(decFilename); err != nil { + _, err = tmpFile.Write(secret.bytes) + if err != nil { return "", err } diff --git a/pkg/helmexec/exec_test.go b/pkg/helmexec/exec_test.go index 5fab7d60..59144ae8 100644 --- a/pkg/helmexec/exec_test.go +++ b/pkg/helmexec/exec_test.go @@ -228,10 +228,16 @@ func Test_DecryptSecret(t *testing.T) { if err != nil { t.Errorf("Error: %v", err) } - expected := fmt.Sprintf(`Decrypting secret %s/secretName + // Run again for caching + helm.DecryptSecret(HelmContext{}, "secretName") + + expected := fmt.Sprintf(`Preparing to decrypt secret %v/secretName +Decrypting secret %s/secretName exec: helm secrets dec %s/secretName --kube-context dev exec: helm secrets dec %s/secretName --kube-context dev: -`, cwd, cwd, cwd) +Preparing to decrypt secret %s/secretName +Found secret in cache %s/secretName +`, cwd, cwd, cwd, cwd, cwd, cwd) if buffer.String() != expected { t.Errorf("helmexec.DecryptSecret()\nactual = %v\nexpect = %v", buffer.String(), expected) } From 63a337ee57811aae57c34e3f775ab8b4d16892d4 Mon Sep 17 00:00:00 2001 From: Travis Groth Date: Wed, 7 Aug 2019 21:20:05 -0400 Subject: [PATCH 06/18] Base concurrency on items parameter (#798) Fixes #793 --- pkg/state/state_run.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/state/state_run.go b/pkg/state/state_run.go index 6db19c79..be2cbe68 100644 --- a/pkg/state/state_run.go +++ b/pkg/state/state_run.go @@ -2,8 +2,9 @@ package state import ( "fmt" - "github.com/roboll/helmfile/pkg/helmexec" "sync" + + "github.com/roboll/helmfile/pkg/helmexec" ) type result struct { @@ -12,11 +13,9 @@ type result struct { } func (st *HelmState) scatterGather(concurrency int, items int, produceInputs func(), receiveInputsAndProduceIntermediates func(int), aggregateIntermediates func()) { - numReleases := len(st.Releases) - if concurrency < 1 { - concurrency = numReleases - } else if concurrency > numReleases { - concurrency = numReleases + + if concurrency < 1 || concurrency > items { + concurrency = items } for _, r := range st.Releases { From 622cba9f1951e8b111d897e4d62eea7418af3a4e Mon Sep 17 00:00:00 2001 From: Jake Hill Date: Thu, 8 Aug 2019 13:24:46 +0100 Subject: [PATCH 07/18] Add kubectl and jq to Dockerfile (#799) * Add kubectl and jq to Dockerfile Signed-off-by: Jake Hill * Update kubectl to use checksum verification, and pin to a specific version Signed-off-by: Jake Hill Resolves #792 --- Dockerfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 020d8e91..063d34af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN make static-linux FROM alpine:3.8 -RUN apk add --no-cache ca-certificates git bash curl +RUN apk add --no-cache ca-certificates git bash curl jq ARG HELM_VERSION=v2.13.0 ARG HELM_LOCATION="https://kubernetes-helm.storage.googleapis.com" @@ -20,6 +20,16 @@ RUN wget ${HELM_LOCATION}/${HELM_FILENAME} && \ tar zxf ${HELM_FILENAME} && mv /linux-amd64/helm /usr/local/bin/ && \ rm ${HELM_FILENAME} && rm -r /linux-amd64 +# using the install documentation found at https://kubernetes.io/docs/tasks/tools/install-kubectl/ +# for now but in a future version of alpine (in the testing version at the time of writing) +# we should be able to install using apk add. +ENV KUBECTL_VERSION="v1.14.5" +ENV KUBECTL_SHA256="26681319de56820a8467c9407e9203d5b15fb010ffc75ac5b99c9945ad0bd28c" +RUN curl -LO "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \ + sha256sum kubectl | grep ${KUBECTL_SHA256} && \ + chmod +x kubectl && \ + mv kubectl /usr/local/bin/kubectl + RUN mkdir -p "$(helm home)/plugins" RUN helm plugin install https://github.com/databus23/helm-diff && \ helm plugin install https://github.com/futuresimple/helm-secrets && \ From 765bfe6cfdb12cebe10d6c2ccd0366f111e2c03f Mon Sep 17 00:00:00 2001 From: Travis Groth Date: Mon, 12 Aug 2019 20:23:29 -0400 Subject: [PATCH 08/18] Handle environment secrets concurrently Ref #782 --- pkg/state/create.go | 109 +++++++++++++++++++++++++++++++------------- 1 file changed, 77 insertions(+), 32 deletions(-) diff --git a/pkg/state/create.go b/pkg/state/create.go index c63ec813..be3ef4ad 100644 --- a/pkg/state/create.go +++ b/pkg/state/create.go @@ -4,14 +4,15 @@ import ( "bytes" "errors" "fmt" + "io" + "os" + "github.com/imdario/mergo" "github.com/roboll/helmfile/pkg/environment" "github.com/roboll/helmfile/pkg/helmexec" "github.com/roboll/helmfile/pkg/maputil" "go.uber.org/zap" "gopkg.in/yaml.v2" - "io" - "os" ) type StateLoadError struct { @@ -201,59 +202,103 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, envSecretFiles = append(envSecretFiles, resolved...) } + if err = st.scatterGatherEnvSecretFiles(envSecretFiles, helm, envVals, readFile); err != nil { + return nil, err + } + } + } else if ctxEnv == nil && name != DefaultEnv { + return nil, &UndefinedEnvError{msg: fmt.Sprintf("environment \"%s\" is not defined", name)} + } + + newEnv := &environment.Environment{Name: name, Values: envVals} + + if ctxEnv != nil { + intEnv := *ctxEnv + + if err := mergo.Merge(&intEnv, newEnv, mergo.WithOverride); err != nil { + return nil, fmt.Errorf("error while merging environment values for \"%s\": %v", name, err) + } + + newEnv = &intEnv + } + + return newEnv, nil +} + +func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, helm helmexec.Interface, envVals map[string]interface{}, readFile func(string) ([]byte, error)) error { + var errs []error + + inputs := envSecretFiles + inputsSize := len(inputs) - for _, path := range envSecretFiles { - // Work-around to allow decrypting environment secrets - // - // We don't have releases loaded yet and therefore unable to decide whether - // helmfile should use helm-tiller to call helm-secrets or not. - // - // This means that, when you use environment secrets + tillerless setup, you still need a tiller - // installed on the cluster, just for decrypting secrets! - // Related: https://github.com/futuresimple/helm-secrets/issues/83 + type secretResult struct { + result map[string]interface{} + err error + path string + } + + secrets := make(chan string, inputsSize) + results := make(chan secretResult, inputsSize) + + st.scatterGather(0, inputsSize, + func() { + for _, secretFile := range envSecretFiles { + secrets <- secretFile + } + close(secrets) + }, + func(id int) { + for path := range secrets { release := &ReleaseSpec{} flags := st.appendConnectionFlags([]string{}, release) decFile, err := helm.DecryptSecret(st.createHelmContext(release, 0), path, flags...) if err != nil { - return nil, err + results <- secretResult{nil, err, path} + continue } bytes, err := readFile(decFile) if err != nil { - return nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err) + results <- secretResult{nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err), path} + continue } m := map[string]interface{}{} if err := yaml.Unmarshal(bytes, &m); err != nil { - return nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err) + results <- secretResult{nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err), path} + continue } // All the nested map key should be string. Otherwise we get strange errors due to that // mergo or reflect is unable to merge map[interface{}]interface{} with map[string]interface{} or vice versa. // See https://github.com/roboll/helmfile/issues/677 vals, err := maputil.CastKeysToStrings(m) if err != nil { - return nil, err + results <- secretResult{nil, fmt.Errorf("failed to load environment secrets file \"%s\": %v", path, err), path} + continue } - if err := mergo.Merge(&envVals, &vals, mergo.WithOverride); err != nil { - return nil, fmt.Errorf("failed to load \"%s\": %v", path, err) + results <- secretResult{vals, nil, path} + } + }, + func() { + for i := 0; i < inputsSize; i++ { + result := <-results + if result.err != nil { + errs = append(errs, result.err) + } else { + if err := mergo.Merge(&envVals, &result.result, mergo.WithOverride); err != nil { + errs = append(errs, fmt.Errorf("failed to load environment secrets file \"%s\": %v", result.path, err)) + } } } - } - } else if ctxEnv == nil && name != DefaultEnv { - return nil, &UndefinedEnvError{msg: fmt.Sprintf("environment \"%s\" is not defined", name)} - } - - newEnv := &environment.Environment{Name: name, Values: envVals} - - if ctxEnv != nil { - intEnv := *ctxEnv + close(results) + }, + ) - if err := mergo.Merge(&intEnv, newEnv, mergo.WithOverride); err != nil { - return nil, fmt.Errorf("error while merging environment values for \"%s\": %v", name, err) + if len(errs) > 1 { + for _, err := range errs { + st.logger.Error(err) } - - newEnv = &intEnv + return fmt.Errorf("Failed loading environment secrets with %d errors", len(errs)) } - - return newEnv, nil + return nil } func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}) (map[string]interface{}, error) { From a584aeab2e2ae31e0d68e93d50867d1597c38b50 Mon Sep 17 00:00:00 2001 From: Travis Groth Date: Wed, 14 Aug 2019 20:27:55 -0400 Subject: [PATCH 09/18] Share helmexec from State Creation (#804) Closes #444 and #782 This is the final PR to fully cache and parallelize helm secret decryption. It threads the shared helmexec.Interface into the StateCreator and HelmState structs to be used during environment secret decryption. This should effectively cache secrets for the duration of a helmfile run, regardless of where they are first decrypted. --- pkg/app/app.go | 8 +++++--- pkg/app/desired_state_file_loader.go | 9 ++++++--- pkg/state/create.go | 13 +++++++------ pkg/state/create_test.go | 7 ++++--- pkg/state/state.go | 1 + 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/pkg/app/app.go b/pkg/app/app.go index 72c7a88c..aa6154de 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -2,9 +2,6 @@ package app import ( "fmt" - "github.com/roboll/helmfile/pkg/helmexec" - "github.com/roboll/helmfile/pkg/remote" - "github.com/roboll/helmfile/pkg/state" "io/ioutil" "log" "os" @@ -12,6 +9,10 @@ import ( "strings" "syscall" + "github.com/roboll/helmfile/pkg/helmexec" + "github.com/roboll/helmfile/pkg/remote" + "github.com/roboll/helmfile/pkg/state" + "go.uber.org/zap" "path/filepath" @@ -239,6 +240,7 @@ func (a *App) loadDesiredStateFromYaml(file string, opts ...LoadOpts) (*state.He Reverse: a.Reverse, KubeContext: a.KubeContext, glob: a.glob, + helm: a.helmExecer, } var op LoadOpts diff --git a/pkg/app/desired_state_file_loader.go b/pkg/app/desired_state_file_loader.go index 2af933d7..6264aad8 100644 --- a/pkg/app/desired_state_file_loader.go +++ b/pkg/app/desired_state_file_loader.go @@ -4,12 +4,14 @@ import ( "bytes" "errors" "fmt" + "path/filepath" + "sort" + "github.com/imdario/mergo" "github.com/roboll/helmfile/pkg/environment" + "github.com/roboll/helmfile/pkg/helmexec" "github.com/roboll/helmfile/pkg/state" "go.uber.org/zap" - "path/filepath" - "sort" ) type desiredStateLoader struct { @@ -25,6 +27,7 @@ type desiredStateLoader struct { glob func(string) ([]string, error) logger *zap.SugaredLogger + helm helmexec.Interface } func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, error) { @@ -125,7 +128,7 @@ func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *e } func (a *desiredStateLoader) underlying() *state.StateCreator { - c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob) + c := state.NewCreator(a.logger, a.readFile, a.fileExists, a.abs, a.glob, a.helm) c.LoadFile = a.loadFile return c } diff --git a/pkg/state/create.go b/pkg/state/create.go index be3ef4ad..d798c61e 100644 --- a/pkg/state/create.go +++ b/pkg/state/create.go @@ -38,13 +38,14 @@ type StateCreator struct { fileExists func(string) (bool, error) abs func(string) (string, error) glob func(string) ([]string, error) + helm helmexec.Interface Strict bool LoadFile func(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*HelmState, error) } -func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error)) *StateCreator { +func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error), fileExists func(string) (bool, error), abs func(string) (string, error), glob func(string) ([]string, error), helm helmexec.Interface) *StateCreator { return &StateCreator{ logger: logger, readFile: readFile, @@ -52,6 +53,7 @@ func NewCreator(logger *zap.SugaredLogger, readFile func(string) ([]byte, error) abs: abs, glob: glob, Strict: true, + helm: helm, } } @@ -61,6 +63,7 @@ func (c *StateCreator) Parse(content []byte, baseDir, file string) (*HelmState, state.FilePath = file state.basePath = baseDir + state.helm = c.helm decoder := yaml.NewDecoder(bytes.NewReader(content)) if !c.Strict { @@ -186,9 +189,6 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, } if len(envSpec.Secrets) > 0 { - helm := helmexec.New(st.logger, "", &helmexec.ShellRunner{ - Logger: st.logger, - }) var envSecretFiles []string for _, urlOrPath := range envSpec.Secrets { @@ -202,7 +202,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, envSecretFiles = append(envSecretFiles, resolved...) } - if err = st.scatterGatherEnvSecretFiles(envSecretFiles, helm, envVals, readFile); err != nil { + if err = st.scatterGatherEnvSecretFiles(envSecretFiles, envVals, readFile); err != nil { return nil, err } } @@ -225,7 +225,7 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment, return newEnv, nil } -func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, helm helmexec.Interface, envVals map[string]interface{}, readFile func(string) ([]byte, error)) error { +func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, envVals map[string]interface{}, readFile func(string) ([]byte, error)) error { var errs []error inputs := envSecretFiles @@ -239,6 +239,7 @@ func (st *HelmState) scatterGatherEnvSecretFiles(envSecretFiles []string, helm h secrets := make(chan string, inputsSize) results := make(chan secretResult, inputsSize) + helm := st.helm st.scatterGather(0, inputsSize, func() { diff --git a/pkg/state/create_test.go b/pkg/state/create_test.go index 76863a3c..3a8ca395 100644 --- a/pkg/state/create_test.go +++ b/pkg/state/create_test.go @@ -1,13 +1,14 @@ package state import ( - "github.com/roboll/helmfile/pkg/testhelper" - "go.uber.org/zap" "io/ioutil" "path/filepath" "reflect" "testing" + "github.com/roboll/helmfile/pkg/testhelper" + "go.uber.org/zap" + . "gotest.tools/assert" "gotest.tools/assert/cmp" ) @@ -107,7 +108,7 @@ bar: {{ readFile "bar.txt" }} }) testFs.Cwd = "/example/path/to" - state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil) + state, err := NewCreator(logger, testFs.ReadFile, testFs.FileExists, testFs.Abs, testFs.Glob, nil).ParseAndLoad(yamlContent, filepath.Dir(yamlFile), yamlFile, "production", false, nil) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/pkg/state/state.go b/pkg/state/state.go index a6ae762e..bc01822d 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -62,6 +62,7 @@ type HelmState struct { tempDir func(string, string) (string, error) runner helmexec.Runner + helm helmexec.Interface } // SubHelmfileSpec defines the subhelmfile path and options From cd5d906afbf26af1733621c4e5cba63cd1921f73 Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Sat, 24 Aug 2019 09:47:49 +0900 Subject: [PATCH 10/18] fix: clean up invalid remote state file cache Fixes #815 --- go.mod | 2 +- pkg/remote/remote.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 00ab7813..c0a9a53a 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939 github.com/urfave/cli v0.0.0-20160620154522-6011f165dc28 go.uber.org/atomic v1.3.2 // indirect - go.uber.org/multierr v1.1.0 // indirect + go.uber.org/multierr v1.1.0 go.uber.org/zap v1.8.0 gopkg.in/yaml.v2 v2.2.1 gotest.tools v2.2.0+incompatible diff --git a/pkg/remote/remote.go b/pkg/remote/remote.go index 0f70dc4a..f81c44f7 100644 --- a/pkg/remote/remote.go +++ b/pkg/remote/remote.go @@ -6,8 +6,10 @@ import ( "fmt" "github.com/hashicorp/go-getter" "github.com/hashicorp/go-getter/helper/url" + "go.uber.org/multierr" "go.uber.org/zap" "gopkg.in/yaml.v2" + "os" "path/filepath" "strings" ) @@ -180,8 +182,10 @@ func (r *Remote) Fetch(goGetterSrc string) (string, error) { cached := false + // e.g. .helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.xx.0 getterDst := filepath.Join(cacheBaseDir, cacheKey) + // e.g. $PWD/.helmfile/cache/https_github_com_cloudposse_helmfiles_git.ref=0.xx.0 cacheDirPath := filepath.Join(r.Home, getterDst) r.Logger.Debugf("home: %s", r.Home) @@ -217,6 +221,10 @@ func (r *Remote) Fetch(goGetterSrc string) (string, error) { r.Logger.Debugf("downloading %s to %s", getterSrc, getterDst) if err := r.Getter.Get(r.Home, getterSrc, getterDst); err != nil { + rmerr := os.RemoveAll(cacheDirPath) + if rmerr != nil { + return "", multierr.Append(err, rmerr) + } return "", err } } From bf9dcc09821b9344cc321bf4757ec7eaa3195543 Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Sat, 24 Aug 2019 09:59:46 +0900 Subject: [PATCH 11/18] release: Make the release process faster --- Makefile | 4 ++-- go.sum | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f8b20095..e2c08807 100644 --- a/Makefile +++ b/Makefile @@ -26,11 +26,11 @@ integration: .PHONY: integration cross: - env CGO_ENABLED=0 gox -os '!freebsd !netbsd' -arch '!arm' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS} + env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS} .PHONY: cross static-linux: - env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o "dist/helmfile_linux_amd64" -ldflags '-X main.Version=${TAG}' ${TARGETS} + env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -o "dist/helmfile_linux_amd64" -ldflags '-X main.Version=${TAG}' ${TARGETS} .PHONY: linux install: diff --git a/go.sum b/go.sum index 0436570f..6e699170 100644 --- a/go.sum +++ b/go.sum @@ -171,6 +171,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 h1:0oC8rFnE+74kEmuHZ46F6KHsMr5Gx2gUQPuNz28iQZM= golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= @@ -185,6 +186,7 @@ google.golang.org/api v0.1.0 h1:K6z2u68e86TPdSdefXdzvXgR1zEMa+459vBSfWYAZkI= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= From ba0891b7b7c893d5e9e646675f71809ecc2392f4 Mon Sep 17 00:00:00 2001 From: KUOKA Yusuke Date: Sat, 24 Aug 2019 10:29:26 +0900 Subject: [PATCH 12/18] release: skip downloading go modules in container image builds (#821) So that the release process can be (probably) 2x faster --- Dockerfile | 11 +---------- Makefile | 8 ++++++-- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 063d34af..5681134f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,3 @@ -FROM golang:1.12.4-alpine3.9 as builder - -RUN apk add --no-cache make git -WORKDIR /workspace/helmfile -COPY . /workspace/helmfile -RUN make static-linux - -# ----------------------------------------------------------------------------- - FROM alpine:3.8 RUN apk add --no-cache ca-certificates git bash curl jq @@ -36,6 +27,6 @@ RUN helm plugin install https://github.com/databus23/helm-diff && \ helm plugin install https://github.com/hypnoglow/helm-s3.git && \ helm plugin install https://github.com/aslafy-z/helm-git.git -COPY --from=builder /workspace/helmfile/dist/helmfile_linux_amd64 /usr/local/bin/helmfile +COPY dist/helmfile_linux_amd64 /usr/local/bin/helmfile CMD ["/usr/local/bin/helmfile"] diff --git a/Makefile b/Makefile index e2c08807..fe1a673b 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ integration: .PHONY: integration cross: - env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS} + env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le !s390x' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS} .PHONY: cross static-linux: @@ -50,9 +50,13 @@ release: pristine cross @ghr -b ${BODY} -t ${GITHUB_TOKEN} -u ${ORG} -recreate ${TAG} dist .PHONY: release -image: +image: static-linux/in-docker docker build -t quay.io/${ORG}/helmfile:${TAG} . +static-linux/in-docker: + if [ -f dist/helmfile_linux_amd64 ]; then echo removing dist/helmfile_linux_amd64; rm dist/helmfile_linux_amd64; fi + docker run -v $(HOME)/go:/go -v $(PWD):/workspace/helmfile -w /workspace/helmfile golang:1.12.4-alpine3.9 sh -c 'apk add --no-cache make git && make static-linux' + run: image docker run --rm -it -t quay.io/${ORG}/helmfile:${TAG} sh From e24c15324be53610d71e46e76883040b95bf4c23 Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Sat, 24 Aug 2019 10:37:01 +0900 Subject: [PATCH 13/18] Revert "release: skip downloading go modules in container image builds (#821)" This reverts commit ba0891b7b7c893d5e9e646675f71809ecc2392f4. --- Dockerfile | 11 ++++++++++- Makefile | 8 ++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5681134f..063d34af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,12 @@ +FROM golang:1.12.4-alpine3.9 as builder + +RUN apk add --no-cache make git +WORKDIR /workspace/helmfile +COPY . /workspace/helmfile +RUN make static-linux + +# ----------------------------------------------------------------------------- + FROM alpine:3.8 RUN apk add --no-cache ca-certificates git bash curl jq @@ -27,6 +36,6 @@ RUN helm plugin install https://github.com/databus23/helm-diff && \ helm plugin install https://github.com/hypnoglow/helm-s3.git && \ helm plugin install https://github.com/aslafy-z/helm-git.git -COPY dist/helmfile_linux_amd64 /usr/local/bin/helmfile +COPY --from=builder /workspace/helmfile/dist/helmfile_linux_amd64 /usr/local/bin/helmfile CMD ["/usr/local/bin/helmfile"] diff --git a/Makefile b/Makefile index fe1a673b..e2c08807 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ integration: .PHONY: integration cross: - env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le !s390x' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS} + env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS} .PHONY: cross static-linux: @@ -50,13 +50,9 @@ release: pristine cross @ghr -b ${BODY} -t ${GITHUB_TOKEN} -u ${ORG} -recreate ${TAG} dist .PHONY: release -image: static-linux/in-docker +image: docker build -t quay.io/${ORG}/helmfile:${TAG} . -static-linux/in-docker: - if [ -f dist/helmfile_linux_amd64 ]; then echo removing dist/helmfile_linux_amd64; rm dist/helmfile_linux_amd64; fi - docker run -v $(HOME)/go:/go -v $(PWD):/workspace/helmfile -w /workspace/helmfile golang:1.12.4-alpine3.9 sh -c 'apk add --no-cache make git && make static-linux' - run: image docker run --rm -it -t quay.io/${ORG}/helmfile:${TAG} sh From dd58badf8123417f9988521961b050bf18a8520c Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Sat, 24 Aug 2019 10:50:18 +0900 Subject: [PATCH 14/18] release: skip building against s390x Please notice me if anyone is using Helmfile on it --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e2c08807..6558ba2a 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ integration: .PHONY: integration cross: - env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS} + env CGO_ENABLED=0 gox -os '!openbsd !freebsd !netbsd' -arch '!arm !mips !mipsle !mips64 !mips64le !s390x' -output "dist/{{.Dir}}_{{.OS}}_{{.Arch}}" -ldflags '-X main.Version=${TAG}' ${TARGETS} .PHONY: cross static-linux: From 11d0abba6e045787dc21d825945b627fdbcfc129 Mon Sep 17 00:00:00 2001 From: astorath Date: Sat, 31 Aug 2019 08:31:31 +0300 Subject: [PATCH 15/18] feat: Advanced Templating (#823) 1. Added `helmfile build` command to print final state Motivation: useful for debugging purposes and some CI scenarios Ref #780 2. Template interpolation is now recursive (you can cross-reference release fields) like: ```yaml templates: release: name: {{`app-{{ .Release.Namespace }}`}} namespace: {{`{{ .Release.Labels.ns }}`}} labels: ns: dev ``` 3. Experimental: Added some boolean release fields interpolation in templates: ```yaml templates: release: name: {{`app-{{ .Release.Namespace }}`}} namespace: dev installedTemplate: {{`{{ eq .Release.Namespace "dev" }}`}} ``` Resolves #818 4. Added more template interpolations: Labels, SetValues 5. Added template interpolation for inline Values 6. Added `helmfile list` command to print target releases in simple tabular form 7. Added release names in some `helm` output messages, e.g.: `Comparing release=%v, chart=%v` --- .gitignore | 4 +- Makefile | 2 +- docs/writing-helmfile.md | 36 ++++++ go.mod | 3 +- go.sum | 24 ++-- main.go | 21 +++- pkg/app/app.go | 33 +++++ pkg/app/app_test.go | 175 +++++++++++++++++++++++++- pkg/app/config.go | 3 + pkg/app/run.go | 3 +- pkg/helmexec/exec.go | 25 ++-- pkg/helmexec/exec_test.go | 31 ++--- pkg/helmexec/helmexec.go | 6 +- pkg/state/environment.go | 6 +- pkg/state/release.go | 101 +++++++++++++++ pkg/state/state.go | 151 +++++++++++++---------- pkg/state/state_exec_tmpl.go | 85 +++++++++++-- pkg/state/state_exec_tmpl_test.go | 199 ++++++++++++++++++++++++++++-- pkg/state/state_test.go | 6 +- 19 files changed, 769 insertions(+), 145 deletions(-) diff --git a/.gitignore b/.gitignore index 3e42cef0..e3f7895a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ dist/ .idea/ -helmfile \ No newline at end of file +helmfile +helmfile.lock +vendor/ diff --git a/Makefile b/Makefile index 6558ba2a..930596e4 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ cross: static-linux: env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -o "dist/helmfile_linux_amd64" -ldflags '-X main.Version=${TAG}' ${TARGETS} -.PHONY: linux +.PHONY: static-linux install: env CGO_ENABLED=0 go install -ldflags '-X main.Version=${TAG}' ${TARGETS} diff --git a/docs/writing-helmfile.md b/docs/writing-helmfile.md index cd5f77ff..285c4124 100644 --- a/docs/writing-helmfile.md +++ b/docs/writing-helmfile.md @@ -89,6 +89,42 @@ releases: <<: *default ``` +Release Templating supports the following parts of release definition: +- basic fields: `name`, `namespace`, `chart`, `version` +- boolean fields: `installed`, `wait`, `tillerless`, `verify` by the means of additional text + fields designed for templating only: `installedTemplate`, `waitTemplate`, `tillerlessTemplate`, `verifyTemplate` + ```yaml + # ... + installedTemplate: '{{`{{ eq .Release.Namespace "kube-system" }}`}}' + waitTemplate: '{{`{{ eq .Release.Labels.tag "safe" | not }}`}}' + # ... + ``` +- `set` block values: + ```yaml + # ... + set: + - name: '{{`{{ .Release.Name }}`}}' + values: '{{`{{ .Release.Namespace }}`}}' + # ... + ``` +- `values` and `secrets` file paths: + ```yaml + # ... + values: + - config/{{`{{ .Release.Name }}`}}/values.yaml + secrets: + - config/{{`{{ .Release.Name }}`}}/secrets.yaml + # ... + ``` +- inline `values` map: + ```yaml + # ... + values: + - image: + tag: `{{ .Release.Labels.tag }}` + # ... + ``` + See the [issue 428](https://github.com/roboll/helmfile/issues/428) for more context on how this is supposed to work. ## Layering State Files diff --git a/go.mod b/go.mod index c0a9a53a..43f6d81a 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,10 @@ require ( github.com/Masterminds/goutils v1.1.0 // indirect github.com/Masterminds/semver v1.4.1 github.com/Masterminds/sprig v2.20.0+incompatible - github.com/aokoli/goutils v1.0.1 // indirect + github.com/go-test/deep v1.0.3 github.com/google/go-cmp v0.3.0 github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c // indirect + github.com/gosuri/uitable v0.0.3 github.com/hashicorp/go-getter v1.3.0 github.com/huandu/xstrings v1.2.0 // indirect github.com/imdario/mergo v0.3.6 diff --git a/go.sum b/go.sum index 6e699170..aec086d0 100644 --- a/go.sum +++ b/go.sum @@ -12,15 +12,9 @@ github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RP github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.4.1 h1:CaDA1wAoM3rj9sAFyyZP37LloExUzxFGYt+DqJ870JA= github.com/Masterminds/semver v1.4.1/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/sprig v2.15.0+incompatible h1:0gSxPGWS9PAr7U2NsQ2YQg6juRDINkUyuvbb4b2Xm8w= -github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= -github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY= -github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Masterminds/sprig v2.20.0+incompatible h1:dJTKKuUkYW3RMFdQFXPU/s6hg10RgctmTjRcbZ98Ap8= github.com/Masterminds/sprig v2.20.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= -github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/aws/aws-sdk-go v1.15.78 h1:LaXy6lWR0YK7LKyuU0QWy2ws/LWTPfYV/UgfiBu4tvY= github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -30,14 +24,19 @@ github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBT github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= +github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -50,6 +49,7 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c h1:jWtZjFEUE/Bz0IeIhqCnyZ3HG6KRXSntXe4SjtuTH7c= @@ -59,6 +59,8 @@ github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk github.com/googleapis/gax-go/v2 v2.0.3 h1:siORttZ36U2R/WjiJuDz8znElWBiAlO9rVt+mqJt0Cc= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gosuri/uitable v0.0.3 h1:9ZY4qCODg6JL1Ui4dL9LqCF4ghWnAOSV2h7xG98SkHE= +github.com/gosuri/uitable v0.0.3/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= @@ -69,8 +71,6 @@ github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhE github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= -github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= @@ -83,7 +83,9 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -98,6 +100,7 @@ github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -129,6 +132,7 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tatsushid/go-prettytable v0.0.0-20141013043238-ed2d14c29939 h1:BhIUXV2ySTLrKgh/Hnts+QTQlIbWtomXt3LMdzME0A0= @@ -147,8 +151,6 @@ go.uber.org/zap v1.8.0 h1:r6Za1Rii8+EGOYRDLvpooNOF6kP3iyDnkpzbw67gCQ8= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= -golang.org/x/crypto v0.0.0-20180403160946-b2aa35443fbc h1:Kx1Ke+iCR1aDjbWXgmEQGFxoHtNL49aRZGV7/+jJ41Y= -golang.org/x/crypto v0.0.0-20180403160946-b2aa35443fbc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -167,6 +169,7 @@ golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcp golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -198,6 +201,7 @@ google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmE google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= diff --git a/main.go b/main.go index f91a3b24..c05b9104 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,9 @@ package main import ( "fmt" + "os" + "strings" + "github.com/roboll/helmfile/pkg/app" "github.com/roboll/helmfile/pkg/helmexec" "github.com/roboll/helmfile/pkg/maputil" @@ -9,8 +12,6 @@ import ( "github.com/urfave/cli" "go.uber.org/zap" "go.uber.org/zap/zapcore" - "os" - "strings" ) var Version string @@ -392,6 +393,22 @@ func main() { return run.Test(c) }), }, + { + Name: "build", + Usage: "output compiled helmfile state(s) as YAML", + Flags: []cli.Flag{}, + Action: action(func(run *app.App, c configImpl) error { + return run.PrintState(c) + }), + }, + { + Name: "list", + Usage: "list releases defined in state file", + Flags: []cli.Flag{}, + Action: action(func(run *app.App, c configImpl) error { + return run.ListReleases(c) + }), + }, } err := cliApp.Run(os.Args) diff --git a/pkg/app/app.go b/pkg/app/app.go index aa6154de..7c725c43 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -9,6 +9,7 @@ import ( "strings" "syscall" + "github.com/gosuri/uitable" "github.com/roboll/helmfile/pkg/helmexec" "github.com/roboll/helmfile/pkg/remote" "github.com/roboll/helmfile/pkg/state" @@ -158,6 +159,38 @@ func (a *App) Test(c TestConfigProvider) error { }) } +func (a *App) PrintState(c StateConfigProvider) error { + + return a.ForEachState(func(run *Run) []error { + state, err := run.state.ToYaml() + if err != nil { + return []error{err} + } + fmt.Printf("---\n# Source: %s\n\n%+v", run.state.FilePath, state) + return []error{} + }) +} + +func (a *App) ListReleases(c StateConfigProvider) error { + table := uitable.New() + table.AddRow("NAME", "NAMESPACE", "INSTALLED", "LABELS") + + err := a.ForEachState(func(run *Run) []error { + //var releases m + for _, r := range run.state.Releases { + labels := "" + for k, v := range r.Labels { + labels = fmt.Sprintf("%s,%s:%s", labels, k, v) + } + installed := r.Installed == nil || *r.Installed + table.AddRow(r.Name, r.Namespace, fmt.Sprintf("%t", installed), strings.Trim(labels, ",")) + } + return []error{} + }) + fmt.Println(table.String()) + return err +} + func (a *App) within(dir string, do func() error) error { if dir == "." { return do() diff --git a/pkg/app/app_test.go b/pkg/app/app_test.go index 8c6ca93f..06dea601 100644 --- a/pkg/app/app_test.go +++ b/pkg/app/app_test.go @@ -3,15 +3,21 @@ package app import ( "bytes" "fmt" - "github.com/roboll/helmfile/pkg/helmexec" - "github.com/roboll/helmfile/pkg/state" - "github.com/roboll/helmfile/pkg/testhelper" + "gotest.tools/assert" + "io" + "log" "os" "path/filepath" "reflect" "regexp" + "strings" + "sync" "testing" + "github.com/roboll/helmfile/pkg/helmexec" + "github.com/roboll/helmfile/pkg/state" + "github.com/roboll/helmfile/pkg/testhelper" + "go.uber.org/zap" "gotest.tools/env" ) @@ -1840,7 +1846,7 @@ type mockTemplates struct { flags []string } -func (helm *mockHelmExec) TemplateRelease(chart string, flags ...string) error { +func (helm *mockHelmExec) TemplateRelease(name, chart string, flags ...string) error { helm.templated = append(helm.templated, mockTemplates{flags: flags}) return nil } @@ -1849,7 +1855,7 @@ func (helm *mockHelmExec) UpdateDeps(chart string) error { return nil } -func (helm *mockHelmExec) BuildDeps(chart string) error { +func (helm *mockHelmExec) BuildDeps(name, chart string) error { return nil } @@ -1889,7 +1895,7 @@ func (helm *mockHelmExec) TestRelease(context helmexec.HelmContext, name string, func (helm *mockHelmExec) Fetch(chart string, flags ...string) error { return nil } -func (helm *mockHelmExec) Lint(chart string, flags ...string) error { +func (helm *mockHelmExec) Lint(name, chart string, flags ...string) error { return nil } @@ -1938,3 +1944,160 @@ releases: } } + +func captureStdout(f func()) string { + reader, writer, err := os.Pipe() + if err != nil { + panic(err) + } + stdout := os.Stdout + defer func() { + os.Stdout = stdout + log.SetOutput(os.Stderr) + }() + os.Stdout = writer + log.SetOutput(writer) + out := make(chan string) + wg := new(sync.WaitGroup) + wg.Add(1) + go func() { + var buf bytes.Buffer + wg.Done() + io.Copy(&buf, reader) + out <- buf.String() + }() + wg.Wait() + f() + writer.Close() + return <-out +} + +func TestPrint_SingleStateFile(t *testing.T) { + files := map[string]string{ + "/path/to/helmfile.yaml": ` +releases: +- name: myrelease1 + chart: mychart1 +- name: myrelease2 + chart: mychart1 +`, + } + stdout := os.Stdout + defer func() { os.Stdout = stdout }() + + var buffer bytes.Buffer + logger := helmexec.NewLogger(&buffer, "debug") + + app := appWithFs(&App{ + glob: filepath.Glob, + abs: filepath.Abs, + KubeContext: "default", + Env: "default", + Logger: logger, + Namespace: "testNamespace", + }, files) + out := captureStdout(func() { + err := app.PrintState(configImpl{}) + assert.NilError(t, err) + }) + assert.Assert(t, strings.Count(out, "---") == 1, + "state should contain '---' yaml doc separator:\n%s\n", out) + assert.Assert(t, strings.Contains(out, "helmfile.yaml"), + "state should contain source helmfile name:\n%s\n", out) + assert.Assert(t, strings.Contains(out, "name: myrelease1"), + "state should contain releases:\n%s\n", out) +} + +func TestPrint_MultiStateFile(t *testing.T) { + files := map[string]string{ + "/path/to/helmfile.d/first.yaml": ` +releases: +- name: myrelease1 + chart: mychart1 +- name: myrelease2 + chart: mychart1 +`, + "/path/to/helmfile.d/second.yaml": ` +releases: +- name: myrelease3 + chart: mychart1 +- name: myrelease4 + chart: mychart1 +`, + } + stdout := os.Stdout + defer func() { os.Stdout = stdout }() + + var buffer bytes.Buffer + logger := helmexec.NewLogger(&buffer, "debug") + + app := appWithFs(&App{ + glob: filepath.Glob, + abs: filepath.Abs, + KubeContext: "default", + Env: "default", + Logger: logger, + Namespace: "testNamespace", + }, files) + out := captureStdout(func() { + err := app.PrintState(configImpl{}) + assert.NilError(t, err) + }) + assert.Assert(t, strings.Count(out, "---") == 2, + "state should contain '---' yaml doc separators:\n%s\n", out) + assert.Assert(t, strings.Contains(out, "second.yaml"), + "state should contain source helmfile name:\n%s\n", out) + assert.Assert(t, strings.Contains(out, "second.yaml"), + "state should contain source helmfile name:\n%s\n", out) +} + +func TestList(t *testing.T) { + files := map[string]string{ + "/path/to/helmfile.d/first.yaml": ` +releases: +- name: myrelease1 + chart: mychart1 + installed: no + labels: + id: myrelease1 +- name: myrelease2 + chart: mychart1 +`, + "/path/to/helmfile.d/second.yaml": ` +releases: +- name: myrelease3 + chart: mychart1 + installed: yes +- name: myrelease4 + chart: mychart1 + labels: + id: myrelease1 +`, + } + stdout := os.Stdout + defer func() { os.Stdout = stdout }() + + var buffer bytes.Buffer + logger := helmexec.NewLogger(&buffer, "debug") + + app := appWithFs(&App{ + glob: filepath.Glob, + abs: filepath.Abs, + KubeContext: "default", + Env: "default", + Logger: logger, + Namespace: "testNamespace", + }, files) + out := captureStdout(func() { + err := app.ListReleases(configImpl{}) + assert.NilError(t, err) + }) + + expected := `NAME NAMESPACE INSTALLED LABELS +myrelease1 false id:myrelease1 +myrelease2 true +myrelease3 true +myrelease4 true id:myrelease1 +` + assert.Equal(t, expected, out) +} diff --git a/pkg/app/config.go b/pkg/app/config.go index a6e1b479..fbac5c64 100644 --- a/pkg/app/config.go +++ b/pkg/app/config.go @@ -120,6 +120,9 @@ type StatusesConfigProvider interface { concurrencyConfig } +type StateConfigProvider interface { +} + type concurrencyConfig interface { Concurrency() int } diff --git a/pkg/app/run.go b/pkg/app/run.go index a3c7b61e..a47c8176 100644 --- a/pkg/app/run.go +++ b/pkg/app/run.go @@ -2,10 +2,11 @@ package app import ( "fmt" + "strings" + "github.com/roboll/helmfile/pkg/argparser" "github.com/roboll/helmfile/pkg/helmexec" "github.com/roboll/helmfile/pkg/state" - "strings" ) type Run struct { diff --git a/pkg/helmexec/exec.go b/pkg/helmexec/exec.go index 9a72f659..e732b477 100644 --- a/pkg/helmexec/exec.go +++ b/pkg/helmexec/exec.go @@ -90,22 +90,22 @@ func (helm *execer) UpdateRepo() error { return err } -func (helm *execer) UpdateDeps(chart string) error { - helm.logger.Infof("Updating dependency %v", chart) - out, err := helm.exec([]string{"dependency", "update", chart}, map[string]string{}) +func (helm *execer) BuildDeps(name, chart string) error { + helm.logger.Infof("Building dependency release=%v, chart=%v", name, chart) + out, err := helm.exec([]string{"dependency", "build", chart}, map[string]string{}) helm.info(out) return err } -func (helm *execer) BuildDeps(chart string) error { - helm.logger.Infof("Building dependency %v", chart) - out, err := helm.exec([]string{"dependency", "build", chart}, map[string]string{}) +func (helm *execer) UpdateDeps(chart string) error { + helm.logger.Infof("Updating dependency %v", chart) + out, err := helm.exec([]string{"dependency", "update", chart}, map[string]string{}) helm.info(out) return err } func (helm *execer) SyncRelease(context HelmContext, name, chart string, flags ...string) error { - helm.logger.Infof("Upgrading %v", chart) + helm.logger.Infof("Upgrading release=%v, chart=%v", name, chart) preArgs := context.GetTillerlessArgs(helm.helmBinary) env := context.getTillerlessEnv() out, err := helm.exec(append(append(preArgs, "upgrade", "--install", "--reset-values", name, chart), flags...), env) @@ -199,14 +199,15 @@ func (helm *execer) DecryptSecret(context HelmContext, name string, flags ...str return tmpFile.Name(), err } -func (helm *execer) TemplateRelease(chart string, flags ...string) error { - out, err := helm.exec(append([]string{"template", chart}, flags...), map[string]string{}) +func (helm *execer) TemplateRelease(name string, chart string, flags ...string) error { + helm.logger.Infof("Templating release=%v, chart=%v", name, chart) + out, err := helm.exec(append([]string{"template", chart, "--name", name}, flags...), map[string]string{}) helm.write(out) return err } func (helm *execer) DiffRelease(context HelmContext, name, chart string, flags ...string) error { - helm.logger.Infof("Comparing %v %v", name, chart) + helm.logger.Infof("Comparing release=%v, chart=%v", name, chart) preArgs := context.GetTillerlessArgs(helm.helmBinary) env := context.getTillerlessEnv() out, err := helm.exec(append(append(preArgs, "diff", "upgrade", "--reset-values", "--allow-unreleased", name, chart), flags...), env) @@ -233,8 +234,8 @@ func (helm *execer) DiffRelease(context HelmContext, name, chart string, flags . return err } -func (helm *execer) Lint(chart string, flags ...string) error { - helm.logger.Infof("Linting %v", chart) +func (helm *execer) Lint(name, chart string, flags ...string) error { + helm.logger.Infof("Linting release=%v, chart=%v", name, chart) out, err := helm.exec(append([]string{"lint", chart}, flags...), map[string]string{}) helm.write(out) return err diff --git a/pkg/helmexec/exec_test.go b/pkg/helmexec/exec_test.go index 59144ae8..3be37cdb 100644 --- a/pkg/helmexec/exec_test.go +++ b/pkg/helmexec/exec_test.go @@ -135,7 +135,7 @@ func Test_SyncRelease(t *testing.T) { logger := NewLogger(&buffer, "debug") helm := MockExecer(logger, "dev") helm.SyncRelease(HelmContext{}, "release", "chart", "--timeout 10", "--wait") - expected := `Upgrading chart + expected := `Upgrading release=release, chart=chart exec: helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev exec: helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev: ` @@ -145,7 +145,7 @@ exec: helm upgrade --install --reset-values release chart --timeout 10 --wait -- buffer.Reset() helm.SyncRelease(HelmContext{}, "release", "chart") - expected = `Upgrading chart + expected = `Upgrading release=release, chart=chart exec: helm upgrade --install --reset-values release chart --kube-context dev exec: helm upgrade --install --reset-values release chart --kube-context dev: ` @@ -160,7 +160,7 @@ func Test_SyncReleaseTillerless(t *testing.T) { helm := MockExecer(logger, "dev") helm.SyncRelease(HelmContext{Tillerless: true, TillerNamespace: "foo"}, "release", "chart", "--timeout 10", "--wait") - expected := `Upgrading chart + expected := `Upgrading release=release, chart=chart exec: helm tiller run foo -- helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev exec: helm tiller run foo -- helm upgrade --install --reset-values release chart --timeout 10 --wait --kube-context dev: ` @@ -198,8 +198,8 @@ func Test_BuildDeps(t *testing.T) { var buffer bytes.Buffer logger := NewLogger(&buffer, "debug") helm := MockExecer(logger, "dev") - helm.BuildDeps("./chart/foo") - expected := `Building dependency ./chart/foo + helm.BuildDeps("foo", "./chart/foo") + expected := `Building dependency release=foo, chart=./chart/foo exec: helm dependency build ./chart/foo --kube-context dev exec: helm dependency build ./chart/foo --kube-context dev: ` @@ -209,8 +209,8 @@ exec: helm dependency build ./chart/foo --kube-context dev: buffer.Reset() helm.SetExtraArgs("--verify") - helm.BuildDeps("./chart/foo") - expected = `Building dependency ./chart/foo + helm.BuildDeps("foo", "./chart/foo") + expected = `Building dependency release=foo, chart=./chart/foo exec: helm dependency build ./chart/foo --verify --kube-context dev exec: helm dependency build ./chart/foo --verify --kube-context dev: ` @@ -248,7 +248,7 @@ func Test_DiffRelease(t *testing.T) { logger := NewLogger(&buffer, "debug") helm := MockExecer(logger, "dev") helm.DiffRelease(HelmContext{}, "release", "chart", "--timeout 10", "--wait") - expected := `Comparing release chart + expected := `Comparing release=release, chart=chart exec: helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev exec: helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev: ` @@ -258,7 +258,7 @@ exec: helm diff upgrade --reset-values --allow-unreleased release chart --timeou buffer.Reset() helm.DiffRelease(HelmContext{}, "release", "chart") - expected = `Comparing release chart + expected = `Comparing release=release, chart=chart exec: helm diff upgrade --reset-values --allow-unreleased release chart --kube-context dev exec: helm diff upgrade --reset-values --allow-unreleased release chart --kube-context dev: ` @@ -272,7 +272,7 @@ func Test_DiffReleaseTillerless(t *testing.T) { logger := NewLogger(&buffer, "debug") helm := MockExecer(logger, "dev") helm.DiffRelease(HelmContext{Tillerless: true}, "release", "chart", "--timeout 10", "--wait") - expected := `Comparing release chart + expected := `Comparing release=release, chart=chart exec: helm tiller run -- helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev exec: helm tiller run -- helm diff upgrade --reset-values --allow-unreleased release chart --timeout 10 --wait --kube-context dev: ` @@ -413,8 +413,8 @@ func Test_Lint(t *testing.T) { var buffer bytes.Buffer logger := NewLogger(&buffer, "debug") helm := MockExecer(logger, "dev") - helm.Lint("path/to/chart", "--values", "file.yml") - expected := `Linting path/to/chart + helm.Lint("release", "path/to/chart", "--values", "file.yml") + expected := `Linting release=release, chart=path/to/chart exec: helm lint path/to/chart --values file.yml --kube-context dev exec: helm lint path/to/chart --values file.yml --kube-context dev: ` @@ -498,9 +498,10 @@ func Test_Template(t *testing.T) { var buffer bytes.Buffer logger := NewLogger(&buffer, "debug") helm := MockExecer(logger, "dev") - helm.TemplateRelease("path/to/chart", "--values", "file.yml") - expected := `exec: helm template path/to/chart --values file.yml --kube-context dev -exec: helm template path/to/chart --values file.yml --kube-context dev: + helm.TemplateRelease("release", "path/to/chart", "--values", "file.yml") + expected := `Templating release=release, chart=path/to/chart +exec: helm template path/to/chart --name release --values file.yml --kube-context dev +exec: helm template path/to/chart --name release --values file.yml --kube-context dev: ` if buffer.String() != expected { t.Errorf("helmexec.Template()\nactual = %v\nexpect = %v", buffer.String(), expected) diff --git a/pkg/helmexec/helmexec.go b/pkg/helmexec/helmexec.go index 881be7f8..3ef820e9 100644 --- a/pkg/helmexec/helmexec.go +++ b/pkg/helmexec/helmexec.go @@ -7,13 +7,13 @@ type Interface interface { AddRepo(name, repository, certfile, keyfile, username, password string) error UpdateRepo() error - BuildDeps(chart string) error + BuildDeps(name, chart string) error UpdateDeps(chart string) error SyncRelease(context HelmContext, name, chart string, flags ...string) error DiffRelease(context HelmContext, name, chart string, flags ...string) error - TemplateRelease(chart string, flags ...string) error + TemplateRelease(name, chart string, flags ...string) error Fetch(chart string, flags ...string) error - Lint(chart string, flags ...string) error + Lint(name, chart string, flags ...string) error ReleaseStatus(context HelmContext, name string, flags ...string) error DeleteRelease(context HelmContext, name string, flags ...string) error TestRelease(context HelmContext, name string, flags ...string) error diff --git a/pkg/state/environment.go b/pkg/state/environment.go index 6fc1bd5c..13259f1d 100644 --- a/pkg/state/environment.go +++ b/pkg/state/environment.go @@ -1,8 +1,8 @@ package state type EnvironmentSpec struct { - Values []interface{} `yaml:"values"` - Secrets []string `yaml:"secrets"` + Values []interface{} `yaml:"values,omitempty"` + Secrets []string `yaml:"secrets,omitempty"` // MissingFileHandler instructs helmfile to fail when unable to find a environment values file listed // under `environments.NAME.values`. @@ -11,5 +11,5 @@ type EnvironmentSpec struct { // // Use "Warn", "Info", or "Debug" if you want helmfile to not fail when a values file is missing, while just leaving // a message about the missing file at the log-level. - MissingFileHandler *string `yaml:"missingFileHandler"` + MissingFileHandler *string `yaml:"missingFileHandler,omitempty"` } diff --git a/pkg/state/release.go b/pkg/state/release.go index 39ab4eef..6a954ba3 100644 --- a/pkg/state/release.go +++ b/pkg/state/release.go @@ -48,6 +48,51 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R } } + if result.WaitTemplate != nil { + ts := *result.WaitTemplate + resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err) + } + result.WaitTemplate = &resultTmpl + } + + if result.InstalledTemplate != nil { + ts := *result.InstalledTemplate + resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err) + } + result.InstalledTemplate = &resultTmpl + } + + if result.TillerlessTemplate != nil { + ts := *result.TillerlessTemplate + resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err) + } + result.TillerlessTemplate = &resultTmpl + } + + if result.VerifyTemplate != nil { + ts := *result.VerifyTemplate + resultTmpl, err := renderer.RenderTemplateContentToString([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".version = \"%s\": %v", r.Name, ts, err) + } + result.VerifyTemplate = &resultTmpl + } + + for key, val := range result.Labels { + ts := val + s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".labels[%s] = \"%s\": %v", r.Name, key, ts, err) + } + result.Labels[key] = s.String() + } + for i, t := range result.Values { switch ts := t.(type) { case string: @@ -56,6 +101,24 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%s\": %v", r.Name, i, ts, err) } result.Values[i] = s.String() + case map[interface{}]interface{}: + serialized, err := yaml.Marshal(ts) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err) + } + + s, err := renderer.RenderTemplateContentToBuffer([]byte(serialized)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, serialized, err) + } + + var deserialized map[interface{}]interface{} + + if err := yaml.Unmarshal(s.Bytes(), &deserialized); err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err) + } + + result.Values[i] = deserialized } } @@ -67,6 +130,44 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R result.Secrets[i] = s.String() } + for i, val := range result.SetValues { + { + // name + ts := val.Name + s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].name = \"%s\": %v", r.Name, i, ts, err) + } + result.SetValues[i].Name = s.String() + } + { + // value + ts := val.Value + s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].value = \"%s\": %v", r.Name, i, ts, err) + } + result.SetValues[i].Value = s.String() + } + { + // file + ts := val.File + s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].file = \"%s\": %v", r.Name, i, ts, err) + } + result.SetValues[i].File = s.String() + } + for j, ts := range val.Values { + // values + s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].values[%d] = \"%s\": %v", r.Name, i, j, ts, err) + } + result.SetValues[i].Values[j] = s.String() + } + } + return result, nil } diff --git a/pkg/state/state.go b/pkg/state/state.go index bc01822d..6a749626 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -34,23 +34,23 @@ type HelmState struct { FilePath string // DefaultValues is the default values to be overrode by environment values and command-line overrides - DefaultValues []interface{} `yaml:"values"` + DefaultValues []interface{} `yaml:"values,omitempty"` - Environments map[string]EnvironmentSpec `yaml:"environments"` + Environments map[string]EnvironmentSpec `yaml:"environments,omitempty"` - Bases []string `yaml:"bases"` - HelmDefaults HelmSpec `yaml:"helmDefaults"` - Helmfiles []SubHelmfileSpec `yaml:"helmfiles"` - DeprecatedContext string `yaml:"context"` - DeprecatedReleases []ReleaseSpec `yaml:"charts"` - Namespace string `yaml:"namespace"` - Repositories []RepositorySpec `yaml:"repositories"` - Releases []ReleaseSpec `yaml:"releases"` - Selectors []string + Bases []string `yaml:"bases,omitempty"` + HelmDefaults HelmSpec `yaml:"helmDefaults,omitempty"` + Helmfiles []SubHelmfileSpec `yaml:"helmfiles,omitempty"` + DeprecatedContext string `yaml:"context,omitempty"` + DeprecatedReleases []ReleaseSpec `yaml:"charts,omitempty"` + Namespace string `yaml:"namespace,omitempty"` + Repositories []RepositorySpec `yaml:"repositories,omitempty"` + Releases []ReleaseSpec `yaml:"releases,omitempty"` + Selectors []string `yaml:"-"` Templates map[string]TemplateSpec `yaml:"templates"` - Env environment.Environment + Env environment.Environment `yaml:"-"` logger *zap.SugaredLogger @@ -67,23 +67,26 @@ type HelmState struct { // SubHelmfileSpec defines the subhelmfile path and options type SubHelmfileSpec struct { - Path string //path or glob pattern for the sub helmfiles - Selectors []string //chosen selectors for the sub helmfiles - SelectorsInherited bool //do the sub helmfiles inherits from parent selectors + //path or glob pattern for the sub helmfiles + Path string `yaml:"path,omitempty"` + //chosen selectors for the sub helmfiles + Selectors []string `yaml:"selectors,omitempty"` + //do the sub helmfiles inherits from parent selectors + SelectorsInherited bool `yaml:"selectorsInherited,omitempty"` Environment SubhelmfileEnvironmentSpec } type SubhelmfileEnvironmentSpec struct { - OverrideValues []interface{} `yaml:"values"` + OverrideValues []interface{} `yaml:"values,omitempty"` } // HelmSpec to defines helmDefault values type HelmSpec struct { - KubeContext string `yaml:"kubeContext"` - TillerNamespace string `yaml:"tillerNamespace"` + KubeContext string `yaml:"kubeContext,omitempty"` + TillerNamespace string `yaml:"tillerNamespace,omitempty"` Tillerless bool `yaml:"tillerless"` - Args []string `yaml:"args"` + Args []string `yaml:"args,omitempty"` Verify bool `yaml:"verify"` // Devel, when set to true, use development versions, too. Equivalent to version '>0.0.0-0' Devel bool `yaml:"devel"` @@ -99,77 +102,83 @@ type HelmSpec struct { Atomic bool `yaml:"atomic"` TLS bool `yaml:"tls"` - TLSCACert string `yaml:"tlsCACert"` - TLSKey string `yaml:"tlsKey"` - TLSCert string `yaml:"tlsCert"` + TLSCACert string `yaml:"tlsCACert,omitempty"` + TLSKey string `yaml:"tlsKey,omitempty"` + TLSCert string `yaml:"tlsCert,omitempty"` } // RepositorySpec that defines values for a helm repo type RepositorySpec struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - CertFile string `yaml:"certFile"` - KeyFile string `yaml:"keyFile"` - Username string `yaml:"username"` - Password string `yaml:"password"` + Name string `yaml:"name,omitempty"` + URL string `yaml:"url,omitempty"` + CertFile string `yaml:"certFile,omitempty"` + KeyFile string `yaml:"keyFile,omitempty"` + Username string `yaml:"username,omitempty"` + Password string `yaml:"password,omitempty"` } // ReleaseSpec defines the structure of a helm release type ReleaseSpec struct { // Chart is the name of the chart being installed to create this release - Chart string `yaml:"chart"` - Version string `yaml:"version"` - Verify *bool `yaml:"verify"` + Chart string `yaml:"chart,omitempty"` + Version string `yaml:"version,omitempty"` + Verify *bool `yaml:"verify,omitempty"` // Devel, when set to true, use development versions, too. Equivalent to version '>0.0.0-0' - Devel *bool `yaml:"devel"` + Devel *bool `yaml:"devel,omitempty"` // Wait, if set to true, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment are in a ready state before marking the release as successful - Wait *bool `yaml:"wait"` + Wait *bool `yaml:"wait,omitempty"` // Timeout is the time in seconds to wait for any individual Kubernetes operation (like Jobs for hooks, and waits on pod/pvc/svc/deployment readiness) (default 300) - Timeout *int `yaml:"timeout"` + Timeout *int `yaml:"timeout,omitempty"` // RecreatePods, when set to true, instruct helmfile to perform pods restart for the resource if applicable - RecreatePods *bool `yaml:"recreatePods"` + RecreatePods *bool `yaml:"recreatePods,omitempty"` // Force, when set to true, forces resource update through delete/recreate if needed - Force *bool `yaml:"force"` + Force *bool `yaml:"force,omitempty"` // Installed, when set to true, `delete --purge` the release - Installed *bool `yaml:"installed"` + Installed *bool `yaml:"installed,omitempty"` // Atomic, when set to true, restore previous state in case of a failed install/upgrade attempt - Atomic *bool `yaml:"atomic"` + Atomic *bool `yaml:"atomic,omitempty"` // MissingFileHandler is set to either "Error" or "Warn". "Error" instructs helmfile to fail when unable to find a values or secrets file. When "Warn", it prints the file and continues. // The default value for MissingFileHandler is "Error". - MissingFileHandler *string `yaml:"missingFileHandler"` + MissingFileHandler *string `yaml:"missingFileHandler,omitempty"` // Hooks is a list of extension points paired with operations, that are executed in specific points of the lifecycle of releases defined in helmfile - Hooks []event.Hook `yaml:"hooks"` + Hooks []event.Hook `yaml:"hooks,omitempty"` // Name is the name of this release - Name string `yaml:"name"` - Namespace string `yaml:"namespace"` - Labels map[string]string `yaml:"labels"` - Values []interface{} `yaml:"values"` - Secrets []string `yaml:"secrets"` - SetValues []SetValue `yaml:"set"` + Name string `yaml:"name,omitempty"` + Namespace string `yaml:"namespace,omitempty"` + Labels map[string]string `yaml:"labels,omitempty"` + Values []interface{} `yaml:"values,omitempty"` + Secrets []string `yaml:"secrets,omitempty"` + SetValues []SetValue `yaml:"set,omitempty"` // The 'env' section is not really necessary any longer, as 'set' would now provide the same functionality - EnvValues []SetValue `yaml:"env"` + EnvValues []SetValue `yaml:"env,omitempty"` - ValuesPathPrefix string `yaml:"valuesPathPrefix"` + ValuesPathPrefix string `yaml:"valuesPathPrefix,omitempty"` - TillerNamespace string `yaml:"tillerNamespace"` - Tillerless *bool `yaml:"tillerless"` + TillerNamespace string `yaml:"tillerNamespace,omitempty"` + Tillerless *bool `yaml:"tillerless,omitempty"` - KubeContext string `yaml:"kubeContext"` + KubeContext string `yaml:"kubeContext,omitempty"` - TLS *bool `yaml:"tls"` - TLSCACert string `yaml:"tlsCACert"` - TLSKey string `yaml:"tlsKey"` - TLSCert string `yaml:"tlsCert"` + TLS *bool `yaml:"tls,omitempty"` + TLSCACert string `yaml:"tlsCACert,omitempty"` + TLSKey string `yaml:"tlsKey,omitempty"` + TLSCert string `yaml:"tlsCert,omitempty"` + + // These values are used in templating + TillerlessTemplate *string `yaml:"tillerlessTemplate,omitempty"` + VerifyTemplate *string `yaml:"verifyTemplate,omitempty"` + WaitTemplate *string `yaml:"waitTemplate,omitempty"` + InstalledTemplate *string `yaml:"installedTemplate,omitempty"` // These settings requires helm-x integration to work - Dependencies []Dependency `yaml:"dependencies"` - JSONPatches []interface{} `yaml:"jsonPatches"` - StrategicMergePatches []interface{} `yaml:"strategicMergePatches"` - Adopt []string `yaml:"adopt"` + Dependencies []Dependency `yaml:"dependencies,omitempty"` + JSONPatches []interface{} `yaml:"jsonPatches,omitempty"` + StrategicMergePatches []interface{} `yaml:"strategicMergePatches,omitempty"` + Adopt []string `yaml:"adopt,omitempty"` // generatedValues are values that need cleaned up on exit generatedValues []string @@ -179,10 +188,10 @@ type ReleaseSpec struct { // SetValue are the key values to set on a helm release type SetValue struct { - Name string `yaml:"name"` - Value string `yaml:"value"` - File string `yaml:"file"` - Values []string `yaml:"values"` + Name string `yaml:"name,omitempty"` + Value string `yaml:"value,omitempty"` + File string `yaml:"file,omitempty"` + Values []string `yaml:"values,omitempty"` } // AffectedReleases hold the list of released that where updated, deleted, or in error @@ -601,7 +610,7 @@ func (st *HelmState) TemplateReleases(helm helmexec.Interface, outputDir string, } if len(errs) == 0 { - if err := helm.TemplateRelease(temp[release.Name], flags...); err != nil { + if err := helm.TemplateRelease(release.Name, temp[release.Name], flags...); err != nil { errs = append(errs, err) } } @@ -666,7 +675,7 @@ func (st *HelmState) LintReleases(helm helmexec.Interface, additionalValues []st } if len(errs) == 0 { - if err := helm.Lint(temp[release.Name], flags...); err != nil { + if err := helm.Lint(release.Name, temp[release.Name], flags...); err != nil { errs = append(errs, err) } } @@ -1051,7 +1060,7 @@ func (st *HelmState) ResolveDeps() (*HelmState, error) { // UpdateDeps wrapper for updating dependencies on the releases func (st *HelmState) UpdateDeps(helm helmexec.Interface) []error { - errs := []error{} + var errs []error for _, release := range st.Releases { if isLocalChart(release.Chart) { @@ -1084,7 +1093,7 @@ func (st *HelmState) BuildDeps(helm helmexec.Interface) []error { for _, release := range st.Releases { if isLocalChart(release.Chart) { - if err := helm.BuildDeps(normalizeChart(st.basePath, release.Chart)); err != nil { + if err := helm.BuildDeps(release.Name, normalizeChart(st.basePath, release.Chart)); err != nil { errs = append(errs, err) } } @@ -1594,3 +1603,11 @@ func (st *HelmState) GenerateOutputDir(outputDir string, release ReleaseSpec) (s return path.Join(outputDir, sb.String()), nil } + +func (st *HelmState) ToYaml() (string, error) { + if result, err := yaml.Marshal(st); err != nil { + return "", err + } else { + return string(result), nil + } +} diff --git a/pkg/state/state_exec_tmpl.go b/pkg/state/state_exec_tmpl.go index 34a3573c..1d570856 100644 --- a/pkg/state/state_exec_tmpl.go +++ b/pkg/state/state_exec_tmpl.go @@ -2,9 +2,12 @@ package state import ( "fmt" + "reflect" + "github.com/imdario/mergo" "github.com/roboll/helmfile/pkg/maputil" "github.com/roboll/helmfile/pkg/tmpl" + "gopkg.in/yaml.v2" ) func (st *HelmState) Values() (map[string]interface{}, error) { @@ -41,6 +44,55 @@ func (st *HelmState) valuesFileTemplateData() EnvironmentTemplateData { } } +func getBoolRefFromStringTemplate(templateRef string) (*bool, error) { + var result bool + if err := yaml.Unmarshal([]byte(templateRef), &result); err != nil { + return nil, fmt.Errorf("failed deserialising string %s: %v", templateRef, err) + } + return &result, nil +} + +func updateBoolTemplatedValues(r *ReleaseSpec) error { + + if r.InstalledTemplate != nil { + if installed, err := getBoolRefFromStringTemplate(*r.InstalledTemplate); err != nil { + return fmt.Errorf("installedTemplate: %v", err) + } else { + r.InstalledTemplate = nil + r.Installed = installed + } + } + + if r.WaitTemplate != nil { + if wait, err := getBoolRefFromStringTemplate(*r.WaitTemplate); err != nil { + return fmt.Errorf("waitTemplate: %v", err) + } else { + r.WaitTemplate = nil + r.Wait = wait + } + } + + if r.TillerlessTemplate != nil { + if tillerless, err := getBoolRefFromStringTemplate(*r.TillerlessTemplate); err != nil { + return fmt.Errorf("tillerlessTemplate: %v", err) + } else { + r.TillerlessTemplate = nil + r.Tillerless = tillerless + } + } + + if r.VerifyTemplate != nil { + if verify, err := getBoolRefFromStringTemplate(*r.VerifyTemplate); err != nil { + return fmt.Errorf("verifyTemplate: %v", err) + } else { + r.VerifyTemplate = nil + r.Verify = verify + } + } + + return nil +} + func (st *HelmState) ExecuteTemplates() (*HelmState, error) { r := *st @@ -50,17 +102,32 @@ func (st *HelmState) ExecuteTemplates() (*HelmState, error) { } for i, rt := range st.Releases { - tmplData := releaseTemplateData{ - Environment: st.Env, - Release: rt, - Values: vals, + successFlag := false + for it, prev := 0, &rt; it < 6; it++ { + tmplData := releaseTemplateData{ + Environment: st.Env, + Release: *prev, + Values: vals, + } + renderer := tmpl.NewFileRenderer(st.readFile, st.basePath, tmplData) + r, err := rt.ExecuteTemplateExpressions(renderer) + if err != nil { + return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %v", st.FilePath, rt.Name, err) + } + if reflect.DeepEqual(prev, r) { + successFlag = true + if err := updateBoolTemplatedValues(r); err != nil { + return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %v", st.FilePath, rt.Name, err) + } + st.Releases[i] = *r + break + } + prev = r } - renderer := tmpl.NewFileRenderer(st.readFile, st.basePath, tmplData) - r, err := rt.ExecuteTemplateExpressions(renderer) - if err != nil { - return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %v", st.FilePath, rt.Name, err) + if !successFlag { + return nil, fmt.Errorf("failed executing templates in release \"%s\".\"%s\": %s", st.FilePath, rt.Name, + "recursive references can't be resolved") } - st.Releases[i] = *r } return &r, nil diff --git a/pkg/state/state_exec_tmpl_test.go b/pkg/state/state_exec_tmpl_test.go index 61187cfe..504caadf 100644 --- a/pkg/state/state_exec_tmpl_test.go +++ b/pkg/state/state_exec_tmpl_test.go @@ -1,11 +1,27 @@ package state import ( - "github.com/roboll/helmfile/pkg/environment" + "fmt" "reflect" + "strings" "testing" + + "github.com/go-test/deep" + "github.com/roboll/helmfile/pkg/environment" ) +func boolPtrToString(ptr *bool) string { + if ptr == nil { + return "" + } + return fmt.Sprintf("&%t", *ptr) +} + +func ptr(v interface{}) interface{} { + r := v + return reflect.ValueOf(r).Addr().Interface() +} + func TestHelmState_executeTemplates(t *testing.T) { tests := []struct { name string @@ -13,24 +29,103 @@ func TestHelmState_executeTemplates(t *testing.T) { want ReleaseSpec }{ { - name: "Has template expressions in chart, values, and secrets", + name: "Has template expressions in chart, values, secrets, version, labels", input: ReleaseSpec{ Chart: "test-charts/{{ .Release.Name }}", Version: "{{ .Release.Name }}-0.1", - Verify: nil, Name: "test-app", Namespace: "test-namespace-{{ .Release.Name }}", Values: []interface{}{"config/{{ .Environment.Name }}/{{ .Release.Name }}/values.yaml"}, Secrets: []string{"config/{{ .Environment.Name }}/{{ .Release.Name }}/secrets.yaml"}, + Labels: map[string]string{"id": "{{ .Release.Name }}"}, }, want: ReleaseSpec{ Chart: "test-charts/test-app", Version: "test-app-0.1", - Verify: nil, Name: "test-app", Namespace: "test-namespace-test-app", Values: []interface{}{"config/test_env/test-app/values.yaml"}, Secrets: []string{"config/test_env/test-app/secrets.yaml"}, + Labels: map[string]string{"id": "test-app"}, + }, + }, + { + name: "Has template expressions in name with recursive refs", + input: ReleaseSpec{ + Chart: "test-chart", + Name: "{{ .Release.Labels.id }}-{{ .Release.Namespace }}", + Namespace: "dev", + Labels: map[string]string{"id": "{{ .Release.Chart }}"}, + }, + want: ReleaseSpec{ + Chart: "test-chart", + Name: "test-chart-dev", + Namespace: "dev", + Labels: map[string]string{"id": "test-chart"}, + }, + }, + { + name: "Has template expressions in boolean values", + input: ReleaseSpec{ + Chart: "test-chart", + Name: "app-dev", + Namespace: "dev", + Labels: map[string]string{"id": "app"}, + InstalledTemplate: func(i string) *string { return &i }(`{{ eq .Release.Labels.id "app" | ternary "yes" "no" }}`), + VerifyTemplate: func(i string) *string { return &i }(`{{ true }}`), + Verify: func(i bool) *bool { return &i }(false), + WaitTemplate: func(i string) *string { return &i }(`{{ false }}`), + TillerlessTemplate: func(i string) *string { return &i }(`yes`), + }, + want: ReleaseSpec{ + Chart: "test-chart", + Name: "app-dev", + Namespace: "dev", + Labels: map[string]string{"id": "app"}, + Installed: func(i bool) *bool { return &i }(true), + Verify: func(i bool) *bool { return &i }(true), + Wait: func(i bool) *bool { return &i }(false), + Tillerless: func(i bool) *bool { return &i }(true), + }, + }, + { + name: "Has template in set-values", + input: ReleaseSpec{ + Chart: "test-charts/chart", + Name: "test-app", + Namespace: "dev", + SetValues: []SetValue{ + SetValue{Name: "val1", Value: "{{ .Release.Name }}-val1"}, + SetValue{Name: "val2", File: "{{ .Release.Name }}.yml"}, + SetValue{Name: "val3", Values: []string{"{{ .Release.Name }}-val2", "{{ .Release.Name }}-val3"}}, + }, + }, + want: ReleaseSpec{ + Chart: "test-charts/chart", + Name: "test-app", + Namespace: "dev", + SetValues: []SetValue{ + SetValue{Name: "val1", Value: "test-app-val1"}, + SetValue{Name: "val2", File: "test-app.yml"}, + SetValue{Name: "val3", Values: []string{"test-app-val2", "test-app-val3"}}, + }, + }, + }, + { + name: "Has template in values (map)", + input: ReleaseSpec{ + Chart: "test-charts/chart", + Verify: nil, + Name: "app", + Namespace: "dev", + Values: []interface{}{map[string]string{"key": "{{ .Release.Name }}-val0"}}, + }, + want: ReleaseSpec{ + Chart: "test-charts/chart", + Verify: nil, + Name: "app", + Namespace: "dev", + Values: []interface{}{map[interface{}]interface{}{"key": "app-val0"}}, }, }, } @@ -59,20 +154,102 @@ func TestHelmState_executeTemplates(t *testing.T) { actual := r.Releases[0] + if !reflect.DeepEqual(actual.Name, tt.want.Name) { + t.Errorf("expected Name %+v, got %+v", tt.want.Name, actual.Name) + } if !reflect.DeepEqual(actual.Chart, tt.want.Chart) { - t.Errorf("expected %+v, got %+v", tt.want.Chart, actual.Chart) + t.Errorf("expected Chart %+v, got %+v", tt.want.Chart, actual.Chart) } if !reflect.DeepEqual(actual.Namespace, tt.want.Namespace) { - t.Errorf("expected %+v, got %+v", tt.want.Namespace, actual.Namespace) + t.Errorf("expected Namespace %+v, got %+v", tt.want.Namespace, actual.Namespace) + } + if diff := deep.Equal(actual.Values, tt.want.Values); diff != nil && len(actual.Values) > 0 { + t.Errorf("Values differs \n%+v", strings.Join(diff, "\n")) } - if !reflect.DeepEqual(actual.Values, tt.want.Values) { - t.Errorf("expected %+v, got %+v", tt.want.Values, actual.Values) + if diff := deep.Equal(actual.Secrets, tt.want.Secrets); diff != nil && len(actual.Secrets) > 0 { + t.Errorf("Secrets differs \n%+v", strings.Join(diff, "\n")) } - if !reflect.DeepEqual(actual.Secrets, tt.want.Secrets) { - t.Errorf("expected %+v, got %+v", tt.want.Secrets, actual.Secrets) + if diff := deep.Equal(actual.SetValues, tt.want.SetValues); diff != nil && len(actual.SetValues) > 0 { + t.Errorf("SetValues differs \n%+v", strings.Join(diff, "\n")) + } + if diff := deep.Equal(actual.Labels, tt.want.Labels); diff != nil && len(actual.Labels) > 0 { + t.Errorf("Labels differs \n%+v", strings.Join(diff, "\n")) } if !reflect.DeepEqual(actual.Version, tt.want.Version) { - t.Errorf("expected %+v, got %+v", tt.want.Version, actual.Version) + t.Errorf("expected Version %+v, got %+v", tt.want.Version, actual.Version) + } + if !reflect.DeepEqual(actual.Installed, tt.want.Installed) { + t.Errorf("expected actual.Installed %+v, got %+v", + boolPtrToString(tt.want.Installed), boolPtrToString(actual.Installed), + ) + } + if !reflect.DeepEqual(actual.Tillerless, tt.want.Tillerless) { + t.Errorf("expected actual.Tillerless %+v, got %+v", + boolPtrToString(tt.want.Tillerless), boolPtrToString(actual.Tillerless), + ) + } + if !reflect.DeepEqual(actual.Verify, tt.want.Verify) { + t.Errorf("expected actual.Verify %+v, got %+v", + boolPtrToString(tt.want.Verify), boolPtrToString(actual.Verify), + ) + } + if !reflect.DeepEqual(actual.Wait, tt.want.Wait) { + t.Errorf("expected actual.Wait %+v, got %+v", + boolPtrToString(tt.want.Wait), boolPtrToString(actual.Wait), + ) + } + }) + } +} + +func TestHelmState_recursiveRefsTemplates(t *testing.T) { + + tests := []struct { + name string + input ReleaseSpec + }{ + { + name: "Has reqursive references", + input: ReleaseSpec{ + Chart: "test-charts/{{ .Release.Name }}", + Verify: nil, + Name: "{{ .Release.Labels.id }}", + Namespace: "dev", + Labels: map[string]string{"id": "app-{{ .Release.Name }}"}, + }, + }, + { + name: "Has unresolvable boolean templates", + input: ReleaseSpec{ + Name: "app-dev", + Chart: "test-charts/app", + Verify: nil, + Namespace: "dev", + WaitTemplate: func(i string) *string { return &i }("hi"), + }, + }, + } + + for i := range tests { + tt := tests[i] + t.Run(tt.name, func(t *testing.T) { + state := &HelmState{ + basePath: ".", + HelmDefaults: HelmSpec{ + KubeContext: "test_context", + }, + Env: environment.Environment{Name: "test_env"}, + Namespace: "test-namespace_", + Repositories: nil, + Releases: []ReleaseSpec{ + tt.input, + }, + } + + r, err := state.ExecuteTemplates() + if err == nil { + t.Errorf("Expected error, got valid response: %v", r) + t.FailNow() } }) } diff --git a/pkg/state/state_test.go b/pkg/state/state_test.go index 4400773f..ae924a6a 100644 --- a/pkg/state/state_test.go +++ b/pkg/state/state_test.go @@ -706,7 +706,7 @@ func (helm *mockHelmExec) UpdateDeps(chart string) error { return nil } -func (helm *mockHelmExec) BuildDeps(chart string) error { +func (helm *mockHelmExec) BuildDeps(name, chart string) error { if strings.Contains(chart, "error") { return errors.New("error") } @@ -769,10 +769,10 @@ func (helm *mockHelmExec) TestRelease(context helmexec.HelmContext, name string, func (helm *mockHelmExec) Fetch(chart string, flags ...string) error { return nil } -func (helm *mockHelmExec) Lint(chart string, flags ...string) error { +func (helm *mockHelmExec) Lint(name, chart string, flags ...string) error { return nil } -func (helm *mockHelmExec) TemplateRelease(chart string, flags ...string) error { +func (helm *mockHelmExec) TemplateRelease(name, chart string, flags ...string) error { return nil } func TestHelmState_SyncRepos(t *testing.T) { From 2a6bd24e3c11432281e66716fe7fc8d1d3bcef3b Mon Sep 17 00:00:00 2001 From: Shane Starcher Date: Fri, 30 Aug 2019 22:32:48 -0700 Subject: [PATCH 16/18] move context outside to limit duplicate repo updates (#828) --- pkg/app/app.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/app/app.go b/pkg/app/app.go index 7c725c43..663e403c 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -381,9 +381,8 @@ func (a *App) visitStates(fileOrDir string, defOpts LoadOpts, converge func(*sta } func (a *App) ForEachState(do func(*Run) []error) error { + ctx := NewContext() err := a.VisitDesiredStatesWithReleasesFiltered(a.FileOrDir, func(st *state.HelmState, helm helmexec.Interface) []error { - ctx := NewContext() - run := NewRun(st, helm, ctx) return do(run) From 01ae59fedd71960e2b5e98579bdd9553f2e2b37c Mon Sep 17 00:00:00 2001 From: Tibo Beijen Date: Sat, 31 Aug 2019 07:38:36 +0200 Subject: [PATCH 17/18] add helm-tiller plugin to docker image (#825) --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 063d34af..adb406a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,8 @@ RUN mkdir -p "$(helm home)/plugins" RUN helm plugin install https://github.com/databus23/helm-diff && \ helm plugin install https://github.com/futuresimple/helm-secrets && \ helm plugin install https://github.com/hypnoglow/helm-s3.git && \ - helm plugin install https://github.com/aslafy-z/helm-git.git + helm plugin install https://github.com/aslafy-z/helm-git.git && \ + helm plugin install https://github.com/rimusz/helm-tiller COPY --from=builder /workspace/helmfile/dist/helmfile_linux_amd64 /usr/local/bin/helmfile From 4bc99337b222c060f00d968141df3550c2f0ac52 Mon Sep 17 00:00:00 2001 From: KUOKA Yusuke Date: Sat, 31 Aug 2019 22:37:46 +0900 Subject: [PATCH 18/18] Make advanced templating backward-compatible (#833) This is a follow-up for #823 Ref https://github.com/roboll/helmfile/pull/823#discussion_r319712283 --- docs/writing-helmfile.md | 6 +- pkg/state/release.go | 111 +++++++++++++++++------------- pkg/state/state.go | 3 + pkg/state/state_exec_tmpl_test.go | 26 +++---- 4 files changed, 81 insertions(+), 65 deletions(-) diff --git a/docs/writing-helmfile.md b/docs/writing-helmfile.md index 285c4124..e621d28a 100644 --- a/docs/writing-helmfile.md +++ b/docs/writing-helmfile.md @@ -102,7 +102,7 @@ Release Templating supports the following parts of release definition: - `set` block values: ```yaml # ... - set: + setTemplate: - name: '{{`{{ .Release.Name }}`}}' values: '{{`{{ .Release.Namespace }}`}}' # ... @@ -110,7 +110,7 @@ Release Templating supports the following parts of release definition: - `values` and `secrets` file paths: ```yaml # ... - values: + valuesTemplate: - config/{{`{{ .Release.Name }}`}}/values.yaml secrets: - config/{{`{{ .Release.Name }}`}}/secrets.yaml @@ -119,7 +119,7 @@ Release Templating supports the following parts of release definition: - inline `values` map: ```yaml # ... - values: + valuesTemplate: - image: tag: `{{ .Release.Labels.tag }}` # ... diff --git a/pkg/state/release.go b/pkg/state/release.go index 6a954ba3..3ea0c24e 100644 --- a/pkg/state/release.go +++ b/pkg/state/release.go @@ -93,6 +93,33 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R result.Labels[key] = s.String() } + if result.ValuesTemplate != nil && len(result.ValuesTemplate) > 0 { + for i, t := range result.ValuesTemplate { + switch ts := t.(type) { + case map[interface{}]interface{}: + serialized, err := yaml.Marshal(ts) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err) + } + + s, err := renderer.RenderTemplateContentToBuffer([]byte(serialized)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, serialized, err) + } + + var deserialized map[interface{}]interface{} + + if err := yaml.Unmarshal(s.Bytes(), &deserialized); err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err) + } + + result.ValuesTemplate[i] = deserialized + } + } + + result.Values = result.ValuesTemplate + } + for i, t := range result.Values { switch ts := t.(type) { case string: @@ -101,24 +128,6 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%s\": %v", r.Name, i, ts, err) } result.Values[i] = s.String() - case map[interface{}]interface{}: - serialized, err := yaml.Marshal(ts) - if err != nil { - return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err) - } - - s, err := renderer.RenderTemplateContentToBuffer([]byte(serialized)) - if err != nil { - return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, serialized, err) - } - - var deserialized map[interface{}]interface{} - - if err := yaml.Unmarshal(s.Bytes(), &deserialized); err != nil { - return nil, fmt.Errorf("failed executing template expressions in release \"%s\".values[%d] = \"%v\": %v", r.Name, i, ts, err) - } - - result.Values[i] = deserialized } } @@ -130,42 +139,46 @@ func (r ReleaseSpec) ExecuteTemplateExpressions(renderer *tmpl.FileRenderer) (*R result.Secrets[i] = s.String() } - for i, val := range result.SetValues { - { - // name - ts := val.Name - s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) - if err != nil { - return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].name = \"%s\": %v", r.Name, i, ts, err) + if result.SetValuesTemplate != nil && len(result.SetValuesTemplate) > 0 { + for i, val := range result.SetValuesTemplate { + { + // name + ts := val.Name + s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].name = \"%s\": %v", r.Name, i, ts, err) + } + result.SetValuesTemplate[i].Name = s.String() } - result.SetValues[i].Name = s.String() - } - { - // value - ts := val.Value - s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) - if err != nil { - return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].value = \"%s\": %v", r.Name, i, ts, err) + { + // value + ts := val.Value + s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].value = \"%s\": %v", r.Name, i, ts, err) + } + result.SetValuesTemplate[i].Value = s.String() } - result.SetValues[i].Value = s.String() - } - { - // file - ts := val.File - s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) - if err != nil { - return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].file = \"%s\": %v", r.Name, i, ts, err) + { + // file + ts := val.File + s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].file = \"%s\": %v", r.Name, i, ts, err) + } + result.SetValuesTemplate[i].File = s.String() } - result.SetValues[i].File = s.String() - } - for j, ts := range val.Values { - // values - s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) - if err != nil { - return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].values[%d] = \"%s\": %v", r.Name, i, j, ts, err) + for j, ts := range val.Values { + // values + s, err := renderer.RenderTemplateContentToBuffer([]byte(ts)) + if err != nil { + return nil, fmt.Errorf("failed executing template expressions in release \"%s\".set[%d].values[%d] = \"%s\": %v", r.Name, i, j, ts, err) + } + result.SetValuesTemplate[i].Values[j] = s.String() } - result.SetValues[i].Values[j] = s.String() } + + result.SetValues = result.SetValuesTemplate } return result, nil diff --git a/pkg/state/state.go b/pkg/state/state.go index 6a749626..c6fd53f4 100644 --- a/pkg/state/state.go +++ b/pkg/state/state.go @@ -153,6 +153,9 @@ type ReleaseSpec struct { Secrets []string `yaml:"secrets,omitempty"` SetValues []SetValue `yaml:"set,omitempty"` + ValuesTemplate []interface{} `yaml:"valuesTemplate,omitempty"` + SetValuesTemplate []SetValue `yaml:"setTemplate,omitempty"` + // The 'env' section is not really necessary any longer, as 'set' would now provide the same functionality EnvValues []SetValue `yaml:"env,omitempty"` diff --git a/pkg/state/state_exec_tmpl_test.go b/pkg/state/state_exec_tmpl_test.go index 504caadf..05aa00fe 100644 --- a/pkg/state/state_exec_tmpl_test.go +++ b/pkg/state/state_exec_tmpl_test.go @@ -31,13 +31,13 @@ func TestHelmState_executeTemplates(t *testing.T) { { name: "Has template expressions in chart, values, secrets, version, labels", input: ReleaseSpec{ - Chart: "test-charts/{{ .Release.Name }}", - Version: "{{ .Release.Name }}-0.1", - Name: "test-app", - Namespace: "test-namespace-{{ .Release.Name }}", - Values: []interface{}{"config/{{ .Environment.Name }}/{{ .Release.Name }}/values.yaml"}, - Secrets: []string{"config/{{ .Environment.Name }}/{{ .Release.Name }}/secrets.yaml"}, - Labels: map[string]string{"id": "{{ .Release.Name }}"}, + Chart: "test-charts/{{ .Release.Name }}", + Version: "{{ .Release.Name }}-0.1", + Name: "test-app", + Namespace: "test-namespace-{{ .Release.Name }}", + ValuesTemplate: []interface{}{"config/{{ .Environment.Name }}/{{ .Release.Name }}/values.yaml"}, + Secrets: []string{"config/{{ .Environment.Name }}/{{ .Release.Name }}/secrets.yaml"}, + Labels: map[string]string{"id": "{{ .Release.Name }}"}, }, want: ReleaseSpec{ Chart: "test-charts/test-app", @@ -94,7 +94,7 @@ func TestHelmState_executeTemplates(t *testing.T) { Chart: "test-charts/chart", Name: "test-app", Namespace: "dev", - SetValues: []SetValue{ + SetValuesTemplate: []SetValue{ SetValue{Name: "val1", Value: "{{ .Release.Name }}-val1"}, SetValue{Name: "val2", File: "{{ .Release.Name }}.yml"}, SetValue{Name: "val3", Values: []string{"{{ .Release.Name }}-val2", "{{ .Release.Name }}-val3"}}, @@ -114,11 +114,11 @@ func TestHelmState_executeTemplates(t *testing.T) { { name: "Has template in values (map)", input: ReleaseSpec{ - Chart: "test-charts/chart", - Verify: nil, - Name: "app", - Namespace: "dev", - Values: []interface{}{map[string]string{"key": "{{ .Release.Name }}-val0"}}, + Chart: "test-charts/chart", + Verify: nil, + Name: "app", + Namespace: "dev", + ValuesTemplate: []interface{}{map[string]string{"key": "{{ .Release.Name }}-val0"}}, }, want: ReleaseSpec{ Chart: "test-charts/chart",