From a72f8af1e2bb301d0b3c0e5b051d38d00901f1f6 Mon Sep 17 00:00:00 2001 From: Michael Burt Date: Sun, 21 Jan 2024 11:42:02 -0700 Subject: [PATCH] I have read the CLA Document and I hereby sign the CLA --- .github/workflows/agreements.yaml | 6 +- .gitignore | 1 + .golangci.yml | 12 +- .../components/apache/datasource.libsonnet | 10 + .../helm3/components/apache/index.jsonnet | 1 + .../victoria-metrics/datasource.libsonnet | 7 + .../components/victoria-metrics/index.jsonnet | 1 + examples/helm3/environments/base.libsonnet | 5 + examples/helm3/environments/default.libsonnet | 7 + examples/helm3/params.libsonnet | 9 + examples/helm3/qbec.yaml | 22 ++ go.mod | 1 + go.sum | 1 + internal/remote/patch.go | 6 +- site/.hugo_build.lock | 0 site/content/userguide/usage/authoring.md | 8 +- vm/internal/ds/exec/exec.go | 2 +- vm/internal/ds/factory/datasource.go | 4 + vm/internal/ds/helm3/helm3.go | 361 ++++++++++++++++++ vm/internal/ds/helm3/helm3_test.go | 205 ++++++++++ vm/internal/ds/helm3/testdata/apache.json | 173 +++++++++ vm/internal/importers/data-source.go | 4 +- vm/internal/importers/glob.go | 21 +- vm/vars_test.go | 20 +- 24 files changed, 855 insertions(+), 32 deletions(-) create mode 100644 examples/helm3/components/apache/datasource.libsonnet create mode 100644 examples/helm3/components/apache/index.jsonnet create mode 100644 examples/helm3/components/victoria-metrics/datasource.libsonnet create mode 100644 examples/helm3/components/victoria-metrics/index.jsonnet create mode 100644 examples/helm3/environments/base.libsonnet create mode 100644 examples/helm3/environments/default.libsonnet create mode 100644 examples/helm3/params.libsonnet create mode 100644 examples/helm3/qbec.yaml create mode 100644 site/.hugo_build.lock create mode 100644 vm/internal/ds/helm3/helm3.go create mode 100644 vm/internal/ds/helm3/helm3_test.go create mode 100644 vm/internal/ds/helm3/testdata/apache.json diff --git a/.github/workflows/agreements.yaml b/.github/workflows/agreements.yaml index 60d811f3..438396cd 100644 --- a/.github/workflows/agreements.yaml +++ b/.github/workflows/agreements.yaml @@ -1,11 +1,9 @@ name: "Agreements" - on: issue_comment: - types: [ created ] + types: [created] pull_request_target: - types: [ opened, closed, synchronize ] - + types: [opened, closed, synchronize] jobs: ContributorLicenseAgreement: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 4de96360..5a65c867 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ bin/ .tools/ .vscode/ *.generated +.gitconfig diff --git a/.golangci.yml b/.golangci.yml index 5f48fe39..006abe15 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,7 +7,17 @@ linters-settings: default-signifies-exhaustive: false golint: min-confidence: 0 - + depguard: + rules: + main: + files: + - $all + - "!$test" + allow: + - $gostd + - github.com/splunk/qbec + - github.com/spf13/cobra + linters: disable-all: true enable: diff --git a/examples/helm3/components/apache/datasource.libsonnet b/examples/helm3/components/apache/datasource.libsonnet new file mode 100644 index 00000000..c4493eae --- /dev/null +++ b/examples/helm3/components/apache/datasource.libsonnet @@ -0,0 +1,10 @@ +{ + objects: import 'data://helm/apache?config-from=apache-config', + config: { + options: { + repo: 'https://charts.bitnami.com/bitnami', + }, + values: { + }, + }, +} diff --git a/examples/helm3/components/apache/index.jsonnet b/examples/helm3/components/apache/index.jsonnet new file mode 100644 index 00000000..7a6a63bf --- /dev/null +++ b/examples/helm3/components/apache/index.jsonnet @@ -0,0 +1 @@ +(import 'datasource.libsonnet').objects diff --git a/examples/helm3/components/victoria-metrics/datasource.libsonnet b/examples/helm3/components/victoria-metrics/datasource.libsonnet new file mode 100644 index 00000000..ccd4468e --- /dev/null +++ b/examples/helm3/components/victoria-metrics/datasource.libsonnet @@ -0,0 +1,7 @@ +{ + objects: import 'data://helm/github.com/VictoriaMetrics/helm-charts/raw/347d4558d9c25cd341718bf5a2ee167da042c080/packages/victoria-metrics-cluster-0.9.6.tgz?config-from=victoria-config', + config: { + options: {}, + values: {}, + }, +} diff --git a/examples/helm3/components/victoria-metrics/index.jsonnet b/examples/helm3/components/victoria-metrics/index.jsonnet new file mode 100644 index 00000000..7a6a63bf --- /dev/null +++ b/examples/helm3/components/victoria-metrics/index.jsonnet @@ -0,0 +1 @@ +(import 'datasource.libsonnet').objects diff --git a/examples/helm3/environments/base.libsonnet b/examples/helm3/environments/base.libsonnet new file mode 100644 index 00000000..3e56f436 --- /dev/null +++ b/examples/helm3/environments/base.libsonnet @@ -0,0 +1,5 @@ +// this file has the baseline default parameters +{ + components: { + }, +} diff --git a/examples/helm3/environments/default.libsonnet b/examples/helm3/environments/default.libsonnet new file mode 100644 index 00000000..50a2b9e7 --- /dev/null +++ b/examples/helm3/environments/default.libsonnet @@ -0,0 +1,7 @@ +// this file has the param overrides for the default environment +local base = import './base.libsonnet'; + +base { + components+: { + }, +} diff --git a/examples/helm3/params.libsonnet b/examples/helm3/params.libsonnet new file mode 100644 index 00000000..7f4e0d1d --- /dev/null +++ b/examples/helm3/params.libsonnet @@ -0,0 +1,9 @@ +// this file returns the params for the current qbec environment +local env = std.extVar('qbec.io/env'); +local paramsMap = import 'glob-import:environments/*.libsonnet'; +local baseFile = if env == '_' then 'base' else env; +local key = 'environments/%s.libsonnet' % baseFile; + +if std.objectHas(paramsMap, key) +then paramsMap[key] +else error 'no param file %s found for environment %s' % [key, env] diff --git a/examples/helm3/qbec.yaml b/examples/helm3/qbec.yaml new file mode 100644 index 00000000..1ef4e79a --- /dev/null +++ b/examples/helm3/qbec.yaml @@ -0,0 +1,22 @@ +apiVersion: qbec.io/v1alpha1 +kind: App +metadata: + name: helm3 +spec: + environments: + default: + defaultNamespace: charts + context: kind + vars: + computed: + - name: helmSetup + code: | + {} + - name: victoria-config + code: | + (import 'components/victoria-metrics/datasource.libsonnet').config + - name: apache-config + code: | + (import 'components/apache/datasource.libsonnet').config + dataSources: + - helm3://helm?configVar=helmSetup diff --git a/go.mod b/go.mod index cdf8b998..99b4993a 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/golang/protobuf v1.5.2 github.com/google/go-jsonnet v0.18.0 github.com/googleapis/gnostic v0.5.5 + github.com/iancoleman/strcase v0.2.0 github.com/jonboulle/clockwork v0.2.2 github.com/mattn/go-isatty v0.0.14 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index ed0cf55d..3b90c81b 100644 --- a/go.sum +++ b/go.sum @@ -456,6 +456,7 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= diff --git a/internal/remote/patch.go b/internal/remote/patch.go index 87613cea..34f136e6 100644 --- a/internal/remote/patch.go +++ b/internal/remote/patch.go @@ -116,10 +116,10 @@ func deleteEmpty(parent map[string]interface{}, key string) { // contains empty objects. It makes an assumption that there is actually no reason an empty object // needs to be updated for a Kubernetes resource considering that the server would already have an object // there on initial create if needed. Things considered empty will be of the form: -// {} -// { metadata: { labels: {}, annotations: {} } -// { metadata: { labels: {}, annotations: {} }, spec: { foo: { bar: {} } } } // +// {} +// { metadata: { labels: {}, annotations: {} } +// { metadata: { labels: {}, annotations: {} }, spec: { foo: { bar: {} } } } func isEmptyPatch(patch []byte) bool { var root map[string]interface{} err := json.Unmarshal(patch, &root) diff --git a/site/.hugo_build.lock b/site/.hugo_build.lock new file mode 100644 index 00000000..e69de29b diff --git a/site/content/userguide/usage/authoring.md b/site/content/userguide/usage/authoring.md index 9130b649..6f6e92a6 100644 --- a/site/content/userguide/usage/authoring.md +++ b/site/content/userguide/usage/authoring.md @@ -26,7 +26,7 @@ A component is: It is valid for a component to return an empty set of objects if runtime parameters determine that nothing should be installed for a specific target environment. -## Using helm charts and external data sources +## Using external data sources qbec provides integration to run external commands and consume their output in jsonnet code. See the [Jsonnet data importer](../../../reference/jsonnet-external-data) for more information on how this works. @@ -34,6 +34,12 @@ See the [Jsonnet data importer](../../../reference/jsonnet-external-data) for mo Note that the [expandHelmTemplate](../../../reference/jsonnet-native-funcs/#expandhelmtemplate) native function is now deprecated in favor of the data importer mechanism. +## Native Helm integration + +qbec provides native support for Helm3, allowing you to render Helm `values` during runtime via jsonnet. To use this +feature, configure a Helm datasource. See [examples/helm3](https://github.com/splunk/qbec/tree/main/examples/helm3/) for +an example component. + ## Using other jsonnet libraries [k8s-yaml-patch](https://github.com/splunk/k8s-yaml-patch), diff --git a/vm/internal/ds/exec/exec.go b/vm/internal/ds/exec/exec.go index cc421a0f..77d809b4 100644 --- a/vm/internal/ds/exec/exec.go +++ b/vm/internal/ds/exec/exec.go @@ -121,11 +121,11 @@ func (d *execSource) Init(p datasource.ConfigProvider) (fErr error) { if err != nil { return err } + c.initDefaults() err = c.assertValid() if err != nil { return err } - c.initDefaults() d.runner = newRunner(&c) return nil } diff --git a/vm/internal/ds/factory/datasource.go b/vm/internal/ds/factory/datasource.go index 230005fa..921161eb 100644 --- a/vm/internal/ds/factory/datasource.go +++ b/vm/internal/ds/factory/datasource.go @@ -21,6 +21,7 @@ import ( "github.com/pkg/errors" "github.com/splunk/qbec/vm/internal/ds" "github.com/splunk/qbec/vm/internal/ds/exec" + "github.com/splunk/qbec/vm/internal/ds/helm3" ) // Create creates a new data source from the supplied URL. @@ -35,6 +36,7 @@ func Create(u string) (ds.DataSourceWithLifecycle, error) { scheme := parsed.Scheme switch scheme { case exec.Scheme: + case helm3.Scheme: default: return nil, fmt.Errorf("data source URL '%s', unsupported scheme '%s'", u, scheme) } @@ -52,6 +54,8 @@ func Create(u string) (ds.DataSourceWithLifecycle, error) { switch scheme { case exec.Scheme: return makeLazy(exec.New(name, varName)), nil + case helm3.Scheme: + return makeLazy(helm3.New(name, varName)), nil default: return nil, fmt.Errorf("internal error: unable to create a data source for %s", u) } diff --git a/vm/internal/ds/helm3/helm3.go b/vm/internal/ds/helm3/helm3.go new file mode 100644 index 00000000..6940e67a --- /dev/null +++ b/vm/internal/ds/helm3/helm3.go @@ -0,0 +1,361 @@ +/* + Copyright 2021 Splunk Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package helm3 provides a data source implementation that can extract k8s objects out of helm3 charts +package helm3 + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/iancoleman/strcase" + "github.com/pkg/errors" + "github.com/splunk/qbec/internal/sio" + "github.com/splunk/qbec/vm/datasource" + "github.com/splunk/qbec/vm/internal/ds" + "github.com/splunk/qbec/vm/internal/natives" +) + +// Scheme is the scheme supported by this data source +const ( + Scheme = "helm3" + configVarParam = "config-from" +) + +var defaultNamespaceVar = "qbec.io/defaultNs" // TODO: make this blank and have qbec set it + +// SetDefaultNamespaceVar sets the name of the external variable that is always set to the default namespace to use. +func SetDefaultNamespaceVar(s string) { + defaultNamespaceVar = s +} + +// Config is the configuration of the data source. TODO: add version check, SHA check options etc. +type Config struct { + Command string `json:"command"` // the executable that is run, default is "helm" + Timeout string `json:"timeout,omitempty"` // command timeout as a duration string + timeout time.Duration // internal representation +} + +// TemplateOptions are a subset of command line arguments that can be passed to `helm template` +type TemplateOptions struct { + Namespace string `json:"namespace,omitempty"` + CreateNamespace bool `json:"createNamespace,omitempty"` + APIVersions []string `json:"apiVersions,omitempty"` + DependencyUpdate bool `json:"dependencyUpdate,omitempty"` + Description string `json:"description"` + Devel bool `json:"devel"` + DisableOpenAPIValidation bool `json:"disableOpenapiValidation,omitempty"` + GenerateName bool `json:"generateName,omitempty"` + IncludeCRDs bool `json:"includeCrds,omitempty"` + IsUpgrade bool `json:"isUpgrade,omitempty"` + SkipCRDs bool `json:"skipCrds,omitempty"` + SkipTests bool `json:"skipTests,omitempty"` + InsecureSkipTLSVerify bool `json:"insecureSkipTlsVerify,omitempty"` + KubeVersion string `json:"kubeVersion,omitempty"` + NameTemplate string `json:"nameTemplate,omitempty"` + NoHooks bool `json:"noHooks,omitempty"` + PassCredentials bool `json:"passCredentials,omitempty"` + Password string `json:"password,omitempty" fld:"secure"` + RenderSubchartNotes bool `json:"renderSubchartNotes,omitempty"` + Replace bool `json:"replace,omitempty"` + Repo string `json:"repo,omitempty"` + ShowOnly []string `json:"showOnly,omitempty"` + Username string `json:"username,omitempty"` + Validate bool `json:"validate,omitempty"` + Verify bool `json:"verify,omitempty"` + Version string `json:"version,omitempty"` + /* + // first pass: omit all options that refer to files since we need to figure out what the exact + // semantics of relative paths are and its corresponding implications of qbec root etc. + --ca-file string verify certificates of HTTPS-enabled servers using this CA bundle + --cert-file string identify HTTPS client using this SSL certificate file + --key-file string identify HTTPS client using this SSL key file + --keyring string location of public keys used for verification (default "/Users/kanantheswaran/.gnupg/pubring.gpg") + --output-dir string writes the executed templates to files in output-dir instead of stdout + --post-renderer postrenderer the path to an executable to be used for post rendering. If it exists in $PATH, the binary will be used, otherwise it will try to look for the executable at the given path (default exec) + --release-name use release name in the output-dir path. + // ignore values related stuff since we send JSON object via stdin + --set stringArray set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) + --set-file stringArray set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2) + --set-string stringArray set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) + -f, --values strings specify values in a YAML file or a URL (can specify multiple) + // values not relevant for template expansion + --timeout duration time to wait for any individual Kubernetes operation (like Jobs for hooks) (default 5m0s) + --wait if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment, StatefulSet, or ReplicaSet are in a ready state before marking the release as successful. It will wait for as long as --timeout + --wait-for-jobs if set and --wait enabled, will wait until all Jobs have been completed before marking the release as successful. It will wait for as long as --timeout + Global Flags: + // we always set debug for better error messages, don't expose + --debug enable verbose output + // we _could_ set all the kube options based on what we know but this requires a dance between qbec knowledge and data source knowledge + --kube-apiserver string the address and the port for the Kubernetes API server + --kube-as-group stringArray group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --kube-as-user string username to impersonate for the operation + --kube-ca-file string the certificate authority file for the Kubernetes API server connection + --kube-context string name of the kubeconfig context to use + --kube-token string bearer token used for authentication + --kubeconfig string path to the kubeconfig file + // more files to ignore for now + --registry-config string path to the registry config file (default "") + --repository-cache string path to the file containing cached repository + --repository-config string path to the file containing repository names and URLs + */ +} + +func (o TemplateOptions) toInternalCommandLine(display bool) []string { + t := reflect.TypeOf(o) + v := reflect.ValueOf(o) + var ret []string + for i := 0; i < t.NumField(); i++ { + fld := t.Field(i) + val := v.FieldByName(fld.Name) + tag := strings.Split(fld.Tag.Get("json"), ",")[0] + secure := strings.Contains(fld.Tag.Get("fld"), "secure") + option := fmt.Sprintf("--%s", strcase.ToKebab(tag)) + kv := func(v interface{}) { + if secure && display { + v = "" + } + ret = append(ret, fmt.Sprintf("%s=%s", option, v)) + } + switch { + case fld.Type.Name() == "string": + out := val.String() + if out != "" { + kv(out) + } + case fld.Type.Name() == "bool": + out := val.Bool() + if out { + ret = append(ret, option) + } + case fld.Type.Kind() == reflect.Slice && fld.Type.Elem().Name() == "string": + for j := 0; j < val.Len(); j++ { + kv(val.Index(j).String()) + } + default: + panic("unsupported field type for field " + fld.Name) + } + } + return ret +} + +func (o TemplateOptions) toCommandLine() []string { + return o.toInternalCommandLine(false) +} + +func (o TemplateOptions) toDisplay() string { + parts := o.toInternalCommandLine(true) + return strings.Join(parts, " ") // TODO: improve me for quoting +} + +// TemplateConfig is the configuration for the template command that includes options and values. +// TODO: make a schema for this and validate inputs before JSON unmarshal +type TemplateConfig struct { + Name string `json:"name,omitempty"` + Options TemplateOptions `json:"options,omitempty"` + Values map[string]interface{} `json:"values,omitempty"` +} + +func findExecutable(cmd string) (string, error) { + if !filepath.IsAbs(cmd) { + p, err := filepath.Abs(cmd) + if err == nil { + stat, err := os.Stat(cmd) + if err == nil { + if m := stat.Mode(); !m.IsDir() && m&0111 != 0 { + return p, nil + } + } + } + } + return exec.LookPath(cmd) +} + +func (c *Config) assertValid() error { + if c.Command == "" { + return fmt.Errorf("command not specified") + } + if c.Timeout != "" { + t, err := time.ParseDuration(c.Timeout) + if err != nil { + return fmt.Errorf("invalid timeout '%s': %v", c.Timeout, err) + } + c.timeout = t + } + exe, err := findExecutable(c.Command) + if err != nil { + return fmt.Errorf("invalid command '%s': %v", c.Command, err) + } + c.Command = exe + // TODO: support version/ SHA checking etc. + return nil +} + +func (c *Config) initDefaults() { + if c.Command == "" { + c.Command = "helm" + } + if c.timeout == 0 { + c.timeout = time.Minute + } +} + +type helm3Source struct { + name string + configVar string + cp datasource.ConfigProvider + config Config +} + +// New creates a new helm3 data source +func New(name string, configVar string) ds.DataSourceWithLifecycle { + return &helm3Source{ + name: name, + configVar: configVar, + } +} + +// Name implements the interface method +func (d *helm3Source) Name() string { + return d.name +} + +// Init implements the interface method. +func (d *helm3Source) Init(p datasource.ConfigProvider) (fErr error) { + defer func() { + fErr = errors.Wrapf(fErr, "init data source %s", d.name) // nil wraps as nil + }() + cfgJSON, err := p(d.configVar) + if err != nil { + return err + } + var c Config + err = json.Unmarshal([]byte(cfgJSON), &c) + if err != nil { + return err + } + c.initDefaults() + err = c.assertValid() + if err != nil { + return err + } + d.cp = p + d.config = c + return nil +} + +// Resolve implements the interface method. +func (d *helm3Source) Resolve(path string) (_ string, finalErr error) { + u, err := url.Parse(path) + if err != nil { + return "", errors.Wrapf(err, "parse path %q", path) + } + configVar := u.Query().Get(configVarParam) + if configVar == "" { + return "", fmt.Errorf("%s query param not set in data source path %q", configVarParam, path) + } + u.Query().Del(configVarParam) + str, err := d.cp(configVar) + if err != nil { + return "", errors.Wrapf(err, "get ext code variable %s", configVar) + } + var tc TemplateConfig + err = json.Unmarshal([]byte(str), &tc) + if err != nil { + return "", errors.Wrapf(err, "json unmarshal of %s value", configVar) + } + if tc.Options.Namespace == "" { + if defaultNamespaceVar == "" { + return "", fmt.Errorf("namespace option not specified and no default value exists") + } + ns, err := d.cp(defaultNamespaceVar) + if err != nil { + return "", errors.Wrapf(err, "get default namespace from %s", defaultNamespaceVar) + } + tc.Options.Namespace = ns + } + out, err := d.runTemplate(u, tc) + if err != nil { + return "", err + } + b, err := json.Marshal(out) + if err != nil { + return "", errors.Wrap(err, "marshal output") + } + return string(b), nil +} + +func (d *helm3Source) runTemplate(u *url.URL, tc TemplateConfig) (interface{}, error) { + path := strings.TrimPrefix(u.Path, "/") + chart := path + if tc.Options.Repo == "" { // then assume path is a URL with https scheme + parts := strings.SplitN(path, "/", 2) // first component of part is actually the host + if len(parts) == 1 { + return nil, fmt.Errorf("unable to extract host and path from %s", path) + } + u.Host = parts[0] + u.Path = "/" + parts[1] + u.Scheme = "https" + chart = u.String() + } + b, err := json.Marshal(tc.Values) + if err != nil { + return "", errors.Wrap(err, "marshal values") + } + args := append([]string{"template", "--debug"}, tc.Options.toCommandLine()...) + if tc.Name != "" { // TODO: figure out if omitting name is the correct strategy + args = append(args, tc.Name) + } + args = append(args, chart) + args = append(args, "--values", "-") + + ctx, cancel := context.WithTimeout(context.Background(), d.config.timeout) + defer cancel() + + displayCommand := fmt.Sprintf("%s template %s %s %s", d.config.Command, tc.Options.toDisplay(), tc.Name, chart) + sio.Debugln(displayCommand) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd := exec.CommandContext(ctx, d.config.Command, args...) + cmd.Stdin = bytes.NewBuffer(b) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err = cmd.Run() + if err != nil { + sio.Warnf("%s\n%s\n", "debug output from helm", stdout.String()) + return "", fmt.Errorf("%s\n%s", err.Error(), stderr.String()) + } + docs, err := natives.ParseYAMLDocuments(bytes.NewReader(stdout.Bytes())) + if err != nil { + return "", err + } + return docs, nil +} + +// Close implements the interface method. +func (d *helm3Source) Close() error { + return nil +} diff --git a/vm/internal/ds/helm3/helm3_test.go b/vm/internal/ds/helm3/helm3_test.go new file mode 100644 index 00000000..218507c8 --- /dev/null +++ b/vm/internal/ds/helm3/helm3_test.go @@ -0,0 +1,205 @@ +/* + Copyright 2021 Splunk Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package helm3 + +import ( + "encoding/json" + "io/ioutil" + "net/url" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTemplateOptionsEmpty(t *testing.T) { + tc := TemplateOptions{} + parts := tc.toCommandLine() + require.Equal(t, 0, len(parts)) +} + +func TestTemplateOptionsFull(t *testing.T) { + tc := TemplateOptions{ + Namespace: "default", + CreateNamespace: true, + APIVersions: []string{"1.2", "2.1"}, + DependencyUpdate: true, + Description: "foo bar", + Devel: true, + DisableOpenAPIValidation: true, + GenerateName: true, + IncludeCRDs: true, + IsUpgrade: true, + SkipCRDs: true, + SkipTests: true, + InsecureSkipTLSVerify: true, + KubeVersion: "1.16", + NameTemplate: "gentemp", + NoHooks: true, + PassCredentials: true, + Password: "foobar", + RenderSubchartNotes: true, + Replace: true, + Repo: "r1", + ShowOnly: []string{"foo", "bar"}, + Username: "me", + Validate: true, + Verify: true, + Version: "1.2.3", + } + ret := tc.toCommandLine() + a := assert.New(t) + a.Contains(ret, "--namespace=default") + a.Contains(ret, "--api-versions=1.2") + a.Contains(ret, "--api-versions=2.1") + a.Contains(ret, "--kube-version=1.16") + a.Contains(ret, "--name-template=gentemp") + a.Contains(ret, "--password=foobar") + a.Contains(ret, "--repo=r1") + a.Contains(ret, "--show-only=foo") + a.Contains(ret, "--show-only=bar") + a.Contains(ret, "--username=me") + a.Contains(ret, "--version=1.2.3") + for _, s := range []string{ + "create-namespace", + "dependency-update", + "devel", + "disable-openapi-validation", + "generate-name", + "include-crds", + "is-upgrade", + "skip-crds", + "skip-tests", + "insecure-skip-tls-verify", + "no-hooks", + "pass-credentials", + "render-subchart-notes", + "replace", + "validate", + "verify", + } { + a.Contains(ret, "--"+s) + } + + str := tc.toDisplay() + a.Contains(str, "--namespace=default") + a.Contains(str, "--api-versions=1.2") + a.Contains(str, "--api-versions=2.1") + a.Contains(str, "--kube-version=1.16") + a.Contains(str, "--name-template=gentemp") + a.Contains(str, "--password=") + a.Contains(str, "--repo=r1") + a.Contains(str, "--show-only=foo") + a.Contains(str, "--show-only=bar") + a.Contains(str, "--username=me") + a.Contains(str, "--version=1.2.3") + for _, s := range []string{ + "create-namespace", + "dependency-update", + "devel", + "disable-openapi-validation", + "generate-name", + "include-crds", + "is-upgrade", + "skip-crds", + "skip-tests", + "insecure-skip-tls-verify", + "no-hooks", + "pass-credentials", + "render-subchart-notes", + "replace", + "validate", + "verify", + } { + a.Contains(str, "--"+s) + } +} + +func TestRunTemplate(t *testing.T) { + + mockConfigProvider := func(varName string) (string, error) { + if varName == "config-var-name" { + return `{"command": "helm", "timeout": "5s"}`, nil + } + return "", nil + } + + mockTemplateConfig := TemplateConfig{ + Name: "mock-release", + Options: TemplateOptions{ + Repo: "https://charts.bitnami.com/bitnami", + Namespace: "foobar", + Version: "10.1.0", + }, + Values: map[string]interface{}{ + "key": "value", + }, + } + + helm3Src := &helm3Source{configVar: "config-var-name"} + err := helm3Src.Init(mockConfigProvider) + require.NoError(t, err) + mockURL := &url.URL{ + Scheme: "http", + Host: "", + Path: "apache", + } + templated, err := helm3Src.runTemplate(mockURL, mockTemplateConfig) + b, err := json.MarshalIndent(templated, "", " ") + require.NoError(t, err) + filePath := filepath.Join("testdata", "apache.json") + fileContent, err := ioutil.ReadFile(filePath) + require.NoError(t, err) + require.Equal(t, fileContent, b) +} + +func TestInitDefaults(t *testing.T) { + cfg := Config{} + cfg.initDefaults() + require.Equal(t, cfg.Command, "helm") + require.Equal(t, cfg.Timeout, "") + require.Equal(t, cfg.timeout, time.Minute) +} + +func TestFindExecutable(t *testing.T) { + cmd, err := findExecutable("helm") + require.NoError(t, err) + require.FileExists(t, cmd) +} + +func TestClose(t *testing.T) { + helm3Src := &helm3Source{configVar: "config-var-name"} + err := helm3Src.Close() + require.NoError(t, err) +} + +func TestName(t *testing.T) { + mockConfigProvider := func(varName string) (string, error) { + if varName == "config-var-name" { + return `{"command": "helm", "timeout": "5s"}`, nil + } + return "", nil + } + + helm3Src := &helm3Source{name: "baz", configVar: "config-var-name"} + err := helm3Src.Init(mockConfigProvider) + require.NoError(t, err) + name := helm3Src.Name() + require.Equal(t, name, "baz") +} diff --git a/vm/internal/ds/helm3/testdata/apache.json b/vm/internal/ds/helm3/testdata/apache.json new file mode 100644 index 00000000..b2a7589d --- /dev/null +++ b/vm/internal/ds/helm3/testdata/apache.json @@ -0,0 +1,173 @@ +[ + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "labels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "apache", + "helm.sh/chart": "apache-10.1.0" + }, + "name": "mock-release-apache", + "namespace": "foobar" + }, + "spec": { + "externalTrafficPolicy": "Cluster", + "loadBalancerSourceRanges": [], + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": "http" + }, + { + "name": "https", + "port": 443, + "targetPort": "https" + } + ], + "selector": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/name": "apache" + }, + "sessionAffinity": "None", + "type": "LoadBalancer" + } + }, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "labels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "apache", + "helm.sh/chart": "apache-10.1.0" + }, + "name": "mock-release-apache", + "namespace": "foobar" + }, + "spec": { + "replicas": 1, + "revisionHistoryLimit": 10, + "selector": { + "matchLabels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/name": "apache" + } + }, + "strategy": { + "type": "RollingUpdate" + }, + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "apache", + "helm.sh/chart": "apache-10.1.0" + } + }, + "spec": { + "affinity": { + "nodeAffinity": null, + "podAffinity": null, + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "podAffinityTerm": { + "labelSelector": { + "matchLabels": { + "app.kubernetes.io/instance": "mock-release", + "app.kubernetes.io/name": "apache" + } + }, + "topologyKey": "kubernetes.io/hostname" + }, + "weight": 1 + } + ] + } + }, + "containers": [ + { + "env": [ + { + "name": "BITNAMI_DEBUG", + "value": "false" + }, + { + "name": "APACHE_HTTP_PORT_NUMBER", + "value": "8080" + }, + { + "name": "APACHE_HTTPS_PORT_NUMBER", + "value": "8443" + } + ], + "envFrom": null, + "image": "docker.io/bitnami/apache:2.4.57-debian-11-r134", + "imagePullPolicy": "IfNotPresent", + "livenessProbe": { + "failureThreshold": 6, + "httpGet": { + "path": "/", + "port": "http" + }, + "initialDelaySeconds": 180, + "periodSeconds": 20, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "name": "apache", + "ports": [ + { + "containerPort": 8080, + "name": "http" + }, + { + "containerPort": 8443, + "name": "https" + } + ], + "readinessProbe": { + "failureThreshold": 6, + "httpGet": { + "path": "/", + "port": "http" + }, + "initialDelaySeconds": 30, + "periodSeconds": 10, + "successThreshold": 1, + "timeoutSeconds": 5 + }, + "resources": { + "limits": {}, + "requests": {} + }, + "securityContext": { + "runAsNonRoot": true, + "runAsUser": 1001 + }, + "volumeMounts": null + } + ], + "hostAliases": [ + { + "hostnames": [ + "status.localhost" + ], + "ip": "127.0.0.1" + } + ], + "priorityClassName": "", + "securityContext": { + "fsGroup": 1001 + }, + "volumes": null + } + } + } + } +] \ No newline at end of file diff --git a/vm/internal/importers/data-source.go b/vm/internal/importers/data-source.go index a9261a34..452caac4 100644 --- a/vm/internal/importers/data-source.go +++ b/vm/internal/importers/data-source.go @@ -44,7 +44,9 @@ type DataSourceImporter struct { // NewDataSourceImporter returns an importer that can resolve paths for the specified datasource. // It processes entries of the form -// data://{name}[/{path-to-be-resolved}] +// +// data://{name}[/{path-to-be-resolved}] +// // If no path is provided, it is set to "/" func NewDataSourceImporter(source datasource.DataSource) *DataSourceImporter { exact := fmt.Sprintf("%s://%s", dsPrefix, source.Name()) diff --git a/vm/internal/importers/glob.go b/vm/internal/importers/glob.go index 1cb6a2b8..3f2d0cd6 100644 --- a/vm/internal/importers/glob.go +++ b/vm/internal/importers/glob.go @@ -73,23 +73,22 @@ type globEntry struct { // // That is, given the following directory structure: // -// lib -// - a.json -// - b.json -// caller -// - c.libsonnet +// lib +// - a.json +// - b.json +// caller +// - c.libsonnet // // where c.libsonnet has the following contents // -// import 'glob-import:../lib/*.json' +// import 'glob-import:../lib/*.json' // // evaluating `c.libsonnet` will return jsonnet code of the following form: // -// { -// '../lib/a.json': import '../lib/a.json', -// '../lib/b.json': import '../lib/b.json', -// } -// +// { +// '../lib/a.json': import '../lib/a.json', +// '../lib/b.json': import '../lib/b.json', +// } type GlobImporter struct { innerVerb string prefix string diff --git a/vm/vars_test.go b/vm/vars_test.go index ebfd7a9b..b8565d10 100644 --- a/vm/vars_test.go +++ b/vm/vars_test.go @@ -1,17 +1,17 @@ /* - Copyright 2021 Splunk Inc. +Copyright 2021 Splunk Inc. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ package vm