Implement Helm3 Datasource

.github/workflows/agreements.yaml

@@ -1,11 +1,9 @@
 name: "Agreements"
-
 on:
  issue_comment:
- types: [ created ]
+ types: [created]
  pull_request_target:
- types: [ opened, closed, synchronize ]
-
+ types: [opened, closed, synchronize]
 jobs:
  ContributorLicenseAgreement:
  runs-on: ubuntu-latest

.gitignore

@@ -13,4 +13,5 @@ bin/
 .tools/
 .vscode/
 *.generated
+.gitconfig
 

.golangci.yml

@@ -7,7 +7,17 @@ linters-settings:
  default-signifies-exhaustive: false
  golint:
  min-confidence: 0
-
+ depguard:
+ rules:
+ main:
+ files:
+ - $all
+ - "!$test"
+ allow:
+ - $gostd
+ - github.com/splunk/qbec
+ - github.com/spf13/cobra
+
 linters:
  disable-all: true
  enable:

examples/helm3/components/apache/datasource.libsonnet

@@ -0,0 +1,10 @@
+{
+ objects: import 'data://helm/apache?config-from=apache-config',
+ config: {
+ options: {
+ repo: 'https://charts.bitnami.com/bitnami',
+ },
+ values: {
+ },
+ },
+}

examples/helm3/components/apache/index.jsonnet

@@ -0,0 +1 @@
+(import 'datasource.libsonnet').objects

examples/helm3/components/victoria-metrics/datasource.libsonnet

@@ -0,0 +1,7 @@
+{
+ objects: import 'data://helm/github.com/VictoriaMetrics/helm-charts/raw/347d4558d9c25cd341718bf5a2ee167da042c080/packages/victoria-metrics-cluster-0.9.6.tgz?config-from=victoria-config',
+ config: {
+ options: {},
+ values: {},
+ },
+}

examples/helm3/components/victoria-metrics/index.jsonnet

@@ -0,0 +1 @@
+(import 'datasource.libsonnet').objects

examples/helm3/environments/base.libsonnet

@@ -0,0 +1,5 @@
+// this file has the baseline default parameters
+{
+ components: {
+ },
+}

examples/helm3/environments/default.libsonnet

@@ -0,0 +1,7 @@
+// this file has the param overrides for the default environment
+local base = import './base.libsonnet';
+
+base {
+ components+: {
+ },
+}

examples/helm3/params.libsonnet

@@ -0,0 +1,9 @@
+// this file returns the params for the current qbec environment
+local env = std.extVar('qbec.io/env');
+local paramsMap = import 'glob-import:environments/*.libsonnet';
+local baseFile = if env == '_' then 'base' else env;
+local key = 'environments/%s.libsonnet' % baseFile;
+
+if std.objectHas(paramsMap, key)
+then paramsMap[key]
+else error 'no param file %s found for environment %s' % [key, env]

examples/helm3/qbec.yaml

@@ -0,0 +1,22 @@
+apiVersion: qbec.io/v1alpha1
+kind: App
+metadata:
+ name: helm3
+spec:
+ environments:
+ default:
+ defaultNamespace: charts
+ context: kind
+ vars:
+ computed:
+ - name: helmSetup
+ code: |
+ {}
+ - name: victoria-config
+ code: |
+ (import 'components/victoria-metrics/datasource.libsonnet').config
+ - name: apache-config
+ code: |
+ (import 'components/apache/datasource.libsonnet').config
+ dataSources:
+ - helm3://helm?configVar=helmSetup

go.mod

@@ -12,6 +12,7 @@ require (
  github.com/golang/protobuf v1.5.2
  github.com/google/go-jsonnet v0.18.0
  github.com/googleapis/gnostic v0.5.5
+ github.com/iancoleman/strcase v0.2.0
  github.com/jonboulle/clockwork v0.2.2
  github.com/mattn/go-isatty v0.0.14
  github.com/pkg/errors v0.9.1

go.sum

@@ -456,6 +456,7 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
 github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
 github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0=
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=

internal/remote/patch.go

@@ -116,10 +116,10 @@ func deleteEmpty(parent map[string]interface{}, key string) {
 // contains empty objects. It makes an assumption that there is actually no reason an empty object
 // needs to be updated for a Kubernetes resource considering that the server would already have an object
 // there on initial create if needed. Things considered empty will be of the form:
-// {}
-// { metadata: { labels: {}, annotations: {} }
-// { metadata: { labels: {}, annotations: {} }, spec: { foo: { bar: {} } } }
 //
+// {}
+// { metadata: { labels: {}, annotations: {} }
+// { metadata: { labels: {}, annotations: {} }, spec: { foo: { bar: {} } } }
 func isEmptyPatch(patch []byte) bool {
  var root map[string]interface{}
  err := json.Unmarshal(patch, &root)

site/content/userguide/usage/authoring.md

@@ -26,14 +26,20 @@ A component is:
 It is valid for a component to return an empty set of objects if runtime parameters determine that
 nothing should be installed for a specific target environment.
 
-## Using helm charts and external data sources
+## Using external data sources
 
 qbec provides integration to run external commands and consume their output in jsonnet code. 
 See the [Jsonnet data importer](../../../reference/jsonnet-external-data) for more information on how this works.
 
 Note that the [expandHelmTemplate](../../../reference/jsonnet-native-funcs/#expandhelmtemplate) native function
 is now deprecated in favor of the data importer mechanism.
 
+## Native Helm integration
+
+qbec provides native support for Helm3, allowing you to render Helm `values` during runtime via jsonnet. To use this
+feature, configure a Helm datasource. See [examples/helm3](https://github.com/splunk/qbec/tree/main/examples/helm3/) for
+an example component.
+
 ## Using other jsonnet libraries
 
 [k8s-yaml-patch](https://github.com/splunk/k8s-yaml-patch),

vm/internal/ds/exec/exec.go

@@ -121,11 +121,11 @@ func (d *execSource) Init(p datasource.ConfigProvider) (fErr error) {
  if err != nil {
  return err
  }
+ c.initDefaults()
  err = c.assertValid()
  if err != nil {
  return err
  }
- c.initDefaults()
  d.runner = newRunner(&c)
  return nil
 }

vm/internal/ds/factory/datasource.go

@@ -21,6 +21,7 @@ import (
  "github.com/pkg/errors"
  "github.com/splunk/qbec/vm/internal/ds"
  "github.com/splunk/qbec/vm/internal/ds/exec"
+ "github.com/splunk/qbec/vm/internal/ds/helm3"
 )
 
 // Create creates a new data source from the supplied URL.
@@ -35,6 +36,7 @@ func Create(u string) (ds.DataSourceWithLifecycle, error) {
  scheme := parsed.Scheme
  switch scheme {
  case exec.Scheme:
+ case helm3.Scheme:
  default:
  return nil, fmt.Errorf("data source URL '%s', unsupported scheme '%s'", u, scheme)
  }
@@ -52,6 +54,8 @@ func Create(u string) (ds.DataSourceWithLifecycle, error) {
  switch scheme {
  case exec.Scheme:
  return makeLazy(exec.New(name, varName)), nil
+ case helm3.Scheme:
+ return makeLazy(helm3.New(name, varName)), nil
  default:
  return nil, fmt.Errorf("internal error: unable to create a data source for %s", u)
  }