Enable setting custom values during porter build. (#1900)

* Enable custom values for porter build command Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com> * Add description and example for nested value Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com> * Change function name to FlattenMap and move it to its own package Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com> * Remove checking for parsedCustoms Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com> * Use existing porter.yaml for testing and remove the other one Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com> * Update testfiles Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com> * Fix integration test Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com> * Add documentation to custom section of Porter manifest Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com> * Use StringArrayVar instead of StringSliceVar for --custom flag on porter build Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com> * Add explanation on how the --custom flag will be processed during the build process Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com> * Updated docs using mage build Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com> * Update doc based on review Signed-off-by: joshuabezaleel <joshua.bezaleel@gmail.com>
getporter · Apr 28, 2022 · 62435ad · 62435ad
1 parent 4a7fd67
commit 62435ad
Show file tree

Hide file tree

Showing 16 changed files with 250 additions and 7 deletions.
diff --git a/cmd/porter/bundle.go b/cmd/porter/bundle.go
@@ -58,6 +58,7 @@ func buildBundleBuildCommand(p *porter.Porter) *cobra.Command {
   porter build --version 0.1.0
   porter build --file path/to/porter.yaml
   porter build --dir path/to/build/context
+  porter build --custom version=0.2.0 --custom myapp.version=0.1.2
 `,
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			return opts.Validate(p)
@@ -84,9 +85,12 @@ func buildBundleBuildCommand(p *porter.Porter) *cobra.Command {
 	f.StringArrayVar(&opts.SSH, "ssh", nil,
 		"SSH agent socket or keys to expose to the build (format: default|<id>[=<socket>|<key>[,<key>]]). May be specified multiple times.")
 	f.StringArrayVar(&opts.Secrets, "secret", nil,
-		"Secret file to expose to the build (format: id=mysecret,src=/local/secret). May be specified multiple times.")
+		"Secret file to expose to the build (format: id=mysecret,src=/local/secret). Custom values are assessible as build arguments in the template Dockerfile and in the manifest using template variables. May be specified multiple times.")
 	f.BoolVar(&opts.NoCache, "no-cache", false,
 		"Do not use the Docker cache when building the bundle's invocation image.")
+	f.StringArrayVar(&opts.Customs, "custom", nil,
+		"Define an individual key-value pair for the custom section in the form of NAME=VALUE. Use dot notation to specify a nested custom field. May be specified multiple times.")
+
 	// Allow configuring the --driver flag with build-driver, to avoid conflicts with other commands
 	cmd.Flag("driver").Annotations = map[string][]string{
 		"viper-key": {"build-driver"},

diff --git a/docs/content/bundle/manifest/_index.md b/docs/content/bundle/manifest/_index.md
@@ -500,6 +500,7 @@ data.
 
 ```yaml
 custom:
+  custom-config: "custom-value"
   some-custom-config:
     item: "value"
   more-custom-config:
@@ -511,6 +512,12 @@ You can access custom data at runtime using the `bundle.custom.KEY.SUBKEY` templ
 For example, `{{ bundle.custom.more-custom-config.enabled}}` allows you to
 access nested values from the custom section.
 
+Multiple custom values that were defined in the manifest can also be injected with new values during build time using the \--custom values tied to the `porter build` command. Currently only supports string values. You can use dot notation to specify a nested field:
+
+```
+porter build --custom custom-config=new-custom-value --custom some-custom-config.item=edited-value
+```
+
 See the [Custom Extensions](https://github.com/cnabio/cnab-spec/blob/master/101-bundle-json.md#custom-extensions)
 section of the CNAB Specification for more details.
 

diff --git a/docs/content/cli/build.md b/docs/content/cli/build.md
@@ -23,20 +23,22 @@ porter build [flags]
   porter build --version 0.1.0
   porter build --file path/to/porter.yaml
   porter build --dir path/to/build/context
+  porter build --custom version=0.2.0 --custom myapp.version=0.1.2
 
 ```
 
 ### Options
 
 ```
       --build-arg stringArray   Set build arguments in the template Dockerfile (format: NAME=VALUE). May be specified multiple times.
+      --custom stringArray      Define an individual key-value pair for the custom section in the form of NAME=VALUE. Use dot notation to specify a nested custom field. May be specified multiple times.
   -d, --dir string              Path to the build context directory where all bundle assets are located.
   -f, --file porter.yaml        Path to the Porter manifest. Defaults to porter.yaml in the current directory.
   -h, --help                    help for build
       --name string             Override the bundle name
       --no-cache                Do not use the Docker cache when building the bundle's invocation image.
       --no-lint                 Do not run the linter
-      --secret stringArray      Secret file to expose to the build (format: id=mysecret,src=/local/secret). May be specified multiple times.
+      --secret stringArray      Secret file to expose to the build (format: id=mysecret,src=/local/secret). Custom values are assessible as build arguments in the template Dockerfile and in the manifest using template variables. May be specified multiple times.
       --ssh stringArray         SSH agent socket or keys to expose to the build (format: default|<id>[=<socket>|<key>[,<key>]]). May be specified multiple times.
   -v, --verbose                 Enable verbose logging
       --version string          Override the bundle version

diff --git a/docs/content/cli/bundles_build.md b/docs/content/cli/bundles_build.md
@@ -23,20 +23,22 @@ porter bundles build [flags]
   porter build --version 0.1.0
   porter build --file path/to/porter.yaml
   porter build --dir path/to/build/context
+  porter build --custom version=0.2.0 --custom myapp.version=0.1.2
 
 ```
 
 ### Options
 
 ```
       --build-arg stringArray   Set build arguments in the template Dockerfile (format: NAME=VALUE). May be specified multiple times.
+      --custom stringArray      Define an individual key-value pair for the custom section in the form of NAME=VALUE. Use dot notation to specify a nested custom field. May be specified multiple times.
   -d, --dir string              Path to the build context directory where all bundle assets are located.
   -f, --file porter.yaml        Path to the Porter manifest. Defaults to porter.yaml in the current directory.
   -h, --help                    help for build
       --name string             Override the bundle name
       --no-cache                Do not use the Docker cache when building the bundle's invocation image.
       --no-lint                 Do not run the linter
-      --secret stringArray      Secret file to expose to the build (format: id=mysecret,src=/local/secret). May be specified multiple times.
+      --secret stringArray      Secret file to expose to the build (format: id=mysecret,src=/local/secret). Custom values are assessible as build arguments in the template Dockerfile and in the manifest using template variables. May be specified multiple times.
       --ssh stringArray         SSH agent socket or keys to expose to the build (format: default|<id>[=<socket>|<key>[,<key>]]). May be specified multiple times.
   -v, --verbose                 Enable verbose logging
       --version string          Override the bundle version

diff --git a/pkg/build/buildkit/buildx.go b/pkg/build/buildkit/buildx.go
@@ -116,6 +116,16 @@ func (b *Builder) BuildInvocationImage(ctx context.Context, manifest *manifest.M
 	parseBuildArgs(opts.BuildArgs, args)
 	args["BUNDLE_DIR"] = build.BUNDLE_DIR
 
+	convertedCustomInput := make(map[string]string)
+	convertedCustomInput, err = flattenMap(manifest.Custom)
+	if err != nil {
+		return err
+	}
+
+	for k, v := range convertedCustomInput {
+		args[strings.ToUpper(strings.Replace(k, ".", "_", -1))] = v
+	}
+
 	buildxOpts := map[string]buildx.Options{
 		"default": {
 			Tags: []string{manifest.Image},
@@ -198,3 +208,31 @@ func (b *Builder) TagInvocationImage(ctx context.Context, origTag, newTag string
 	}
 	return nil
 }
+
+// flattenMap recursively walks through nested map and flattent it
+// to one-level map of key-value with string type.
+func flattenMap(mapInput map[string]interface{}) (map[string]string, error) {
+	out := make(map[string]string)
+
+	for key, value := range mapInput {
+		switch v := value.(type) {
+		case string:
+			out[key] = v
+		case map[string]interface{}:
+			tmp, err := flattenMap(v)
+			if err != nil {
+				return nil, err
+			}
+			for innerKey, innerValue := range tmp {
+				out[key+"."+innerKey] = innerValue
+			}
+		case map[string]string:
+			for innerKey, innerValue := range v {
+				out[key+"."+innerKey] = innerValue
+			}
+		default:
+			return nil, errors.Errorf("Unknown type %#v: %t", v, v)
+		}
+	}
+	return out, nil
+}
diff --git a/pkg/build/buildkit/buildx_test.go b/pkg/build/buildkit/buildx_test.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func Test_parseBuildArgs(t *testing.T) {
@@ -26,3 +27,86 @@ func Test_parseBuildArgs(t *testing.T) {
 		})
 	}
 }
+
+func Test_flattenMap(t *testing.T) {
+	tt := []struct {
+		desc string
+		inp  map[string]interface{}
+		out  map[string]string
+		err  bool
+	}{
+		{
+			desc: "one pair",
+			inp: map[string]interface{}{
+				"key": "value",
+			},
+			out: map[string]string{
+				"key": "value",
+			},
+			err: false,
+		},
+		{
+			desc: "nested input",
+			inp: map[string]interface{}{
+				"key": map[string]string{
+					"nestedKey": "value",
+				},
+			},
+			out: map[string]string{
+				"key.nestedKey": "value",
+			},
+			err: false,
+		},
+		{
+			desc: "nested input",
+			inp: map[string]interface{}{
+				"key1": map[string]interface{}{
+					"key2": map[string]string{
+						"key3": "value",
+					},
+				},
+			},
+			out: map[string]string{
+				"key1.key2.key3": "value",
+			},
+			err: false,
+		},
+		{
+			desc: "multiple nested input",
+			inp: map[string]interface{}{
+				"key11": map[string]interface{}{
+					"key12": map[string]string{
+						"key13": "value1",
+					},
+				},
+				"key21": map[string]string{
+					"key22": "value2",
+				},
+			},
+			out: map[string]string{
+				"key11.key12.key13": "value1",
+				"key21.key22":       "value2",
+			},
+			err: false,
+		},
+		{
+			desc: "empty interface value other than map[string]interface{}, map[string]string or string",
+			inp: map[string]interface{}{
+				"a": 1,
+			},
+			err: true,
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.desc, func(t *testing.T) {
+			out, err := flattenMap(tc.inp)
+			if tc.err {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, out, tc.out)
+		})
+	}
+}
diff --git a/pkg/porter/build.go b/pkg/porter/build.go
@@ -12,6 +12,7 @@ import (
 	"get.porter.sh/porter/pkg/config"
 	"get.porter.sh/porter/pkg/manifest"
 	"get.porter.sh/porter/pkg/mixin"
+	"get.porter.sh/porter/pkg/parameters"
 	"get.porter.sh/porter/pkg/printer"
 	"github.com/Masterminds/semver/v3"
 	"github.com/opencontainers/go-digest"
@@ -29,6 +30,12 @@ type BuildOptions struct {
 
 	// Driver to use when building the invocation image.
 	Driver string
+
+	// Custom is the unparsed list of NAME=VALUE custom inputs set on the command line.
+	Customs []string
+
+	// parsedCustoms is the parsed set of custom inputs from Customs.
+	parsedCustoms map[string]string
 }
 
 const BuildDriverDefault = config.BuildDriverBuildkit
@@ -56,6 +63,11 @@ func (o *BuildOptions) Validate(p *Porter) error {
 	// This would be less awkward if we didn't do an automatic build during publish
 	p.Data.BuildDriver = o.Driver
 
+	err := o.parseCustomInputs()
+	if err != nil {
+		return err
+	}
+
 	return o.bundleFileOptions.Validate(p.Context)
 }
 
@@ -68,6 +80,17 @@ func stringSliceContains(allowedValues []string, value string) bool {
 	return false
 }
 
+func (o *BuildOptions) parseCustomInputs() error {
+	p, err := parameters.ParseVariableAssignments(o.Customs)
+	if err != nil {
+		return err
+	}
+
+	o.parsedCustoms = p
+
+	return nil
+}
+
 func (p *Porter) Build(ctx context.Context, opts BuildOptions) error {
 	opts.Apply(p.Context)
 

diff --git a/pkg/porter/build_integration_test.go b/pkg/porter/build_integration_test.go
@@ -265,3 +265,27 @@ func TestBuildOptions_Defaults(t *testing.T) {
 		assert.Equal(t, config.BuildDriverBuildkit, opts.Driver)
 	})
 }
+
+func TestPorter_BuildWithCustomValues(t *testing.T) {
+	p := NewTestPorter(t)
+	defer p.Teardown()
+
+	p.TestConfig.TestContext.AddTestFile("./testdata/porter.yaml", config.Name)
+
+	m, err := manifest.LoadManifestFrom(context.Background(), p.Config, config.Name)
+	require.NoError(t, err)
+
+	err = p.buildBundle(m, "digest")
+	require.NoError(t, err)
+
+	opts := BuildOptions{Customs: []string{"customKey1=editedCustomValue1"}}
+	require.NoError(t, opts.Validate(p.Porter), "Validate failed")
+
+	err = p.Build(context.Background(), opts)
+	require.NoError(t, err)
+
+	bun, err := p.CNAB.LoadBundle(build.LOCAL_BUNDLE)
+	require.NoError(t, err)
+
+	assert.Equal(t, bun.Custom["customKey1"], "editedCustomValue1")
+}
diff --git a/pkg/porter/generateManifest.go b/pkg/porter/generateManifest.go
@@ -42,5 +42,11 @@ func (p *Porter) generateInternalManifest(opts BuildOptions) error {
 		}
 	}
 
+	for k, v := range opts.parsedCustoms {
+		if err = e.SetValue("custom."+k, v); err != nil {
+			return err
+		}
+	}
+
 	return e.WriteFile(build.LOCAL_MANIFEST)
 }
diff --git a/pkg/porter/generateManifest_test.go b/pkg/porter/generateManifest_test.go
@@ -39,6 +39,10 @@ func Test_generateInternalManifest(t *testing.T) {
 		name:         "name and value set",
 		opts:         BuildOptions{metadataOpts: metadataOpts{Name: "newname", Version: "1.0.0"}},
 		wantManifest: "all-fields.yaml",
+	}, {
+		name:         "custom input set",
+		opts:         BuildOptions{Customs: []string{"key1=editedValue1", "key2.nestedKey2=editedValue2"}},
+		wantManifest: "custom-input.yaml",
 	}}
 
 	p := NewTestPorter(t)

diff --git a/pkg/porter/testdata/generateManifest/all-fields.yaml b/pkg/porter/testdata/generateManifest/all-fields.yaml
@@ -3,6 +3,10 @@ name: newname
 version: 1.0.0
 description: "An example Porter configuration"
 registry: "localhost:5000"
+custom:
+  key1: value1
+  key2:
+    nestedKey2: value2
 mixins:
   - exec
 install:

diff --git a/pkg/porter/testdata/generateManifest/custom-input.yaml b/pkg/porter/testdata/generateManifest/custom-input.yaml
@@ -0,0 +1,30 @@
+schemaVersion: 1.0.0-alpha.1
+name: porter-hello
+version: 0.1.0
+description: "An example Porter configuration"
+registry: "localhost:5000"
+custom:
+  key1: editedValue1
+  key2:
+    nestedKey2: editedValue2
+mixins:
+  - exec
+install:
+  - exec:
+      description: "Install Hello World"
+      command: ./helpers.sh
+      arguments:
+        - install
+status:
+  - exec:
+      description: "World Status"
+      command: ./helpers.sh
+      arguments:
+        - status
+uninstall:
+  - exec:
+      description: "Uninstall Hello World"
+      command: ./helpers.sh
+      arguments:
+        - uninstall
+# comments n stuff
diff --git a/pkg/porter/testdata/generateManifest/new-name.yaml b/pkg/porter/testdata/generateManifest/new-name.yaml
@@ -3,6 +3,10 @@ name: newname
 version: 0.1.0
 description: "An example Porter configuration"
 registry: "localhost:5000"
+custom:
+  key1: value1
+  key2:
+    nestedKey2: value2
 mixins:
   - exec
 install: