From d08a650670589976b609c31f3acba5706309a7d1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Jun 2024 18:57:34 +0200 Subject: [PATCH 1/4] Serialize dynamic value for bundle validate output --- bundle/config/root.go | 6 ++++ cmd/bundle/validate.go | 2 +- internal/bundle/helpers.go | 7 ++++ internal/bundle/validate_test.go | 60 ++++++++++++++++++++++++++++++++ internal/testutil/touch.go | 12 +++++++ 5 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 internal/bundle/validate_test.go diff --git a/bundle/config/root.go b/bundle/config/root.go index 88197c2b87..2bc905bd60 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -471,3 +471,9 @@ func (r Root) GetLocation(path string) dyn.Location { } return v.Location() } + +// Value returns the dynamic configuration value of the root object. This value +// is the source of truth and is kept in sync with values in the typed configuration. +func (r Root) Value() dyn.Value { + return r.value +} diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go index 8d49ec961b..a1f8d26819 100644 --- a/cmd/bundle/validate.go +++ b/cmd/bundle/validate.go @@ -119,7 +119,7 @@ func renderTextOutput(cmd *cobra.Command, b *bundle.Bundle, diags diag.Diagnosti } func renderJsonOutput(cmd *cobra.Command, b *bundle.Bundle, diags diag.Diagnostics) error { - buf, err := json.MarshalIndent(b.Config, "", " ") + buf, err := json.MarshalIndent(b.Config.Value().AsAny(), "", " ") if err != nil { return err } diff --git a/internal/bundle/helpers.go b/internal/bundle/helpers.go index 560a0474b1..a17964b167 100644 --- a/internal/bundle/helpers.go +++ b/internal/bundle/helpers.go @@ -51,6 +51,13 @@ func writeConfigFile(t *testing.T, config map[string]any) (string, error) { return filepath, err } +func validateBundle(t *testing.T, ctx context.Context, path string) ([]byte, error) { + t.Setenv("BUNDLE_ROOT", path) + c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "validate", "--output", "json") + stdout, _, err := c.Run() + return stdout.Bytes(), err +} + func deployBundle(t *testing.T, ctx context.Context, path string) error { t.Setenv("BUNDLE_ROOT", path) c := internal.NewCobraTestRunnerWithContext(t, ctx, "bundle", "deploy", "--force-lock") diff --git a/internal/bundle/validate_test.go b/internal/bundle/validate_test.go new file mode 100644 index 0000000000..e4e24d3d32 --- /dev/null +++ b/internal/bundle/validate_test.go @@ -0,0 +1,60 @@ +package bundle + +import ( + "context" + "encoding/json" + "testing" + + "github.com/databricks/cli/internal/testutil" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAccBundleValidate(t *testing.T) { + testutil.GetEnvOrSkipTest(t, "CLOUD_ENV") + + tmpDir := t.TempDir() + testutil.TouchWithContent(t, + ` +bundle: + name: "foobar" + +resources: + jobs: + outer_loop: + name: outer loop + tasks: + - task_key: my task + run_job_task: + job_id: ${resources.jobs.inner_loop.id} + + inner_loop: + name: inner loop + +`, tmpDir, "databricks.yml") + + ctx := context.Background() + stdout, err := validateBundle(t, ctx, tmpDir) + require.NoError(t, err) + + config := make(map[string]any) + err = json.Unmarshal(stdout, &config) + + getValue := func(key string) any { + v, err := convert.FromTyped(config, dyn.NilValue) + require.NoError(t, err) + v, err = dyn.GetByPath(v, dyn.MustPathFromString(key)) + require.NoError(t, err) + return v.AsAny() + } + require.NoError(t, err) + + assert.Equal(t, "foobar", getValue("bundle.name")) + assert.Equal(t, "outer loop", getValue("resources.jobs.outer_loop.name")) + assert.Equal(t, "inner loop", getValue("resources.jobs.inner_loop.name")) + assert.Equal(t, "my task", getValue("resources.jobs.outer_loop.tasks[0].task_key")) + // Assert resource references are retained in the output. + assert.Equal(t, "${resources.jobs.inner_loop.id}", getValue("resources.jobs.outer_loop.tasks[0].run_job_task.job_id")) +} diff --git a/internal/testutil/touch.go b/internal/testutil/touch.go index 55683f3eda..4ebd7c5e7c 100644 --- a/internal/testutil/touch.go +++ b/internal/testutil/touch.go @@ -24,3 +24,15 @@ func Touch(t *testing.T, elems ...string) string { f.Close() return path } + +func TouchWithContent(t *testing.T, content string, elems ...string) string { + path := filepath.Join(elems...) + os.MkdirAll(filepath.Dir(path), 0755) + f, err := os.Create(path) + require.NoError(t, err) + _, err = f.WriteString(content) + require.NoError(t, err) + err = f.Close() + require.NoError(t, err) + return path +} From 9c31e7adb75c6c22236b79dfaf8ebde6480389bf Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Jun 2024 16:19:36 +0200 Subject: [PATCH 2/4] address comments --- internal/bundle/validate_test.go | 5 +++-- internal/testutil/{touch.go => file.go} | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) rename internal/testutil/{touch.go => file.go} (91%) diff --git a/internal/bundle/validate_test.go b/internal/bundle/validate_test.go index e4e24d3d32..3f7d7ae809 100644 --- a/internal/bundle/validate_test.go +++ b/internal/bundle/validate_test.go @@ -16,7 +16,7 @@ func TestAccBundleValidate(t *testing.T) { testutil.GetEnvOrSkipTest(t, "CLOUD_ENV") tmpDir := t.TempDir() - testutil.TouchWithContent(t, + testutil.WriteFile(t, ` bundle: name: "foobar" @@ -41,6 +41,7 @@ resources: config := make(map[string]any) err = json.Unmarshal(stdout, &config) + require.NoError(t, err) getValue := func(key string) any { v, err := convert.FromTyped(config, dyn.NilValue) @@ -49,7 +50,7 @@ resources: require.NoError(t, err) return v.AsAny() } - require.NoError(t, err) + assert.Equal(t, "foobar", getValue("bundle.name")) assert.Equal(t, "outer loop", getValue("resources.jobs.outer_loop.name")) diff --git a/internal/testutil/touch.go b/internal/testutil/file.go similarity index 91% rename from internal/testutil/touch.go rename to internal/testutil/file.go index 4ebd7c5e7c..7abe3de020 100644 --- a/internal/testutil/touch.go +++ b/internal/testutil/file.go @@ -25,7 +25,7 @@ func Touch(t *testing.T, elems ...string) string { return path } -func TouchWithContent(t *testing.T, content string, elems ...string) string { +func WriteFile(t *testing.T, content string, elems ...string) string { path := filepath.Join(elems...) os.MkdirAll(filepath.Dir(path), 0755) f, err := os.Create(path) From 301bb6334a0cfa49ec52856488cf917a8f4ca94c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Jun 2024 16:23:12 +0200 Subject: [PATCH 3/4] add error handling --- internal/testutil/file.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/internal/testutil/file.go b/internal/testutil/file.go index 7abe3de020..ba2c3280e2 100644 --- a/internal/testutil/file.go +++ b/internal/testutil/file.go @@ -10,28 +10,38 @@ import ( func TouchNotebook(t *testing.T, elems ...string) string { path := filepath.Join(elems...) - os.MkdirAll(filepath.Dir(path), 0755) - err := os.WriteFile(path, []byte("# Databricks notebook source"), 0644) + err := os.MkdirAll(filepath.Dir(path), 0755) + require.NoError(t, err) + + err = os.WriteFile(path, []byte("# Databricks notebook source"), 0644) require.NoError(t, err) return path } func Touch(t *testing.T, elems ...string) string { path := filepath.Join(elems...) - os.MkdirAll(filepath.Dir(path), 0755) + err := os.MkdirAll(filepath.Dir(path), 0755) + require.NoError(t, err) + f, err := os.Create(path) require.NoError(t, err) - f.Close() + + err = f.Close() + require.NoError(t, err) return path } func WriteFile(t *testing.T, content string, elems ...string) string { path := filepath.Join(elems...) - os.MkdirAll(filepath.Dir(path), 0755) + err := os.MkdirAll(filepath.Dir(path), 0755) + require.NoError(t, err) + f, err := os.Create(path) require.NoError(t, err) + _, err = f.WriteString(content) require.NoError(t, err) + err = f.Close() require.NoError(t, err) return path From 927cd23cbdfea0308ad973fce4224df6805dc470 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Jun 2024 16:24:34 +0200 Subject: [PATCH 4/4] fmt --- internal/bundle/validate_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/bundle/validate_test.go b/internal/bundle/validate_test.go index 3f7d7ae809..18da89e4ce 100644 --- a/internal/bundle/validate_test.go +++ b/internal/bundle/validate_test.go @@ -51,7 +51,6 @@ resources: return v.AsAny() } - assert.Equal(t, "foobar", getValue("bundle.name")) assert.Equal(t, "outer loop", getValue("resources.jobs.outer_loop.name")) assert.Equal(t, "inner loop", getValue("resources.jobs.inner_loop.name"))