From 2adc677c9eca0da3d0d9ff1ef2a208e2e7babb2c Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 26 Mar 2024 17:18:52 +0100 Subject: [PATCH 1/4] Move path field to bundle type The bundle path was previously stored on the `config.Root` type under the assumption that the first configuration file being loaded would set it. This is slightly counterintuitive and we know what the path is upon construction of the bundle. The new location for this property reflects this. --- bundle/artifacts/build.go | 2 +- bundle/artifacts/upload_test.go | 4 +-- bundle/artifacts/whl/autodetect.go | 6 ++-- bundle/artifacts/whl/from_libraries.go | 2 +- bundle/bundle.go | 12 ++++--- bundle/bundle_test.go | 4 +-- .../expand_pipeline_glob_paths_test.go | 2 +- bundle/config/mutator/load_git_details.go | 4 +-- bundle/config/mutator/process_include_test.go | 4 +-- .../config/mutator/process_root_includes.go | 8 ++--- .../mutator/process_root_includes_test.go | 34 ++++++++----------- bundle/config/mutator/rewrite_sync_paths.go | 4 +-- .../config/mutator/rewrite_sync_paths_test.go | 10 +++--- bundle/config/mutator/trampoline.go | 2 +- bundle/config/mutator/trampoline_test.go | 2 +- bundle/config/mutator/translate_paths.go | 2 +- bundle/config/mutator/translate_paths_test.go | 24 ++++++------- bundle/config/root.go | 11 +----- bundle/deploy/files/sync.go | 2 +- bundle/deploy/metadata/compute.go | 2 +- bundle/deploy/state_pull.go | 2 +- bundle/deploy/state_pull_test.go | 10 +++--- bundle/deploy/state_push_test.go | 2 +- bundle/deploy/state_update_test.go | 12 +++---- bundle/deploy/terraform/init_test.go | 14 ++++---- bundle/deploy/terraform/load_test.go | 2 +- bundle/deploy/terraform/state_pull_test.go | 2 +- bundle/deploy/terraform/state_push_test.go | 2 +- bundle/libraries/libraries.go | 2 +- bundle/libraries/libraries_test.go | 2 +- bundle/python/conditional_transform_test.go | 6 ++-- bundle/python/transform_test.go | 2 +- bundle/root_test.go | 4 +-- bundle/scripts/scripts.go | 2 +- bundle/scripts/scripts_test.go | 2 +- bundle/tests/python_wheel_test.go | 4 +-- cmd/bundle/generate/generate_test.go | 9 ++--- cmd/sync/sync_test.go | 3 +- internal/bundle/artifacts_test.go | 2 +- 39 files changed, 102 insertions(+), 123 deletions(-) diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index f3ee097c28..5f7e111528 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -46,7 +46,7 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // If artifact path is not provided, use bundle root dir if artifact.Path == "" { - artifact.Path = b.Config.Path + artifact.Path = b.Path } if !filepath.IsAbs(artifact.Path) { diff --git a/bundle/artifacts/upload_test.go b/bundle/artifacts/upload_test.go index ec71100958..ec7ee73a17 100644 --- a/bundle/artifacts/upload_test.go +++ b/bundle/artifacts/upload_test.go @@ -36,8 +36,8 @@ func TestExpandGlobFilesSource(t *testing.T) { t2.Close(t) b := &bundle.Bundle{ + Path: rootPath, Config: config.Root{ - Path: rootPath, Artifacts: map[string]*config.Artifact{ "test": { Type: "custom", @@ -72,8 +72,8 @@ func TestExpandGlobFilesSourceWithNoMatches(t *testing.T) { require.NoError(t, err) b := &bundle.Bundle{ + Path: rootPath, Config: config.Root{ - Path: rootPath, Artifacts: map[string]*config.Artifact{ "test": { Type: "custom", diff --git a/bundle/artifacts/whl/autodetect.go b/bundle/artifacts/whl/autodetect.go index d11db83110..ddb1797f43 100644 --- a/bundle/artifacts/whl/autodetect.go +++ b/bundle/artifacts/whl/autodetect.go @@ -35,21 +35,21 @@ func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic log.Infof(ctx, "Detecting Python wheel project...") // checking if there is setup.py in the bundle root - setupPy := filepath.Join(b.Config.Path, "setup.py") + setupPy := filepath.Join(b.Path, "setup.py") _, err := os.Stat(setupPy) if err != nil { log.Infof(ctx, "No Python wheel project found at bundle root folder") return nil } - log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.Config.Path)) + log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.Path)) module := extractModuleName(setupPy) if b.Config.Artifacts == nil { b.Config.Artifacts = make(map[string]*config.Artifact) } - pkgPath, err := filepath.Abs(b.Config.Path) + pkgPath, err := filepath.Abs(b.Path) if err != nil { return diag.FromErr(err) } diff --git a/bundle/artifacts/whl/from_libraries.go b/bundle/artifacts/whl/from_libraries.go index a2045aaf8c..f3660bbae9 100644 --- a/bundle/artifacts/whl/from_libraries.go +++ b/bundle/artifacts/whl/from_libraries.go @@ -30,7 +30,7 @@ func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost tasks := libraries.FindAllWheelTasksWithLocalLibraries(b) for _, task := range tasks { for _, lib := range task.Libraries { - matches, err := filepath.Glob(filepath.Join(b.Config.Path, lib.Whl)) + matches, err := filepath.Glob(filepath.Join(b.Path, lib.Whl)) // File referenced from libraries section does not exists, skipping if err != nil { continue diff --git a/bundle/bundle.go b/bundle/bundle.go index a178ea090b..b1c514751e 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -30,6 +30,10 @@ import ( const internalFolder = ".internal" type Bundle struct { + // Path contains the directory path to the root of the bundle. + // It is set when we instantiate a new bundle instance. + Path string + Config config.Root // Metadata about the bundle deployment. This is the interface Databricks services @@ -74,8 +78,8 @@ func Load(ctx context.Context, path string) (*Bundle, error) { _, hasIncludesEnv := env.Includes(ctx) if hasRootEnv && hasIncludesEnv && stat.IsDir() { log.Debugf(ctx, "No bundle configuration; using bundle root: %s", path) + b.Path = path b.Config = config.Root{ - Path: path, Bundle: config.Bundle{ Name: filepath.Base(path), }, @@ -158,7 +162,7 @@ func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error) if !exists || cacheDirName == "" { cacheDirName = filepath.Join( // Anchor at bundle root directory. - b.Config.Path, + b.Path, // Static cache directory. ".databricks", "bundle", @@ -210,7 +214,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) { if err != nil { return nil, err } - internalDirRel, err := filepath.Rel(b.Config.Path, internalDir) + internalDirRel, err := filepath.Rel(b.Path, internalDir) if err != nil { return nil, err } @@ -218,7 +222,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) { } func (b *Bundle) GitRepository() (*git.Repository, error) { - rootPath, err := folders.FindDirWithLeaf(b.Config.Path, ".git") + rootPath, err := folders.FindDirWithLeaf(b.Path, ".git") if err != nil { return nil, fmt.Errorf("unable to locate repository root: %w", err) } diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 887a4ee83f..0ebc4c9be8 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -77,7 +77,7 @@ func TestBundleMustLoadSuccess(t *testing.T) { t.Setenv(env.RootVariable, "./tests/basic") b, err := MustLoad(context.Background()) require.NoError(t, err) - assert.Equal(t, "tests/basic", filepath.ToSlash(b.Config.Path)) + assert.Equal(t, "tests/basic", filepath.ToSlash(b.Path)) } func TestBundleMustLoadFailureWithEnv(t *testing.T) { @@ -96,7 +96,7 @@ func TestBundleTryLoadSuccess(t *testing.T) { t.Setenv(env.RootVariable, "./tests/basic") b, err := TryLoad(context.Background()) require.NoError(t, err) - assert.Equal(t, "tests/basic", filepath.ToSlash(b.Config.Path)) + assert.Equal(t, "tests/basic", filepath.ToSlash(b.Path)) } func TestBundleTryLoadFailureWithEnv(t *testing.T) { diff --git a/bundle/config/mutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/expand_pipeline_glob_paths_test.go index db80be028a..effa6f244c 100644 --- a/bundle/config/mutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/expand_pipeline_glob_paths_test.go @@ -41,8 +41,8 @@ func TestExpandGlobPathsInPipelines(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "skip/test7.py")) b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { diff --git a/bundle/config/mutator/load_git_details.go b/bundle/config/mutator/load_git_details.go index 6ff9aad622..4d354b8d70 100644 --- a/bundle/config/mutator/load_git_details.go +++ b/bundle/config/mutator/load_git_details.go @@ -22,7 +22,7 @@ func (m *loadGitDetails) Name() string { func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Load relevant git repository - repo, err := git.NewRepository(b.Config.Path) + repo, err := git.NewRepository(b.Path) if err != nil { return diag.FromErr(err) } @@ -56,7 +56,7 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } // Compute relative path of the bundle root from the Git repo root. - absBundlePath, err := filepath.Abs(b.Config.Path) + absBundlePath, err := filepath.Abs(b.Path) if err != nil { return diag.FromErr(err) } diff --git a/bundle/config/mutator/process_include_test.go b/bundle/config/mutator/process_include_test.go index 0e5351b634..7103d5074b 100644 --- a/bundle/config/mutator/process_include_test.go +++ b/bundle/config/mutator/process_include_test.go @@ -16,8 +16,8 @@ import ( func TestProcessInclude(t *testing.T) { b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Workspace: config.Workspace{ Host: "foo", }, @@ -25,7 +25,7 @@ func TestProcessInclude(t *testing.T) { } relPath := "./file.yml" - fullPath := filepath.Join(b.Config.Path, relPath) + fullPath := filepath.Join(b.Path, relPath) f, err := os.Create(fullPath) require.NoError(t, err) fmt.Fprint(f, "workspace:\n host: bar\n") diff --git a/bundle/config/mutator/process_root_includes.go b/bundle/config/mutator/process_root_includes.go index dbf99f2dc6..e4edabc532 100644 --- a/bundle/config/mutator/process_root_includes.go +++ b/bundle/config/mutator/process_root_includes.go @@ -51,7 +51,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // Converts extra include paths from environment variable to relative paths for _, extraIncludePath := range getExtraIncludePaths(ctx) { if filepath.IsAbs(extraIncludePath) { - rel, err := filepath.Rel(b.Config.Path, extraIncludePath) + rel, err := filepath.Rel(b.Path, extraIncludePath) if err != nil { return diag.Errorf("unable to include file '%s': %v", extraIncludePath, err) } @@ -70,7 +70,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. } // Anchor includes to the bundle root path. - matches, err := filepath.Glob(filepath.Join(b.Config.Path, entry)) + matches, err := filepath.Glob(filepath.Join(b.Path, entry)) if err != nil { return diag.FromErr(err) } @@ -84,7 +84,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // Filter matches to ones we haven't seen yet. var includes []string for _, match := range matches { - rel, err := filepath.Rel(b.Config.Path, match) + rel, err := filepath.Rel(b.Path, match) if err != nil { return diag.FromErr(err) } @@ -99,7 +99,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. slices.Sort(includes) files = append(files, includes...) for _, include := range includes { - out = append(out, ProcessInclude(filepath.Join(b.Config.Path, include), include)) + out = append(out, ProcessInclude(filepath.Join(b.Path, include), include)) } } diff --git a/bundle/config/mutator/process_root_includes_test.go b/bundle/config/mutator/process_root_includes_test.go index 7b21945539..c356628635 100644 --- a/bundle/config/mutator/process_root_includes_test.go +++ b/bundle/config/mutator/process_root_includes_test.go @@ -19,9 +19,7 @@ import ( func TestProcessRootIncludesEmpty(t *testing.T) { b := &bundle.Bundle{ - Config: config.Root{ - Path: ".", - }, + Path: ".", } diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -36,8 +34,8 @@ func TestProcessRootIncludesAbs(t *testing.T) { } b := &bundle.Bundle{ + Path: ".", Config: config.Root{ - Path: ".", Include: []string{ "/tmp/*.yml", }, @@ -50,17 +48,17 @@ func TestProcessRootIncludesAbs(t *testing.T) { func TestProcessRootIncludesSingleGlob(t *testing.T) { b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Include: []string{ "*.yml", }, }, } - testutil.Touch(t, b.Config.Path, "databricks.yml") - testutil.Touch(t, b.Config.Path, "a.yml") - testutil.Touch(t, b.Config.Path, "b.yml") + testutil.Touch(t, b.Path, "databricks.yml") + testutil.Touch(t, b.Path, "a.yml") + testutil.Touch(t, b.Path, "b.yml") diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -69,8 +67,8 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { func TestProcessRootIncludesMultiGlob(t *testing.T) { b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Include: []string{ "a*.yml", "b*.yml", @@ -78,8 +76,8 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) { }, } - testutil.Touch(t, b.Config.Path, "a1.yml") - testutil.Touch(t, b.Config.Path, "b1.yml") + testutil.Touch(t, b.Path, "a1.yml") + testutil.Touch(t, b.Path, "b1.yml") diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -88,8 +86,8 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) { func TestProcessRootIncludesRemoveDups(t *testing.T) { b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Include: []string{ "*.yml", "*.yml", @@ -97,7 +95,7 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) { }, } - testutil.Touch(t, b.Config.Path, "a.yml") + testutil.Touch(t, b.Path, "a.yml") diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -106,8 +104,8 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) { func TestProcessRootIncludesNotExists(t *testing.T) { b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Include: []string{ "notexist.yml", }, @@ -125,9 +123,7 @@ func TestProcessRootIncludesExtrasFromEnvVar(t *testing.T) { t.Setenv(env.IncludesVariable, path.Join(rootPath, testYamlName)) b := &bundle.Bundle{ - Config: config.Root{ - Path: rootPath, - }, + Path: rootPath, } diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) @@ -148,9 +144,7 @@ func TestProcessRootIncludesDedupExtrasFromEnvVar(t *testing.T) { )) b := &bundle.Bundle{ - Config: config.Root{ - Path: rootPath, - }, + Path: rootPath, } diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) diff --git a/bundle/config/mutator/rewrite_sync_paths.go b/bundle/config/mutator/rewrite_sync_paths.go index 0785c64300..ea56b7afe9 100644 --- a/bundle/config/mutator/rewrite_sync_paths.go +++ b/bundle/config/mutator/rewrite_sync_paths.go @@ -45,11 +45,11 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc { func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) { - v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.Config.Path))) + v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.Path))) if err != nil { return dyn.NilValue, err } - v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.Config.Path))) + v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.Path))) if err != nil { return dyn.NilValue, err } diff --git a/bundle/config/mutator/rewrite_sync_paths_test.go b/bundle/config/mutator/rewrite_sync_paths_test.go index 667f811ac9..55b9a826bf 100644 --- a/bundle/config/mutator/rewrite_sync_paths_test.go +++ b/bundle/config/mutator/rewrite_sync_paths_test.go @@ -14,8 +14,8 @@ import ( func TestRewriteSyncPathsRelative(t *testing.T) { b := &bundle.Bundle{ + Path: ".", Config: config.Root{ - Path: ".", Sync: config.Sync{ Include: []string{ "foo", @@ -45,8 +45,8 @@ func TestRewriteSyncPathsRelative(t *testing.T) { func TestRewriteSyncPathsAbsolute(t *testing.T) { b := &bundle.Bundle{ + Path: "/tmp/dir", Config: config.Root{ - Path: "/tmp/dir", Sync: config.Sync{ Include: []string{ "foo", @@ -77,9 +77,7 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { func TestRewriteSyncPathsErrorPaths(t *testing.T) { t.Run("no sync block", func(t *testing.T) { b := &bundle.Bundle{ - Config: config.Root{ - Path: ".", - }, + Path: ".", } diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) @@ -88,8 +86,8 @@ func TestRewriteSyncPathsErrorPaths(t *testing.T) { t.Run("empty include/exclude blocks", func(t *testing.T) { b := &bundle.Bundle{ + Path: ".", Config: config.Root{ - Path: ".", Sync: config.Sync{ Include: []string{}, Exclude: []string{}, diff --git a/bundle/config/mutator/trampoline.go b/bundle/config/mutator/trampoline.go index 72c053b594..f7e20ff98b 100644 --- a/bundle/config/mutator/trampoline.go +++ b/bundle/config/mutator/trampoline.go @@ -82,7 +82,7 @@ func (m *trampoline) generateNotebookWrapper(ctx context.Context, b *bundle.Bund return err } - internalDirRel, err := filepath.Rel(b.Config.Path, internalDir) + internalDirRel, err := filepath.Rel(b.Path, internalDir) if err != nil { return err } diff --git a/bundle/config/mutator/trampoline_test.go b/bundle/config/mutator/trampoline_test.go index 8a375aa9ba..088885a2be 100644 --- a/bundle/config/mutator/trampoline_test.go +++ b/bundle/config/mutator/trampoline_test.go @@ -57,8 +57,8 @@ func TestGenerateTrampoline(t *testing.T) { } b := &bundle.Bundle{ + Path: tmpDir, Config: config.Root{ - Path: tmpDir, Bundle: config.Bundle{ Target: "development", }, diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index af6896ee0d..768253e255 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -85,7 +85,7 @@ func (m *translatePaths) rewritePath( } // Remote path must be relative to the bundle root. - localRelPath, err := filepath.Rel(b.Config.Path, localPath) + localRelPath, err := filepath.Rel(b.Path, localPath) if err != nil { return err } diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index bd2ec809ba..701a0b9c8b 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -36,8 +36,8 @@ func touchEmptyFile(t *testing.T, path string) { func TestTranslatePathsSkippedWithGitSource(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Workspace: config.Workspace{ FilePath: "/bundle", }, @@ -106,8 +106,8 @@ func TestTranslatePaths(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar")) b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Workspace: config.Workspace{ FilePath: "/bundle", }, @@ -273,8 +273,8 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "job", "my_dbt_project", "dbt_project.yml")) b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Workspace: config.Workspace{ FilePath: "/bundle", }, @@ -367,8 +367,8 @@ func TestTranslatePathsOutsideBundleRoot(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Workspace: config.Workspace{ FilePath: "/bundle", }, @@ -400,8 +400,8 @@ func TestJobNotebookDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Resources: config.Resources{ Jobs: map[string]*resources.Job{ "job": { @@ -430,8 +430,8 @@ func TestJobFileDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Resources: config.Resources{ Jobs: map[string]*resources.Job{ "job": { @@ -460,8 +460,8 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { @@ -490,8 +490,8 @@ func TestPipelineFileDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ "pipeline": { @@ -521,8 +521,8 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) { touchNotebookFile(t, filepath.Join(dir, "my_notebook.py")) b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Workspace: config.Workspace{ FilePath: "/bundle", }, @@ -555,8 +555,8 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "my_file.py")) b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Workspace: config.Workspace{ FilePath: "/bundle", }, @@ -589,8 +589,8 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "my_file.py")) b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Workspace: config.Workspace{ FilePath: "/bundle", }, @@ -623,8 +623,8 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) { touchNotebookFile(t, filepath.Join(dir, "my_notebook.py")) b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Workspace: config.Workspace{ FilePath: "/bundle", }, diff --git a/bundle/config/root.go b/bundle/config/root.go index 8e1ff65077..a3dd0d28bb 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "os" - "path/filepath" "strings" "github.com/databricks/cli/bundle/config/resources" @@ -24,10 +23,6 @@ type Root struct { diags diag.Diagnostics depth int - // Path contains the directory path to the root of the bundle. - // It is set when loading `databricks.yml`. - Path string `json:"-" bundle:"readonly"` - // Contains user defined variables Variables map[string]*variable.Variable `json:"variables,omitempty"` @@ -80,9 +75,7 @@ func Load(path string) (*Root, error) { return nil, err } - r := Root{ - Path: filepath.Dir(path), - } + r := Root{} // Load configuration tree from YAML. v, err := yamlloader.LoadYAML(path, bytes.NewBuffer(raw)) @@ -135,12 +128,10 @@ func (r *Root) updateWithDynamicValue(nv dyn.Value) error { // the configuration equals nil (happens in tests). diags := r.diags depth := r.depth - path := r.Path defer func() { r.diags = diags r.depth = depth - r.Path = path }() // Convert normalized configuration tree to typed configuration. diff --git a/bundle/deploy/files/sync.go b/bundle/deploy/files/sync.go index 8de80c22fa..c74e41090d 100644 --- a/bundle/deploy/files/sync.go +++ b/bundle/deploy/files/sync.go @@ -28,7 +28,7 @@ func GetSyncOptions(ctx context.Context, b *bundle.Bundle) (*sync.SyncOptions, e } opts := &sync.SyncOptions{ - LocalPath: b.Config.Path, + LocalPath: b.Path, RemotePath: b.Config.Workspace.FilePath, Include: includes, Exclude: b.Config.Sync.Exclude, diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index 5a46cd67f8..d10ed1e4c7 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -39,7 +39,7 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { for name, job := range b.Config.Resources.Jobs { // Compute config file path the job is defined in, relative to the bundle // root - relativePath, err := filepath.Rel(b.Config.Path, job.ConfigFilePath) + relativePath, err := filepath.Rel(b.Path, job.ConfigFilePath) if err != nil { return diag.Errorf("failed to compute relative path for job %s: %v", name, err) } diff --git a/bundle/deploy/state_pull.go b/bundle/deploy/state_pull.go index 61f5426a09..6f8591d6bd 100644 --- a/bundle/deploy/state_pull.go +++ b/bundle/deploy/state_pull.go @@ -85,7 +85,7 @@ func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } log.Infof(ctx, "Creating new snapshot") - snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.Config.Path), opts) + snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.Path), opts) if err != nil { return diag.FromErr(err) } diff --git a/bundle/deploy/state_pull_test.go b/bundle/deploy/state_pull_test.go index 9716a1e04b..a0ac8eeb42 100644 --- a/bundle/deploy/state_pull_test.go +++ b/bundle/deploy/state_pull_test.go @@ -59,8 +59,8 @@ func testStatePull(t *testing.T, opts statePullOpts) { }} b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "default", }, @@ -77,11 +77,11 @@ func testStatePull(t *testing.T, opts statePullOpts) { ctx := context.Background() for _, file := range opts.localFiles { - testutil.Touch(t, filepath.Join(b.Config.Path, "bar"), file) + testutil.Touch(t, filepath.Join(b.Path, "bar"), file) } for _, file := range opts.localNotebooks { - testutil.TouchNotebook(t, filepath.Join(b.Config.Path, "bar"), file) + testutil.TouchNotebook(t, filepath.Join(b.Path, "bar"), file) } if opts.withExistingSnapshot { @@ -251,8 +251,8 @@ func TestStatePullNoState(t *testing.T) { }} b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "default", }, @@ -439,8 +439,8 @@ func TestStatePullNewerDeploymentStateVersion(t *testing.T) { }} b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "default", }, diff --git a/bundle/deploy/state_push_test.go b/bundle/deploy/state_push_test.go index c6d9f88f5a..39d6366ddd 100644 --- a/bundle/deploy/state_push_test.go +++ b/bundle/deploy/state_push_test.go @@ -45,8 +45,8 @@ func TestStatePush(t *testing.T) { }} b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "default", }, diff --git a/bundle/deploy/state_update_test.go b/bundle/deploy/state_update_test.go index 73b7fe4b34..1127331735 100644 --- a/bundle/deploy/state_update_test.go +++ b/bundle/deploy/state_update_test.go @@ -22,8 +22,8 @@ func TestStateUpdate(t *testing.T) { s := &stateUpdate{} b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "default", }, @@ -39,8 +39,8 @@ func TestStateUpdate(t *testing.T) { }, } - testutil.Touch(t, b.Config.Path, "test1.py") - testutil.Touch(t, b.Config.Path, "test2.py") + testutil.Touch(t, b.Path, "test1.py") + testutil.Touch(t, b.Path, "test2.py") m := mocks.NewMockWorkspaceClient(t) m.WorkspaceClient.Config = &databrickscfg.Config{ @@ -82,8 +82,8 @@ func TestStateUpdateWithExistingState(t *testing.T) { s := &stateUpdate{} b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "default", }, @@ -99,8 +99,8 @@ func TestStateUpdateWithExistingState(t *testing.T) { }, } - testutil.Touch(t, b.Config.Path, "test1.py") - testutil.Touch(t, b.Config.Path, "test2.py") + testutil.Touch(t, b.Path, "test1.py") + testutil.Touch(t, b.Path, "test2.py") m := mocks.NewMockWorkspaceClient(t) m.WorkspaceClient.Config = &databrickscfg.Config{ diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index bbef7f0f79..e1a04c31d8 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -28,8 +28,8 @@ func TestInitEnvironmentVariables(t *testing.T) { } b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "whatever", Terraform: &config.Terraform{ @@ -55,8 +55,8 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirSet(t *testing.T) { } b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "whatever", }, @@ -83,8 +83,8 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirNotSet(t *testing.T) { } b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "whatever", }, @@ -109,8 +109,8 @@ func TestSetTempDirEnvVarsForWindowWithAllTmpDirEnvVarsSet(t *testing.T) { } b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "whatever", }, @@ -139,8 +139,8 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileAndTempSet(t *testing.T) { } b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "whatever", }, @@ -169,8 +169,8 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) { } b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "whatever", }, @@ -197,8 +197,8 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) { func TestSetProxyEnvVars(t *testing.T) { b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "whatever", }, diff --git a/bundle/deploy/terraform/load_test.go b/bundle/deploy/terraform/load_test.go index a912c52133..d18ac05e96 100644 --- a/bundle/deploy/terraform/load_test.go +++ b/bundle/deploy/terraform/load_test.go @@ -17,8 +17,8 @@ func TestLoadWithNoState(t *testing.T) { } b := &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ - Path: t.TempDir(), Bundle: config.Bundle{ Target: "whatever", Terraform: &config.Terraform{ diff --git a/bundle/deploy/terraform/state_pull_test.go b/bundle/deploy/terraform/state_pull_test.go index 805b5af0fc..f4e53762f7 100644 --- a/bundle/deploy/terraform/state_pull_test.go +++ b/bundle/deploy/terraform/state_pull_test.go @@ -32,11 +32,11 @@ func mockStateFilerForPull(t *testing.T, contents map[string]int, merr error) fi func statePullTestBundle(t *testing.T) *bundle.Bundle { return &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", }, - Path: t.TempDir(), }, } } diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go index 41d3849000..b1ddd23840 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/deploy/terraform/state_push_test.go @@ -29,11 +29,11 @@ func mockStateFilerForPush(t *testing.T, fn func(body io.Reader)) filer.Filer { func statePushTestBundle(t *testing.T) *bundle.Bundle { return &bundle.Bundle{ + Path: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", }, - Path: t.TempDir(), }, } } diff --git a/bundle/libraries/libraries.go b/bundle/libraries/libraries.go index e0cb3fa382..971f6e0f90 100644 --- a/bundle/libraries/libraries.go +++ b/bundle/libraries/libraries.go @@ -65,7 +65,7 @@ func findLibraryMatches(lib *compute.Library, b *bundle.Bundle) ([]string, error return nil, nil } - fullPath := filepath.Join(b.Config.Path, path) + fullPath := filepath.Join(b.Path, path) return filepath.Glob(fullPath) } diff --git a/bundle/libraries/libraries_test.go b/bundle/libraries/libraries_test.go index 0bec2c6d01..873b322a3e 100644 --- a/bundle/libraries/libraries_test.go +++ b/bundle/libraries/libraries_test.go @@ -15,8 +15,8 @@ import ( func TestMapFilesToTaskLibrariesNoGlob(t *testing.T) { b := &bundle.Bundle{ + Path: "testdata", Config: config.Root{ - Path: "testdata", Resources: config.Resources{ Jobs: map[string]*resources.Job{ "job1": { diff --git a/bundle/python/conditional_transform_test.go b/bundle/python/conditional_transform_test.go index b4d7f9edb6..39ddd46334 100644 --- a/bundle/python/conditional_transform_test.go +++ b/bundle/python/conditional_transform_test.go @@ -18,8 +18,8 @@ func TestNoTransformByDefault(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ + Path: tmpDir, Config: config.Root{ - Path: tmpDir, Bundle: config.Bundle{ Target: "development", }, @@ -63,8 +63,8 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ + Path: tmpDir, Config: config.Root{ - Path: tmpDir, Bundle: config.Bundle{ Target: "development", }, @@ -106,7 +106,7 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) { dir, err := b.InternalDir(context.Background()) require.NoError(t, err) - internalDirRel, err := filepath.Rel(b.Config.Path, dir) + internalDirRel, err := filepath.Rel(b.Path, dir) require.NoError(t, err) require.Equal(t, path.Join(filepath.ToSlash(internalDirRel), "notebook_job1_key1"), task.NotebookTask.NotebookPath) diff --git a/bundle/python/transform_test.go b/bundle/python/transform_test.go index 729efe1a97..d49bea59a4 100644 --- a/bundle/python/transform_test.go +++ b/bundle/python/transform_test.go @@ -116,8 +116,8 @@ func TestTransformFiltersWheelTasksOnly(t *testing.T) { func TestNoPanicWithNoPythonWheelTasks(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ + Path: tmpDir, Config: config.Root{ - Path: tmpDir, Bundle: config.Bundle{ Target: "development", }, diff --git a/bundle/root_test.go b/bundle/root_test.go index e6c53e8249..d9365f3599 100644 --- a/bundle/root_test.go +++ b/bundle/root_test.go @@ -106,7 +106,7 @@ func TestLoadYamlWhenIncludesEnvPresent(t *testing.T) { cwd, err := os.Getwd() assert.NoError(t, err) - assert.Equal(t, cwd, bundle.Config.Path) + assert.Equal(t, cwd, bundle.Path) } func TestLoadDefautlBundleWhenNoYamlAndRootAndIncludesEnvPresent(t *testing.T) { @@ -118,7 +118,7 @@ func TestLoadDefautlBundleWhenNoYamlAndRootAndIncludesEnvPresent(t *testing.T) { bundle, err := MustLoad(ctx) assert.NoError(t, err) - assert.Equal(t, dir, bundle.Config.Path) + assert.Equal(t, dir, bundle.Path) } func TestErrorIfNoYamlNoRootEnvAndIncludesEnvPresent(t *testing.T) { diff --git a/bundle/scripts/scripts.go b/bundle/scripts/scripts.go index f8ed7d6a38..e59e6a6415 100644 --- a/bundle/scripts/scripts.go +++ b/bundle/scripts/scripts.go @@ -30,7 +30,7 @@ func (m *script) Name() string { } func (m *script) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - executor, err := exec.NewCommandExecutor(b.Config.Path) + executor, err := exec.NewCommandExecutor(b.Path) if err != nil { return diag.FromErr(err) } diff --git a/bundle/scripts/scripts_test.go b/bundle/scripts/scripts_test.go index fa5c239701..2034202e0d 100644 --- a/bundle/scripts/scripts_test.go +++ b/bundle/scripts/scripts_test.go @@ -23,7 +23,7 @@ func TestExecutesHook(t *testing.T) { }, } - executor, err := exec.NewCommandExecutor(b.Config.Path) + executor, err := exec.NewCommandExecutor(b.Path) require.NoError(t, err) _, out, err := executeHook(context.Background(), executor, b, config.ScriptPreBuild) require.NoError(t, err) diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index c44e80a578..655b3df6b9 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -79,9 +79,7 @@ func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { artifact := b.Config.Artifacts["my_test_code-0.0.1-py3-none-any.whl"] require.NotNil(t, artifact) require.Empty(t, artifact.BuildCommand) - require.Contains(t, artifact.Files[0].Source, filepath.Join( - b.Config.Path, - "package", + require.Contains(t, artifact.Files[0].Source, filepath.Join(b.Path, "package", "my_test_code-0.0.1-py3-none-any.whl", )) } diff --git a/cmd/bundle/generate/generate_test.go b/cmd/bundle/generate/generate_test.go index b71f1edfde..691110668a 100644 --- a/cmd/bundle/generate/generate_test.go +++ b/cmd/bundle/generate/generate_test.go @@ -10,7 +10,6 @@ import ( "testing" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -25,9 +24,7 @@ func TestGeneratePipelineCommand(t *testing.T) { root := t.TempDir() b := &bundle.Bundle{ - Config: config.Root{ - Path: root, - }, + Path: root, } m := mocks.NewMockWorkspaceClient(t) @@ -125,9 +122,7 @@ func TestGenerateJobCommand(t *testing.T) { root := t.TempDir() b := &bundle.Bundle{ - Config: config.Root{ - Path: root, - }, + Path: root, } m := mocks.NewMockWorkspaceClient(t) diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go index 827c4d5097..e8754ef560 100644 --- a/cmd/sync/sync_test.go +++ b/cmd/sync/sync_test.go @@ -16,9 +16,8 @@ import ( func TestSyncOptionsFromBundle(t *testing.T) { tempDir := t.TempDir() b := &bundle.Bundle{ + Path: tempDir, Config: config.Root{ - Path: tempDir, - Bundle: config.Bundle{ Target: "default", }, diff --git a/internal/bundle/artifacts_test.go b/internal/bundle/artifacts_test.go index 2ced12fdd1..8f6d117d00 100644 --- a/internal/bundle/artifacts_test.go +++ b/internal/bundle/artifacts_test.go @@ -36,8 +36,8 @@ func TestAccUploadArtifactFileToCorrectRemotePath(t *testing.T) { wsDir := internal.TemporaryWorkspaceDir(t, w) b := &bundle.Bundle{ + Path: dir, Config: config.Root{ - Path: dir, Bundle: config.Bundle{ Target: "whatever", }, From f48cf03579d6406d22f4c069f6ac27ffafefe8cb Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 26 Mar 2024 17:21:36 +0100 Subject: [PATCH 2/4] Path -> RootPath --- bundle/artifacts/build.go | 2 +- bundle/artifacts/upload_test.go | 4 +-- bundle/artifacts/whl/autodetect.go | 6 ++-- bundle/artifacts/whl/from_libraries.go | 2 +- bundle/bundle.go | 12 ++++---- bundle/bundle_test.go | 4 +-- .../expand_pipeline_glob_paths_test.go | 2 +- bundle/config/mutator/load_git_details.go | 4 +-- bundle/config/mutator/process_include_test.go | 4 +-- .../config/mutator/process_root_includes.go | 8 +++--- .../mutator/process_root_includes_test.go | 28 +++++++++---------- bundle/config/mutator/rewrite_sync_paths.go | 4 +-- .../config/mutator/rewrite_sync_paths_test.go | 8 +++--- bundle/config/mutator/trampoline.go | 2 +- bundle/config/mutator/trampoline_test.go | 2 +- bundle/config/mutator/translate_paths.go | 2 +- bundle/config/mutator/translate_paths_test.go | 24 ++++++++-------- bundle/deploy/files/sync.go | 2 +- bundle/deploy/metadata/compute.go | 2 +- bundle/deploy/state_pull.go | 2 +- bundle/deploy/state_pull_test.go | 10 +++---- bundle/deploy/state_push_test.go | 2 +- bundle/deploy/state_update_test.go | 12 ++++---- bundle/deploy/terraform/init_test.go | 14 +++++----- bundle/deploy/terraform/load_test.go | 2 +- bundle/deploy/terraform/state_pull_test.go | 2 +- bundle/deploy/terraform/state_push_test.go | 2 +- bundle/libraries/libraries.go | 2 +- bundle/libraries/libraries_test.go | 2 +- bundle/python/conditional_transform_test.go | 6 ++-- bundle/python/transform_test.go | 2 +- bundle/root_test.go | 4 +-- bundle/scripts/scripts.go | 2 +- bundle/scripts/scripts_test.go | 2 +- bundle/tests/python_wheel_test.go | 2 +- cmd/bundle/generate/generate_test.go | 4 +-- cmd/sync/sync_test.go | 2 +- internal/bundle/artifacts_test.go | 2 +- 38 files changed, 99 insertions(+), 99 deletions(-) diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 5f7e111528..349b1ff898 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -46,7 +46,7 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // If artifact path is not provided, use bundle root dir if artifact.Path == "" { - artifact.Path = b.Path + artifact.Path = b.RootPath } if !filepath.IsAbs(artifact.Path) { diff --git a/bundle/artifacts/upload_test.go b/bundle/artifacts/upload_test.go index ec7ee73a17..687d73b4a8 100644 --- a/bundle/artifacts/upload_test.go +++ b/bundle/artifacts/upload_test.go @@ -36,7 +36,7 @@ func TestExpandGlobFilesSource(t *testing.T) { t2.Close(t) b := &bundle.Bundle{ - Path: rootPath, + RootPath: rootPath, Config: config.Root{ Artifacts: map[string]*config.Artifact{ "test": { @@ -72,7 +72,7 @@ func TestExpandGlobFilesSourceWithNoMatches(t *testing.T) { require.NoError(t, err) b := &bundle.Bundle{ - Path: rootPath, + RootPath: rootPath, Config: config.Root{ Artifacts: map[string]*config.Artifact{ "test": { diff --git a/bundle/artifacts/whl/autodetect.go b/bundle/artifacts/whl/autodetect.go index ddb1797f43..ee77fff01b 100644 --- a/bundle/artifacts/whl/autodetect.go +++ b/bundle/artifacts/whl/autodetect.go @@ -35,21 +35,21 @@ func (m *detectPkg) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic log.Infof(ctx, "Detecting Python wheel project...") // checking if there is setup.py in the bundle root - setupPy := filepath.Join(b.Path, "setup.py") + setupPy := filepath.Join(b.RootPath, "setup.py") _, err := os.Stat(setupPy) if err != nil { log.Infof(ctx, "No Python wheel project found at bundle root folder") return nil } - log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.Path)) + log.Infof(ctx, fmt.Sprintf("Found Python wheel project at %s", b.RootPath)) module := extractModuleName(setupPy) if b.Config.Artifacts == nil { b.Config.Artifacts = make(map[string]*config.Artifact) } - pkgPath, err := filepath.Abs(b.Path) + pkgPath, err := filepath.Abs(b.RootPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/artifacts/whl/from_libraries.go b/bundle/artifacts/whl/from_libraries.go index f3660bbae9..84ef712acb 100644 --- a/bundle/artifacts/whl/from_libraries.go +++ b/bundle/artifacts/whl/from_libraries.go @@ -30,7 +30,7 @@ func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost tasks := libraries.FindAllWheelTasksWithLocalLibraries(b) for _, task := range tasks { for _, lib := range task.Libraries { - matches, err := filepath.Glob(filepath.Join(b.Path, lib.Whl)) + matches, err := filepath.Glob(filepath.Join(b.RootPath, lib.Whl)) // File referenced from libraries section does not exists, skipping if err != nil { continue diff --git a/bundle/bundle.go b/bundle/bundle.go index b1c514751e..d5673b493e 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -30,9 +30,9 @@ import ( const internalFolder = ".internal" type Bundle struct { - // Path contains the directory path to the root of the bundle. + // RootPath contains the directory path to the root of the bundle. // It is set when we instantiate a new bundle instance. - Path string + RootPath string Config config.Root @@ -78,7 +78,7 @@ func Load(ctx context.Context, path string) (*Bundle, error) { _, hasIncludesEnv := env.Includes(ctx) if hasRootEnv && hasIncludesEnv && stat.IsDir() { log.Debugf(ctx, "No bundle configuration; using bundle root: %s", path) - b.Path = path + b.RootPath = path b.Config = config.Root{ Bundle: config.Bundle{ Name: filepath.Base(path), @@ -162,7 +162,7 @@ func (b *Bundle) CacheDir(ctx context.Context, paths ...string) (string, error) if !exists || cacheDirName == "" { cacheDirName = filepath.Join( // Anchor at bundle root directory. - b.Path, + b.RootPath, // Static cache directory. ".databricks", "bundle", @@ -214,7 +214,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) { if err != nil { return nil, err } - internalDirRel, err := filepath.Rel(b.Path, internalDir) + internalDirRel, err := filepath.Rel(b.RootPath, internalDir) if err != nil { return nil, err } @@ -222,7 +222,7 @@ func (b *Bundle) GetSyncIncludePatterns(ctx context.Context) ([]string, error) { } func (b *Bundle) GitRepository() (*git.Repository, error) { - rootPath, err := folders.FindDirWithLeaf(b.Path, ".git") + rootPath, err := folders.FindDirWithLeaf(b.RootPath, ".git") if err != nil { return nil, fmt.Errorf("unable to locate repository root: %w", err) } diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 0ebc4c9be8..be716a40ac 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -77,7 +77,7 @@ func TestBundleMustLoadSuccess(t *testing.T) { t.Setenv(env.RootVariable, "./tests/basic") b, err := MustLoad(context.Background()) require.NoError(t, err) - assert.Equal(t, "tests/basic", filepath.ToSlash(b.Path)) + assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath)) } func TestBundleMustLoadFailureWithEnv(t *testing.T) { @@ -96,7 +96,7 @@ func TestBundleTryLoadSuccess(t *testing.T) { t.Setenv(env.RootVariable, "./tests/basic") b, err := TryLoad(context.Background()) require.NoError(t, err) - assert.Equal(t, "tests/basic", filepath.ToSlash(b.Path)) + assert.Equal(t, "tests/basic", filepath.ToSlash(b.RootPath)) } func TestBundleTryLoadFailureWithEnv(t *testing.T) { diff --git a/bundle/config/mutator/expand_pipeline_glob_paths_test.go b/bundle/config/mutator/expand_pipeline_glob_paths_test.go index effa6f244c..d1671c256f 100644 --- a/bundle/config/mutator/expand_pipeline_glob_paths_test.go +++ b/bundle/config/mutator/expand_pipeline_glob_paths_test.go @@ -41,7 +41,7 @@ func TestExpandGlobPathsInPipelines(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "skip/test7.py")) b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ diff --git a/bundle/config/mutator/load_git_details.go b/bundle/config/mutator/load_git_details.go index 4d354b8d70..7ce8476f1f 100644 --- a/bundle/config/mutator/load_git_details.go +++ b/bundle/config/mutator/load_git_details.go @@ -22,7 +22,7 @@ func (m *loadGitDetails) Name() string { func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Load relevant git repository - repo, err := git.NewRepository(b.Path) + repo, err := git.NewRepository(b.RootPath) if err != nil { return diag.FromErr(err) } @@ -56,7 +56,7 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn } // Compute relative path of the bundle root from the Git repo root. - absBundlePath, err := filepath.Abs(b.Path) + absBundlePath, err := filepath.Abs(b.RootPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/config/mutator/process_include_test.go b/bundle/config/mutator/process_include_test.go index 7103d5074b..b4fa3ccda5 100644 --- a/bundle/config/mutator/process_include_test.go +++ b/bundle/config/mutator/process_include_test.go @@ -16,7 +16,7 @@ import ( func TestProcessInclude(t *testing.T) { b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Workspace: config.Workspace{ Host: "foo", @@ -25,7 +25,7 @@ func TestProcessInclude(t *testing.T) { } relPath := "./file.yml" - fullPath := filepath.Join(b.Path, relPath) + fullPath := filepath.Join(b.RootPath, relPath) f, err := os.Create(fullPath) require.NoError(t, err) fmt.Fprint(f, "workspace:\n host: bar\n") diff --git a/bundle/config/mutator/process_root_includes.go b/bundle/config/mutator/process_root_includes.go index e4edabc532..4e4aeef43c 100644 --- a/bundle/config/mutator/process_root_includes.go +++ b/bundle/config/mutator/process_root_includes.go @@ -51,7 +51,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // Converts extra include paths from environment variable to relative paths for _, extraIncludePath := range getExtraIncludePaths(ctx) { if filepath.IsAbs(extraIncludePath) { - rel, err := filepath.Rel(b.Path, extraIncludePath) + rel, err := filepath.Rel(b.RootPath, extraIncludePath) if err != nil { return diag.Errorf("unable to include file '%s': %v", extraIncludePath, err) } @@ -70,7 +70,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. } // Anchor includes to the bundle root path. - matches, err := filepath.Glob(filepath.Join(b.Path, entry)) + matches, err := filepath.Glob(filepath.Join(b.RootPath, entry)) if err != nil { return diag.FromErr(err) } @@ -84,7 +84,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // Filter matches to ones we haven't seen yet. var includes []string for _, match := range matches { - rel, err := filepath.Rel(b.Path, match) + rel, err := filepath.Rel(b.RootPath, match) if err != nil { return diag.FromErr(err) } @@ -99,7 +99,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. slices.Sort(includes) files = append(files, includes...) for _, include := range includes { - out = append(out, ProcessInclude(filepath.Join(b.Path, include), include)) + out = append(out, ProcessInclude(filepath.Join(b.RootPath, include), include)) } } diff --git a/bundle/config/mutator/process_root_includes_test.go b/bundle/config/mutator/process_root_includes_test.go index c356628635..d3aaa974d6 100644 --- a/bundle/config/mutator/process_root_includes_test.go +++ b/bundle/config/mutator/process_root_includes_test.go @@ -19,7 +19,7 @@ import ( func TestProcessRootIncludesEmpty(t *testing.T) { b := &bundle.Bundle{ - Path: ".", + RootPath: ".", } diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -34,7 +34,7 @@ func TestProcessRootIncludesAbs(t *testing.T) { } b := &bundle.Bundle{ - Path: ".", + RootPath: ".", Config: config.Root{ Include: []string{ "/tmp/*.yml", @@ -48,7 +48,7 @@ func TestProcessRootIncludesAbs(t *testing.T) { func TestProcessRootIncludesSingleGlob(t *testing.T) { b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Include: []string{ "*.yml", @@ -56,9 +56,9 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { }, } - testutil.Touch(t, b.Path, "databricks.yml") - testutil.Touch(t, b.Path, "a.yml") - testutil.Touch(t, b.Path, "b.yml") + testutil.Touch(t, b.RootPath, "databricks.yml") + testutil.Touch(t, b.RootPath, "a.yml") + testutil.Touch(t, b.RootPath, "b.yml") diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -67,7 +67,7 @@ func TestProcessRootIncludesSingleGlob(t *testing.T) { func TestProcessRootIncludesMultiGlob(t *testing.T) { b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Include: []string{ "a*.yml", @@ -76,8 +76,8 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) { }, } - testutil.Touch(t, b.Path, "a1.yml") - testutil.Touch(t, b.Path, "b1.yml") + testutil.Touch(t, b.RootPath, "a1.yml") + testutil.Touch(t, b.RootPath, "b1.yml") diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -86,7 +86,7 @@ func TestProcessRootIncludesMultiGlob(t *testing.T) { func TestProcessRootIncludesRemoveDups(t *testing.T) { b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Include: []string{ "*.yml", @@ -95,7 +95,7 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) { }, } - testutil.Touch(t, b.Path, "a.yml") + testutil.Touch(t, b.RootPath, "a.yml") diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) require.NoError(t, diags.Error()) @@ -104,7 +104,7 @@ func TestProcessRootIncludesRemoveDups(t *testing.T) { func TestProcessRootIncludesNotExists(t *testing.T) { b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Include: []string{ "notexist.yml", @@ -123,7 +123,7 @@ func TestProcessRootIncludesExtrasFromEnvVar(t *testing.T) { t.Setenv(env.IncludesVariable, path.Join(rootPath, testYamlName)) b := &bundle.Bundle{ - Path: rootPath, + RootPath: rootPath, } diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) @@ -144,7 +144,7 @@ func TestProcessRootIncludesDedupExtrasFromEnvVar(t *testing.T) { )) b := &bundle.Bundle{ - Path: rootPath, + RootPath: rootPath, } diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) diff --git a/bundle/config/mutator/rewrite_sync_paths.go b/bundle/config/mutator/rewrite_sync_paths.go index ea56b7afe9..710190230e 100644 --- a/bundle/config/mutator/rewrite_sync_paths.go +++ b/bundle/config/mutator/rewrite_sync_paths.go @@ -45,11 +45,11 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc { func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) { - v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.Path))) + v, err = dyn.Map(v, "include", dyn.Foreach(m.makeRelativeTo(b.RootPath))) if err != nil { return dyn.NilValue, err } - v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.Path))) + v, err = dyn.Map(v, "exclude", dyn.Foreach(m.makeRelativeTo(b.RootPath))) if err != nil { return dyn.NilValue, err } diff --git a/bundle/config/mutator/rewrite_sync_paths_test.go b/bundle/config/mutator/rewrite_sync_paths_test.go index 55b9a826bf..56ada19e67 100644 --- a/bundle/config/mutator/rewrite_sync_paths_test.go +++ b/bundle/config/mutator/rewrite_sync_paths_test.go @@ -14,7 +14,7 @@ import ( func TestRewriteSyncPathsRelative(t *testing.T) { b := &bundle.Bundle{ - Path: ".", + RootPath: ".", Config: config.Root{ Sync: config.Sync{ Include: []string{ @@ -45,7 +45,7 @@ func TestRewriteSyncPathsRelative(t *testing.T) { func TestRewriteSyncPathsAbsolute(t *testing.T) { b := &bundle.Bundle{ - Path: "/tmp/dir", + RootPath: "/tmp/dir", Config: config.Root{ Sync: config.Sync{ Include: []string{ @@ -77,7 +77,7 @@ func TestRewriteSyncPathsAbsolute(t *testing.T) { func TestRewriteSyncPathsErrorPaths(t *testing.T) { t.Run("no sync block", func(t *testing.T) { b := &bundle.Bundle{ - Path: ".", + RootPath: ".", } diags := bundle.Apply(context.Background(), b, mutator.RewriteSyncPaths()) @@ -86,7 +86,7 @@ func TestRewriteSyncPathsErrorPaths(t *testing.T) { t.Run("empty include/exclude blocks", func(t *testing.T) { b := &bundle.Bundle{ - Path: ".", + RootPath: ".", Config: config.Root{ Sync: config.Sync{ Include: []string{}, diff --git a/bundle/config/mutator/trampoline.go b/bundle/config/mutator/trampoline.go index f7e20ff98b..dde9a299eb 100644 --- a/bundle/config/mutator/trampoline.go +++ b/bundle/config/mutator/trampoline.go @@ -82,7 +82,7 @@ func (m *trampoline) generateNotebookWrapper(ctx context.Context, b *bundle.Bund return err } - internalDirRel, err := filepath.Rel(b.Path, internalDir) + internalDirRel, err := filepath.Rel(b.RootPath, internalDir) if err != nil { return err } diff --git a/bundle/config/mutator/trampoline_test.go b/bundle/config/mutator/trampoline_test.go index 088885a2be..e39076647f 100644 --- a/bundle/config/mutator/trampoline_test.go +++ b/bundle/config/mutator/trampoline_test.go @@ -57,7 +57,7 @@ func TestGenerateTrampoline(t *testing.T) { } b := &bundle.Bundle{ - Path: tmpDir, + RootPath: tmpDir, Config: config.Root{ Bundle: config.Bundle{ Target: "development", diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 768253e255..8fab3abb38 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -85,7 +85,7 @@ func (m *translatePaths) rewritePath( } // Remote path must be relative to the bundle root. - localRelPath, err := filepath.Rel(b.Path, localPath) + localRelPath, err := filepath.Rel(b.RootPath, localPath) if err != nil { return err } diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 701a0b9c8b..9650ae8ba9 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -36,7 +36,7 @@ func touchEmptyFile(t *testing.T, path string) { func TestTranslatePathsSkippedWithGitSource(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -106,7 +106,7 @@ func TestTranslatePaths(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "dist", "task.jar")) b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -273,7 +273,7 @@ func TestTranslatePathsInSubdirectories(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "job", "my_dbt_project", "dbt_project.yml")) b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -367,7 +367,7 @@ func TestTranslatePathsOutsideBundleRoot(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -400,7 +400,7 @@ func TestJobNotebookDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -430,7 +430,7 @@ func TestJobFileDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ @@ -460,7 +460,7 @@ func TestPipelineNotebookDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ @@ -490,7 +490,7 @@ func TestPipelineFileDoesNotExistError(t *testing.T) { dir := t.TempDir() b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Resources: config.Resources{ Pipelines: map[string]*resources.Pipeline{ @@ -521,7 +521,7 @@ func TestJobSparkPythonTaskWithNotebookSourceError(t *testing.T) { touchNotebookFile(t, filepath.Join(dir, "my_notebook.py")) b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -555,7 +555,7 @@ func TestJobNotebookTaskWithFileSourceError(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "my_file.py")) b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -589,7 +589,7 @@ func TestPipelineNotebookLibraryWithFileSourceError(t *testing.T) { touchEmptyFile(t, filepath.Join(dir, "my_file.py")) b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", @@ -623,7 +623,7 @@ func TestPipelineFileLibraryWithNotebookSourceError(t *testing.T) { touchNotebookFile(t, filepath.Join(dir, "my_notebook.py")) b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Workspace: config.Workspace{ FilePath: "/bundle", diff --git a/bundle/deploy/files/sync.go b/bundle/deploy/files/sync.go index c74e41090d..e8c54c6332 100644 --- a/bundle/deploy/files/sync.go +++ b/bundle/deploy/files/sync.go @@ -28,7 +28,7 @@ func GetSyncOptions(ctx context.Context, b *bundle.Bundle) (*sync.SyncOptions, e } opts := &sync.SyncOptions{ - LocalPath: b.Path, + LocalPath: b.RootPath, RemotePath: b.Config.Workspace.FilePath, Include: includes, Exclude: b.Config.Sync.Exclude, diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index d10ed1e4c7..0347654848 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -39,7 +39,7 @@ func (m *compute) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { for name, job := range b.Config.Resources.Jobs { // Compute config file path the job is defined in, relative to the bundle // root - relativePath, err := filepath.Rel(b.Path, job.ConfigFilePath) + relativePath, err := filepath.Rel(b.RootPath, job.ConfigFilePath) if err != nil { return diag.Errorf("failed to compute relative path for job %s: %v", name, err) } diff --git a/bundle/deploy/state_pull.go b/bundle/deploy/state_pull.go index 6f8591d6bd..bae457ea09 100644 --- a/bundle/deploy/state_pull.go +++ b/bundle/deploy/state_pull.go @@ -85,7 +85,7 @@ func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic } log.Infof(ctx, "Creating new snapshot") - snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.Path), opts) + snapshot, err := sync.NewSnapshot(state.Files.ToSlice(b.RootPath), opts) if err != nil { return diag.FromErr(err) } diff --git a/bundle/deploy/state_pull_test.go b/bundle/deploy/state_pull_test.go index a0ac8eeb42..80acb254f4 100644 --- a/bundle/deploy/state_pull_test.go +++ b/bundle/deploy/state_pull_test.go @@ -59,7 +59,7 @@ func testStatePull(t *testing.T, opts statePullOpts) { }} b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", @@ -77,11 +77,11 @@ func testStatePull(t *testing.T, opts statePullOpts) { ctx := context.Background() for _, file := range opts.localFiles { - testutil.Touch(t, filepath.Join(b.Path, "bar"), file) + testutil.Touch(t, filepath.Join(b.RootPath, "bar"), file) } for _, file := range opts.localNotebooks { - testutil.TouchNotebook(t, filepath.Join(b.Path, "bar"), file) + testutil.TouchNotebook(t, filepath.Join(b.RootPath, "bar"), file) } if opts.withExistingSnapshot { @@ -251,7 +251,7 @@ func TestStatePullNoState(t *testing.T) { }} b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", @@ -439,7 +439,7 @@ func TestStatePullNewerDeploymentStateVersion(t *testing.T) { }} b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/state_push_test.go b/bundle/deploy/state_push_test.go index 39d6366ddd..39e4d13a56 100644 --- a/bundle/deploy/state_push_test.go +++ b/bundle/deploy/state_push_test.go @@ -45,7 +45,7 @@ func TestStatePush(t *testing.T) { }} b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/state_update_test.go b/bundle/deploy/state_update_test.go index 1127331735..dd8a1336ec 100644 --- a/bundle/deploy/state_update_test.go +++ b/bundle/deploy/state_update_test.go @@ -22,7 +22,7 @@ func TestStateUpdate(t *testing.T) { s := &stateUpdate{} b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", @@ -39,8 +39,8 @@ func TestStateUpdate(t *testing.T) { }, } - testutil.Touch(t, b.Path, "test1.py") - testutil.Touch(t, b.Path, "test2.py") + testutil.Touch(t, b.RootPath, "test1.py") + testutil.Touch(t, b.RootPath, "test2.py") m := mocks.NewMockWorkspaceClient(t) m.WorkspaceClient.Config = &databrickscfg.Config{ @@ -82,7 +82,7 @@ func TestStateUpdateWithExistingState(t *testing.T) { s := &stateUpdate{} b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", @@ -99,8 +99,8 @@ func TestStateUpdateWithExistingState(t *testing.T) { }, } - testutil.Touch(t, b.Path, "test1.py") - testutil.Touch(t, b.Path, "test2.py") + testutil.Touch(t, b.RootPath, "test1.py") + testutil.Touch(t, b.RootPath, "test2.py") m := mocks.NewMockWorkspaceClient(t) m.WorkspaceClient.Config = &databrickscfg.Config{ diff --git a/bundle/deploy/terraform/init_test.go b/bundle/deploy/terraform/init_test.go index e1a04c31d8..29bd80a3e0 100644 --- a/bundle/deploy/terraform/init_test.go +++ b/bundle/deploy/terraform/init_test.go @@ -28,7 +28,7 @@ func TestInitEnvironmentVariables(t *testing.T) { } b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -55,7 +55,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirSet(t *testing.T) { } b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -83,7 +83,7 @@ func TestSetTempDirEnvVarsForUnixWithTmpDirNotSet(t *testing.T) { } b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -109,7 +109,7 @@ func TestSetTempDirEnvVarsForWindowWithAllTmpDirEnvVarsSet(t *testing.T) { } b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -139,7 +139,7 @@ func TestSetTempDirEnvVarsForWindowWithUserProfileAndTempSet(t *testing.T) { } b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -169,7 +169,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) { } b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", @@ -197,7 +197,7 @@ func TestSetTempDirEnvVarsForWindowsWithoutAnyTempDirEnvVarsSet(t *testing.T) { func TestSetProxyEnvVars(t *testing.T) { b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", diff --git a/bundle/deploy/terraform/load_test.go b/bundle/deploy/terraform/load_test.go index d18ac05e96..c62217187d 100644 --- a/bundle/deploy/terraform/load_test.go +++ b/bundle/deploy/terraform/load_test.go @@ -17,7 +17,7 @@ func TestLoadWithNoState(t *testing.T) { } b := &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", diff --git a/bundle/deploy/terraform/state_pull_test.go b/bundle/deploy/terraform/state_pull_test.go index f4e53762f7..26297bfcbe 100644 --- a/bundle/deploy/terraform/state_pull_test.go +++ b/bundle/deploy/terraform/state_pull_test.go @@ -32,7 +32,7 @@ func mockStateFilerForPull(t *testing.T, contents map[string]int, merr error) fi func statePullTestBundle(t *testing.T) *bundle.Bundle { return &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/deploy/terraform/state_push_test.go b/bundle/deploy/terraform/state_push_test.go index b1ddd23840..e054773f31 100644 --- a/bundle/deploy/terraform/state_push_test.go +++ b/bundle/deploy/terraform/state_push_test.go @@ -29,7 +29,7 @@ func mockStateFilerForPush(t *testing.T, fn func(body io.Reader)) filer.Filer { func statePushTestBundle(t *testing.T) *bundle.Bundle { return &bundle.Bundle{ - Path: t.TempDir(), + RootPath: t.TempDir(), Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/bundle/libraries/libraries.go b/bundle/libraries/libraries.go index 971f6e0f90..8dd63a75a6 100644 --- a/bundle/libraries/libraries.go +++ b/bundle/libraries/libraries.go @@ -65,7 +65,7 @@ func findLibraryMatches(lib *compute.Library, b *bundle.Bundle) ([]string, error return nil, nil } - fullPath := filepath.Join(b.Path, path) + fullPath := filepath.Join(b.RootPath, path) return filepath.Glob(fullPath) } diff --git a/bundle/libraries/libraries_test.go b/bundle/libraries/libraries_test.go index 873b322a3e..3da10d47bb 100644 --- a/bundle/libraries/libraries_test.go +++ b/bundle/libraries/libraries_test.go @@ -15,7 +15,7 @@ import ( func TestMapFilesToTaskLibrariesNoGlob(t *testing.T) { b := &bundle.Bundle{ - Path: "testdata", + RootPath: "testdata", Config: config.Root{ Resources: config.Resources{ Jobs: map[string]*resources.Job{ diff --git a/bundle/python/conditional_transform_test.go b/bundle/python/conditional_transform_test.go index 39ddd46334..677970d70a 100644 --- a/bundle/python/conditional_transform_test.go +++ b/bundle/python/conditional_transform_test.go @@ -18,7 +18,7 @@ func TestNoTransformByDefault(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - Path: tmpDir, + RootPath: tmpDir, Config: config.Root{ Bundle: config.Bundle{ Target: "development", @@ -63,7 +63,7 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - Path: tmpDir, + RootPath: tmpDir, Config: config.Root{ Bundle: config.Bundle{ Target: "development", @@ -106,7 +106,7 @@ func TestTransformWithExperimentalSettingSetToTrue(t *testing.T) { dir, err := b.InternalDir(context.Background()) require.NoError(t, err) - internalDirRel, err := filepath.Rel(b.Path, dir) + internalDirRel, err := filepath.Rel(b.RootPath, dir) require.NoError(t, err) require.Equal(t, path.Join(filepath.ToSlash(internalDirRel), "notebook_job1_key1"), task.NotebookTask.NotebookPath) diff --git a/bundle/python/transform_test.go b/bundle/python/transform_test.go index d49bea59a4..c15feb4241 100644 --- a/bundle/python/transform_test.go +++ b/bundle/python/transform_test.go @@ -116,7 +116,7 @@ func TestTransformFiltersWheelTasksOnly(t *testing.T) { func TestNoPanicWithNoPythonWheelTasks(t *testing.T) { tmpDir := t.TempDir() b := &bundle.Bundle{ - Path: tmpDir, + RootPath: tmpDir, Config: config.Root{ Bundle: config.Bundle{ Target: "development", diff --git a/bundle/root_test.go b/bundle/root_test.go index d9365f3599..a83f36ace7 100644 --- a/bundle/root_test.go +++ b/bundle/root_test.go @@ -106,7 +106,7 @@ func TestLoadYamlWhenIncludesEnvPresent(t *testing.T) { cwd, err := os.Getwd() assert.NoError(t, err) - assert.Equal(t, cwd, bundle.Path) + assert.Equal(t, cwd, bundle.RootPath) } func TestLoadDefautlBundleWhenNoYamlAndRootAndIncludesEnvPresent(t *testing.T) { @@ -118,7 +118,7 @@ func TestLoadDefautlBundleWhenNoYamlAndRootAndIncludesEnvPresent(t *testing.T) { bundle, err := MustLoad(ctx) assert.NoError(t, err) - assert.Equal(t, dir, bundle.Path) + assert.Equal(t, dir, bundle.RootPath) } func TestErrorIfNoYamlNoRootEnvAndIncludesEnvPresent(t *testing.T) { diff --git a/bundle/scripts/scripts.go b/bundle/scripts/scripts.go index e59e6a6415..38d204f99e 100644 --- a/bundle/scripts/scripts.go +++ b/bundle/scripts/scripts.go @@ -30,7 +30,7 @@ func (m *script) Name() string { } func (m *script) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - executor, err := exec.NewCommandExecutor(b.Path) + executor, err := exec.NewCommandExecutor(b.RootPath) if err != nil { return diag.FromErr(err) } diff --git a/bundle/scripts/scripts_test.go b/bundle/scripts/scripts_test.go index 2034202e0d..1bc216b610 100644 --- a/bundle/scripts/scripts_test.go +++ b/bundle/scripts/scripts_test.go @@ -23,7 +23,7 @@ func TestExecutesHook(t *testing.T) { }, } - executor, err := exec.NewCommandExecutor(b.Path) + executor, err := exec.NewCommandExecutor(b.RootPath) require.NoError(t, err) _, out, err := executeHook(context.Background(), executor, b, config.ScriptPreBuild) require.NoError(t, err) diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index 655b3df6b9..412b507fe9 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -79,7 +79,7 @@ func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { artifact := b.Config.Artifacts["my_test_code-0.0.1-py3-none-any.whl"] require.NotNil(t, artifact) require.Empty(t, artifact.BuildCommand) - require.Contains(t, artifact.Files[0].Source, filepath.Join(b.Path, "package", + require.Contains(t, artifact.Files[0].Source, filepath.Join(b.RootPath, "package", "my_test_code-0.0.1-py3-none-any.whl", )) } diff --git a/cmd/bundle/generate/generate_test.go b/cmd/bundle/generate/generate_test.go index 691110668a..69ef639ae0 100644 --- a/cmd/bundle/generate/generate_test.go +++ b/cmd/bundle/generate/generate_test.go @@ -24,7 +24,7 @@ func TestGeneratePipelineCommand(t *testing.T) { root := t.TempDir() b := &bundle.Bundle{ - Path: root, + RootPath: root, } m := mocks.NewMockWorkspaceClient(t) @@ -122,7 +122,7 @@ func TestGenerateJobCommand(t *testing.T) { root := t.TempDir() b := &bundle.Bundle{ - Path: root, + RootPath: root, } m := mocks.NewMockWorkspaceClient(t) diff --git a/cmd/sync/sync_test.go b/cmd/sync/sync_test.go index e8754ef560..026d840f73 100644 --- a/cmd/sync/sync_test.go +++ b/cmd/sync/sync_test.go @@ -16,7 +16,7 @@ import ( func TestSyncOptionsFromBundle(t *testing.T) { tempDir := t.TempDir() b := &bundle.Bundle{ - Path: tempDir, + RootPath: tempDir, Config: config.Root{ Bundle: config.Bundle{ Target: "default", diff --git a/internal/bundle/artifacts_test.go b/internal/bundle/artifacts_test.go index 8f6d117d00..866a1f6e9e 100644 --- a/internal/bundle/artifacts_test.go +++ b/internal/bundle/artifacts_test.go @@ -36,7 +36,7 @@ func TestAccUploadArtifactFileToCorrectRemotePath(t *testing.T) { wsDir := internal.TemporaryWorkspaceDir(t, w) b := &bundle.Bundle{ - Path: dir, + RootPath: dir, Config: config.Root{ Bundle: config.Bundle{ Target: "whatever", From bd14db077d338364e675fe44c76c8b99afb0f338 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 26 Mar 2024 17:27:38 +0100 Subject: [PATCH 3/4] Fix --- bundle/bundle.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/bundle.go b/bundle/bundle.go index d5673b493e..0aa44df0b9 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -67,7 +67,9 @@ type Bundle struct { } func Load(ctx context.Context, path string) (*Bundle, error) { - b := &Bundle{} + b := &Bundle{ + RootPath: filepath.Clean(path), + } stat, err := os.Stat(path) if err != nil { return nil, err @@ -78,7 +80,6 @@ func Load(ctx context.Context, path string) (*Bundle, error) { _, hasIncludesEnv := env.Includes(ctx) if hasRootEnv && hasIncludesEnv && stat.IsDir() { log.Debugf(ctx, "No bundle configuration; using bundle root: %s", path) - b.RootPath = path b.Config = config.Root{ Bundle: config.Bundle{ Name: filepath.Base(path), From 99f027ef9cdf43d74107473e0f4a2c02faaef5e7 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 26 Mar 2024 19:39:38 +0100 Subject: [PATCH 4/4] Remove support for DATABRICKS_BUNDLE_INCLUDES PR #604 added functionality to load a bundle without a `databricks.yml` if both the `DATABRICKS_BUNDLE_ROOT` and `DATABRICKS_BUNDLE_INCLUDES` environment variables were set. We never ended up using this in downstream tools so this can be removed. --- bundle/bundle.go | 15 ------ .../config/mutator/process_root_includes.go | 23 --------- .../mutator/process_root_includes_test.go | 40 ---------------- bundle/env/includes.go | 14 ------ bundle/env/includes_test.go | 28 ----------- bundle/root_test.go | 47 ------------------- 6 files changed, 167 deletions(-) delete mode 100644 bundle/env/includes.go delete mode 100644 bundle/env/includes_test.go diff --git a/bundle/bundle.go b/bundle/bundle.go index 0aa44df0b9..2e193bbf39 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -70,23 +70,8 @@ func Load(ctx context.Context, path string) (*Bundle, error) { b := &Bundle{ RootPath: filepath.Clean(path), } - stat, err := os.Stat(path) - if err != nil { - return nil, err - } configFile, err := config.FileNames.FindInPath(path) if err != nil { - _, hasRootEnv := env.Root(ctx) - _, hasIncludesEnv := env.Includes(ctx) - if hasRootEnv && hasIncludesEnv && stat.IsDir() { - log.Debugf(ctx, "No bundle configuration; using bundle root: %s", path) - b.Config = config.Root{ - Bundle: config.Bundle{ - Name: filepath.Base(path), - }, - } - return b, nil - } return nil, err } log.Debugf(ctx, "Loading bundle configuration from: %s", configFile) diff --git a/bundle/config/mutator/process_root_includes.go b/bundle/config/mutator/process_root_includes.go index 4e4aeef43c..c5e0a22c5e 100644 --- a/bundle/config/mutator/process_root_includes.go +++ b/bundle/config/mutator/process_root_includes.go @@ -2,26 +2,15 @@ package mutator import ( "context" - "os" "path/filepath" "slices" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/diag" ) -// Get extra include paths from environment variable -func getExtraIncludePaths(ctx context.Context) []string { - value, exists := env.Includes(ctx) - if !exists { - return nil - } - return strings.Split(value, string(os.PathListSeparator)) -} - type processRootIncludes struct{} // ProcessRootIncludes expands the patterns in the configuration's include list @@ -48,18 +37,6 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // This is stored in the bundle configuration for observability. var files []string - // Converts extra include paths from environment variable to relative paths - for _, extraIncludePath := range getExtraIncludePaths(ctx) { - if filepath.IsAbs(extraIncludePath) { - rel, err := filepath.Rel(b.RootPath, extraIncludePath) - if err != nil { - return diag.Errorf("unable to include file '%s': %v", extraIncludePath, err) - } - extraIncludePath = rel - } - b.Config.Include = append(b.Config.Include, extraIncludePath) - } - // For each glob, find all files to load. // Ordering of the list of globs is maintained in the output. // For matches that appear in multiple globs, only the first is kept. diff --git a/bundle/config/mutator/process_root_includes_test.go b/bundle/config/mutator/process_root_includes_test.go index d3aaa974d6..675dd9acfb 100644 --- a/bundle/config/mutator/process_root_includes_test.go +++ b/bundle/config/mutator/process_root_includes_test.go @@ -2,16 +2,12 @@ package mutator_test import ( "context" - "os" - "path" "runtime" - "strings" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/internal/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -115,39 +111,3 @@ func TestProcessRootIncludesNotExists(t *testing.T) { require.True(t, diags.HasError()) assert.ErrorContains(t, diags.Error(), "notexist.yml defined in 'include' section does not match any files") } - -func TestProcessRootIncludesExtrasFromEnvVar(t *testing.T) { - rootPath := t.TempDir() - testYamlName := "extra_include_path.yml" - testutil.Touch(t, rootPath, testYamlName) - t.Setenv(env.IncludesVariable, path.Join(rootPath, testYamlName)) - - b := &bundle.Bundle{ - RootPath: rootPath, - } - - diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) - require.NoError(t, diags.Error()) - assert.Contains(t, b.Config.Include, testYamlName) -} - -func TestProcessRootIncludesDedupExtrasFromEnvVar(t *testing.T) { - rootPath := t.TempDir() - testYamlName := "extra_include_path.yml" - testutil.Touch(t, rootPath, testYamlName) - t.Setenv(env.IncludesVariable, strings.Join( - []string{ - path.Join(rootPath, testYamlName), - path.Join(rootPath, testYamlName), - }, - string(os.PathListSeparator), - )) - - b := &bundle.Bundle{ - RootPath: rootPath, - } - - diags := bundle.Apply(context.Background(), b, mutator.ProcessRootIncludes()) - require.NoError(t, diags.Error()) - assert.Equal(t, []string{testYamlName}, b.Config.Include) -} diff --git a/bundle/env/includes.go b/bundle/env/includes.go deleted file mode 100644 index 4ade018776..0000000000 --- a/bundle/env/includes.go +++ /dev/null @@ -1,14 +0,0 @@ -package env - -import "context" - -// IncludesVariable names the environment variable that holds additional configuration paths to include -// during bundle configuration loading. Also see `bundle/config/mutator/process_root_includes.go`. -const IncludesVariable = "DATABRICKS_BUNDLE_INCLUDES" - -// Includes returns the bundle Includes environment variable. -func Includes(ctx context.Context) (string, bool) { - return get(ctx, []string{ - IncludesVariable, - }) -} diff --git a/bundle/env/includes_test.go b/bundle/env/includes_test.go deleted file mode 100644 index d9366a59ff..0000000000 --- a/bundle/env/includes_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package env - -import ( - "context" - "testing" - - "github.com/databricks/cli/internal/testutil" - "github.com/stretchr/testify/assert" -) - -func TestIncludes(t *testing.T) { - ctx := context.Background() - - testutil.CleanupEnvironment(t) - - t.Run("set", func(t *testing.T) { - t.Setenv("DATABRICKS_BUNDLE_INCLUDES", "foo") - includes, ok := Includes(ctx) - assert.True(t, ok) - assert.Equal(t, "foo", includes) - }) - - t.Run("not set", func(t *testing.T) { - includes, ok := Includes(ctx) - assert.False(t, ok) - assert.Equal(t, "", includes) - }) -} diff --git a/bundle/root_test.go b/bundle/root_test.go index a83f36ace7..99bf58a00a 100644 --- a/bundle/root_test.go +++ b/bundle/root_test.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/internal/testutil" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -94,49 +93,3 @@ func TestRootLookupError(t *testing.T) { _, err := mustGetRoot(ctx) require.ErrorContains(t, err, "unable to locate bundle root") } - -func TestLoadYamlWhenIncludesEnvPresent(t *testing.T) { - ctx := context.Background() - testutil.Chdir(t, filepath.Join(".", "tests", "basic")) - t.Setenv(env.IncludesVariable, "test") - - bundle, err := MustLoad(ctx) - assert.NoError(t, err) - assert.Equal(t, "basic", bundle.Config.Bundle.Name) - - cwd, err := os.Getwd() - assert.NoError(t, err) - assert.Equal(t, cwd, bundle.RootPath) -} - -func TestLoadDefautlBundleWhenNoYamlAndRootAndIncludesEnvPresent(t *testing.T) { - ctx := context.Background() - dir := t.TempDir() - testutil.Chdir(t, dir) - t.Setenv(env.RootVariable, dir) - t.Setenv(env.IncludesVariable, "test") - - bundle, err := MustLoad(ctx) - assert.NoError(t, err) - assert.Equal(t, dir, bundle.RootPath) -} - -func TestErrorIfNoYamlNoRootEnvAndIncludesEnvPresent(t *testing.T) { - ctx := context.Background() - dir := t.TempDir() - testutil.Chdir(t, dir) - t.Setenv(env.IncludesVariable, "test") - - _, err := MustLoad(ctx) - assert.Error(t, err) -} - -func TestErrorIfNoYamlNoIncludesEnvAndRootEnvPresent(t *testing.T) { - ctx := context.Background() - dir := t.TempDir() - testutil.Chdir(t, dir) - t.Setenv(env.RootVariable, dir) - - _, err := MustLoad(ctx) - assert.Error(t, err) -}