Skip to content

Commit

Permalink
add int test
Browse files Browse the repository at this point in the history
  • Loading branch information
HariGS-DB committed May 26, 2024
1 parent 7b0c251 commit 8892060
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 7 deletions.
15 changes: 8 additions & 7 deletions src/databricks/labs/ucx/installer/workflows.py
Original file line number Diff line number Diff line change
Expand Up @@ -403,14 +403,14 @@ def __init__( # pylint: disable=too-many-arguments
super().__init__(config, installation, ws)

def create_jobs(self):
remote_wheel = self._upload_wheel()
remote_wheels = self._upload_wheel()
desired_workflows = {t.workflow for t in self._tasks if t.cloud_compatible(self._ws.config)}
wheel_runner = None

if self._config.override_clusters:
wheel_runner = self._upload_wheel_runner(remote_wheel)
wheel_runner = self._upload_wheel_runner(remote_wheels)
for workflow_name in desired_workflows:
settings = self._job_settings(workflow_name, remote_wheel)
settings = self._job_settings(workflow_name, remote_wheels)
if self._config.override_clusters:
settings = self._apply_cluster_overrides(
workflow_name,
Expand All @@ -430,7 +430,7 @@ def create_jobs(self):
continue

self._install_state.save()
self._create_debug(remote_wheel)
self._create_debug(remote_wheels)
return self._create_readme()

@property
Expand Down Expand Up @@ -534,12 +534,13 @@ def _upload_wheel(self):
with self._wheels:
if self._config.upload_dependencies:
wheel_paths = self._wheels.upload_wheel_dependencies(["databricks", "sqlglot"])
wheel_paths = [f"/Workspace{wheel_path}" for wheel_path in wheel_paths]
wheel_paths.append(f"/Workspace{self._wheels.upload_to_wsfs()}")
return wheel_paths

def _upload_wheel_runner(self, remote_wheels: list[str]):
# TODO: we have to be doing this workaround until ES-897453 is solved in the platform
remote_wheels_str = " \n".join(list(remote_wheels))
remote_wheels_str = " ".join(list(remote_wheels))
code = TEST_RUNNER_NOTEBOOK.format(remote_wheel=remote_wheels_str, config_file=self._config_file).encode("utf8")
return self._installation.upload(f"wheels/wheel-test-runner-{self._product_info.version()}.py", code)

Expand Down Expand Up @@ -727,14 +728,14 @@ def _job_parse_logs_task(
)
return self._job_wheel_task(jobs_task, workflow, remote_wheels)

def _create_debug(self, remote_wheel: str):
def _create_debug(self, remote_wheels: list[str]):
readme_link = self._installation.workspace_link('README')
job_links = ", ".join(
f"[{self._name(step_name)}]({self._ws.config.host}#job/{job_id})"
for step_name, job_id in self._install_state.jobs.items()
)
content = DEBUG_NOTEBOOK.format(
remote_wheel=remote_wheel, readme_link=readme_link, job_links=job_links, config_file=self._config_file
remote_wheel=remote_wheels, readme_link=readme_link, job_links=job_links, config_file=self._config_file
).encode("utf8")
self._installation.upload('DEBUG.py', content)

Expand Down
12 changes: 12 additions & 0 deletions tests/integration/test_installation.py
Original file line number Diff line number Diff line change
Expand Up @@ -471,3 +471,15 @@ def test_new_collection(ws, sql_backend, installation_ctx, env_or_skip):
config = installation_ctx.installation.load(WorkspaceConfig)
workspace_id = installation_ctx.workspace_installer.workspace_client.get_workspace_id()
assert config.installed_workspace_ids == [workspace_id]


def test_installation_with_dependency_upload(ws, installation_ctx, mocker):
config = dataclasses.replace(installation_ctx.config, upload_dependencies=True)
installation_ctx = installation_ctx.replace(config=config)
mocker.patch("webbrowser.open")
installation_ctx.workspace_installation.run()
with pytest.raises(ManyError):
installation_ctx.deployed_workflows.run_workflow("failing")

installation_ctx.deployed_workflows.repair_run("failing")
assert installation_ctx.deployed_workflows.validate_step("failing")

0 comments on commit 8892060

Please sign in to comment.