From 9493875919988bd738d1cb0fdba62d9608068ea5 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 7 Mar 2024 12:51:06 +0000 Subject: [PATCH 01/13] Try remove bootstrap_project() Signed-off-by: Merel Theisen --- package/kedro_viz/integrations/kedro/data_loader.py | 3 --- package/kedro_viz/server.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index dccf95d8af..a9501eabd0 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -86,9 +86,6 @@ def load_data( and the session store. """ from kedro.framework.project import pipelines - from kedro.framework.startup import bootstrap_project - - bootstrap_project(project_path) with KedroSession.create( project_path=project_path, diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 4c8c5ac04f..723783ab59 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -127,8 +127,6 @@ def run_server( if __name__ == "__main__": # pragma: no cover import argparse - from kedro.framework.startup import bootstrap_project - parser = argparse.ArgumentParser(description="Launch a development viz server") parser.add_argument("project_path", help="Path to a Kedro project") parser.add_argument( @@ -140,7 +138,6 @@ def run_server( args = parser.parse_args() project_path = (Path.cwd() / args.project_path).absolute() - bootstrap_project(project_path) run_process_kwargs = { "path": project_path, From 36c1c32b7b80a2a94e9163d35fcd70f2c7244e41 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 7 Mar 2024 13:22:07 +0000 Subject: [PATCH 02/13] Try set the package name for spawn process Signed-off-by: Merel Theisen --- package/kedro_viz/launchers/cli.py | 3 ++- package/kedro_viz/server.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index fa9c972010..4bb08119fc 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -27,7 +27,7 @@ viz_deploy_progress_timer, ) from kedro_viz.server import load_and_populate_data - +from kedro.framework.project import PACKAGE_NAME _VIZ_PROCESSES: Dict[str, int] = {} @@ -151,6 +151,7 @@ def run( "autoreload": autoreload, "ignore_plugins": ignore_plugins, "extra_params": params, + "package_name": PACKAGE_NAME, } if autoreload: project_path = Path.cwd() diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 723783ab59..8c8a08e09d 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -84,6 +84,7 @@ def run_server( autoreload: bool = False, ignore_plugins: bool = False, extra_params: Optional[Dict[str, Any]] = None, + package_name: Optional[str] = None, ): # pylint: disable=redefined-outer-name """Run a uvicorn server with a FastAPI app that either launches API response data from a file or from reading data from a real Kedro project. @@ -106,7 +107,10 @@ def run_server( take precedence over) the parameters retrieved from the project configuration. """ + from kedro.framework.project import configure_project + path = Path(project_path) if project_path else Path.cwd() + configure_project(package_name) if load_file is None: load_and_populate_data(path, env, ignore_plugins, extra_params, pipeline_name) From 8fc1b8799e5136fd1e5090ed91e2b301f6a7c78d Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Tue, 2 Apr 2024 22:05:34 -0500 Subject: [PATCH 03/13] move configure project before kedro integration Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/integrations/kedro/data_loader.py | 8 ++++++-- package/kedro_viz/server.py | 13 +++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index a9501eabd0..43390fcd29 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -70,10 +70,11 @@ def load_data( env: Optional[str] = None, ignore_plugins: bool = False, extra_params: Optional[Dict[str, Any]] = None, + package_name: Optional[str] = None, ) -> Tuple[DataCatalog, Dict[str, Pipeline], BaseSessionStore, Dict]: """Load data from a Kedro project. Args: - project_path: the path whether the Kedro project is located. + project_path: the path where the Kedro project is located. env: the Kedro environment to load the data. If not provided. it will use Kedro default, which is local. ignore_plugins: the flag to unregister all installed plugins in a kedro project. @@ -81,11 +82,14 @@ def load_data( for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project configuration. + package_name: The name of the current package Returns: A tuple containing the data catalog and the pipeline dictionary and the session store. """ - from kedro.framework.project import pipelines + from kedro.framework.project import pipelines, configure_project + + configure_project(package_name) with KedroSession.create( project_path=project_path, diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 0084f233d8..8fc0214fee 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -55,12 +55,13 @@ def load_and_populate_data( ignore_plugins: bool = False, extra_params: Optional[Dict[str, Any]] = None, pipeline_name: Optional[str] = None, + package_name: Optional[str] = None, ): """Loads underlying Kedro project data and populates Kedro Viz Repositories""" # Loads data from underlying Kedro Project catalog, pipelines, session_store, stats_dict = kedro_data_loader.load_data( - path, env, ignore_plugins, extra_params + path, env, ignore_plugins, extra_params, package_name ) pipelines = ( @@ -106,15 +107,15 @@ def run_server( for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project configuration. - package_name: The name of the current package + package_name: The name of the current package """ - from kedro.framework.project import configure_project - print(package_name) + path = Path(project_path) if project_path else Path.cwd() - configure_project(package_name) if load_file is None: - load_and_populate_data(path, env, ignore_plugins, extra_params, pipeline_name) + load_and_populate_data( + path, env, ignore_plugins, extra_params, pipeline_name, package_name + ) if save_file: save_api_responses_to_fs(save_file, fsspec.filesystem("file")) From 2fb36aeb94f94942f4bad5a6cb768928ba3a46bb Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Wed, 3 Apr 2024 23:06:08 -0500 Subject: [PATCH 04/13] include package name in jupyter and dev run Signed-off-by: ravi-kumar-pilla --- .../kedro_viz/integrations/kedro/data_loader.py | 10 ++++++++-- package/kedro_viz/launchers/cli.py | 17 +++++++++++++---- package/kedro_viz/launchers/jupyter.py | 3 +++ package/tests/test_launchers/test_cli.py | 11 +++++++---- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index 43390fcd29..fa55668d1e 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -87,9 +87,15 @@ def load_data( A tuple containing the data catalog and the pipeline dictionary and the session store. """ - from kedro.framework.project import pipelines, configure_project + from kedro.framework.project import configure_project, pipelines - configure_project(package_name) + if package_name: + configure_project(package_name) + else: + from kedro.framework.startup import bootstrap_project + + # bootstrap project when viz is run in dev mode + bootstrap_project(project_path) with KedroSession.create( project_path=project_path, diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index 00fdaeecc4..6eaa49403c 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -9,6 +9,7 @@ from click_default_group import DefaultGroup from kedro.framework.cli.project import PARAMS_ARG_HELP from kedro.framework.cli.utils import KedroCliError, _split_params +from kedro.framework.project import PACKAGE_NAME from packaging.version import parse from watchgod import RegExpWatcher, run_process @@ -28,7 +29,6 @@ viz_deploy_progress_timer, ) from kedro_viz.server import load_and_populate_data -from kedro.framework.project import PACKAGE_NAME try: from azure.core.exceptions import ServiceRequestError @@ -253,7 +253,14 @@ def create_shareableviz_process(platform, endpoint=None, bucket_name=None): viz_deploy_process = multiprocessing.Process( target=load_and_deploy_viz, - args=(platform, endpoint, bucket_name, process_completed, exception_queue), + args=( + platform, + endpoint, + bucket_name, + PACKAGE_NAME, + process_completed, + exception_queue, + ), ) viz_deploy_process.start() @@ -321,11 +328,13 @@ def create_shareableviz_process(platform, endpoint=None, bucket_name=None): def load_and_deploy_viz( - platform, endpoint, bucket_name, process_completed, exception_queue + platform, endpoint, bucket_name, package_name, process_completed, exception_queue ): """Loads Kedro Project data, creates a deployer and deploys to a platform""" try: - load_and_populate_data(Path.cwd(), ignore_plugins=True) + load_and_populate_data( + Path.cwd(), ignore_plugins=True, package_name=package_name + ) # Start the deployment deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) diff --git a/package/kedro_viz/launchers/jupyter.py b/package/kedro_viz/launchers/jupyter.py index 5c9da6dcd6..fc0507d890 100644 --- a/package/kedro_viz/launchers/jupyter.py +++ b/package/kedro_viz/launchers/jupyter.py @@ -1,6 +1,7 @@ """`kedro_viz.launchers.jupyter` provides line_magic to launch the viz server from a jupyter notebook. """ + # pragma: no cover import logging import multiprocessing @@ -12,6 +13,7 @@ import IPython from IPython.display import HTML, display +from kedro.framework.project import PACKAGE_NAME from watchgod import RegExpWatcher, run_process from kedro_viz.launchers.utils import _check_viz_up, _wait_for @@ -142,6 +144,7 @@ def run_viz( # pylint: disable=too-many-locals "ignore_plugins": ignore_plugins, "extra_params": params, "project_path": project_path, + "package_name": PACKAGE_NAME, } process_context = multiprocessing.get_context("spawn") if autoreload: diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index 727aa83ebc..09f2eabf02 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -550,26 +550,29 @@ def test_create_shareableviz_process( @pytest.mark.parametrize( - "platform, endpoint, bucket_name", + "platform, endpoint, bucket_name, package_name", [ - ("azure", "https://example-bucket.web.core.windows.net", "example-bucket"), + ("azure", "https://example-bucket.web.core.windows.net", "example-bucket", "demo_project"), ( "aws", "http://example-bucket.s3-website.us-east-2.amazonaws.com/", "example-bucket", + "demo_project" ), ( "gcp", "http://34.120.87.227/", "example-bucket", + "demo_project" ), - ("local", None, None), + ("local", None, None, "demo_project"), ], ) def test_load_and_deploy_viz_success( platform, endpoint, bucket_name, + package_name, mock_DeployerFactory, mock_load_and_populate_data, mock_process_completed, @@ -580,7 +583,7 @@ def test_load_and_deploy_viz_success( deployer_mock = mock_DeployerFactory.create_deployer.return_value cli.load_and_deploy_viz( - platform, endpoint, bucket_name, mock_process_completed, mock_exception_queue + platform, endpoint, bucket_name, package_name, mock_process_completed, mock_exception_queue ) mock_load_and_populate_data.assert_called_once_with( From e0446acecb1dc3322c494cb154a8018fd15b5c73 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 4 Apr 2024 13:35:08 -0500 Subject: [PATCH 05/13] fix tests and lint Signed-off-by: ravi-kumar-pilla --- .../integrations/kedro/data_loader.py | 4 ++-- package/kedro_viz/launchers/cli.py | 4 ++-- package/kedro_viz/launchers/jupyter.py | 2 +- package/kedro_viz/server.py | 21 +++++++++++++------ package/tests/test_launchers/test_cli.py | 10 ++++++++- package/tests/test_launchers/test_jupyter.py | 3 +++ 6 files changed, 32 insertions(+), 12 deletions(-) diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index 5751de1b50..88199cb867 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -69,8 +69,8 @@ def load_data( project_path: Path, env: Optional[str] = None, include_hooks: bool = False, - extra_params: Optional[Dict[str, Any]] = None, package_name: Optional[str] = None, + extra_params: Optional[Dict[str, Any]] = None, ) -> Tuple[DataCatalog, Dict[str, Pipeline], BaseSessionStore, Dict]: """Load data from a Kedro project. Args: @@ -78,11 +78,11 @@ def load_data( env: the Kedro environment to load the data. If not provided. it will use Kedro default, which is local. include_hooks: A flag to include all registered hooks in your Kedro Project. + package_name: The name of the current package extra_params: Optional dictionary containing extra project parameters for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project configuration. - package_name: The name of the current package Returns: A tuple containing the data catalog and the pipeline dictionary and the session store. diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index e19826a19e..87fa5b2676 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -154,8 +154,8 @@ def run( "env": env, "autoreload": autoreload, "include_hooks": include_hooks, - "extra_params": params, "package_name": PACKAGE_NAME, + "extra_params": params, } if autoreload: project_path = Path.cwd() @@ -269,8 +269,8 @@ def create_shareableviz_process( platform, endpoint, bucket_name, - PACKAGE_NAME, include_hooks, + PACKAGE_NAME, process_completed, exception_queue, ), diff --git a/package/kedro_viz/launchers/jupyter.py b/package/kedro_viz/launchers/jupyter.py index e1ac2273bc..f51f6ce7eb 100644 --- a/package/kedro_viz/launchers/jupyter.py +++ b/package/kedro_viz/launchers/jupyter.py @@ -142,9 +142,9 @@ def run_viz( # pylint: disable=too-many-locals "env": env, "autoreload": autoreload, "include_hooks": include_hooks, + "package_name": PACKAGE_NAME, "extra_params": params, "project_path": project_path, - "package_name": PACKAGE_NAME, } process_context = multiprocessing.get_context("spawn") if autoreload: diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 7f9ef26fc4..eb31f7a9c9 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -53,15 +53,19 @@ def load_and_populate_data( path: Path, env: Optional[str] = None, include_hooks: bool = False, - extra_params: Optional[Dict[str, Any]] = None, - pipeline_name: Optional[str] = None, package_name: Optional[str] = None, + pipeline_name: Optional[str] = None, + extra_params: Optional[Dict[str, Any]] = None, ): """Loads underlying Kedro project data and populates Kedro Viz Repositories""" # Loads data from underlying Kedro Project catalog, pipelines, session_store, stats_dict = kedro_data_loader.load_data( - path, env, include_hooks, extra_params, package_name + path, + env, + include_hooks, + package_name, + extra_params, ) pipelines = ( @@ -84,8 +88,8 @@ def run_server( project_path: Optional[str] = None, autoreload: bool = False, include_hooks: bool = False, - extra_params: Optional[Dict[str, Any]] = None, package_name: Optional[str] = None, + extra_params: Optional[Dict[str, Any]] = None, ): # pylint: disable=redefined-outer-name """Run a uvicorn server with a FastAPI app that either launches API response data from a file or from reading data from a real Kedro project. @@ -103,18 +107,23 @@ def run_server( project_path: the optional path of the Kedro project that contains the pipelines to visualise. If not supplied, the current working directory will be used. include_hooks: A flag to include all registered hooks in your Kedro Project. + package_name: The name of the current package extra_params: Optional dictionary containing extra project parameters for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project configuration. - package_name: The name of the current package """ path = Path(project_path) if project_path else Path.cwd() if load_file is None: load_and_populate_data( - path, env, include_hooks, extra_params, pipeline_name, package_name + path, + env, + include_hooks, + package_name, + pipeline_name, + extra_params, ) if save_file: diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index 68b477c64f..df53b70ca5 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -87,6 +87,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -101,6 +102,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -120,6 +122,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {}, }, ), @@ -150,6 +153,7 @@ def mock_project_path(mocker): "env": "local", "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": {"extra_param": "param"}, }, ), @@ -164,6 +168,7 @@ def mock_project_path(mocker): "env": None, "autoreload": False, "include_hooks": True, + "package_name": None, "extra_params": {}, }, ), @@ -284,6 +289,7 @@ def test_kedro_viz_command_with_autoreload( "autoreload": True, "project_path": mock_project_path, "include_hooks": False, + "package_name": None, "extra_params": {}, }, "watcher_cls": RegExpWatcher, @@ -579,6 +585,7 @@ def test_create_shareableviz_process( endpoint, bucket_name, include_hooks, + None, mock_process_completed.return_value, mock_exception_queue.return_value, ), @@ -627,6 +634,7 @@ def test_create_shareableviz_process( "aws", "http://example-bucket.s3-website.us-east-2.amazonaws.com/", "example-bucket", + True, "demo_project", ), ("gcp", "http://34.120.87.227/", "example-bucket", False, "demo_project"), @@ -659,7 +667,7 @@ def test_load_and_deploy_viz_success( ) mock_load_and_populate_data.assert_called_once_with( - mock_project_path, include_hooks=include_hooks + mock_project_path, include_hooks=include_hooks, package_name=package_name ) mock_DeployerFactory.create_deployer.assert_called_once_with( platform, endpoint, bucket_name diff --git a/package/tests/test_launchers/test_jupyter.py b/package/tests/test_launchers/test_jupyter.py index af41f9b884..dd489778ca 100644 --- a/package/tests/test_launchers/test_jupyter.py +++ b/package/tests/test_launchers/test_jupyter.py @@ -35,6 +35,7 @@ def test_run_viz(self, mocker, patched_check_viz_up): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": "", }, ) @@ -59,6 +60,7 @@ def test_run_viz(self, mocker, patched_check_viz_up): "env": None, "autoreload": False, "include_hooks": True, + "package_name": None, "extra_params": "", }, ) @@ -105,6 +107,7 @@ def test_run_viz_on_databricks(self, mocker, patched_check_viz_up, monkeypatch): "env": None, "autoreload": False, "include_hooks": False, + "package_name": None, "extra_params": "", }, ) From fb41851a27e6c1e54e9a8ddaf0c3fbd6eb51e683 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 4 Apr 2024 14:02:19 -0500 Subject: [PATCH 06/13] update release note Signed-off-by: ravi-kumar-pilla --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 97ba55003b..4c9bf94b70 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -18,6 +18,7 @@ Please follow the established format: - Increase Kedro-Viz timeout. (#1803) - Remove demo data source and update feature hints. (#1804) - Add markdown support for backticks in the pop-up reminder. (#1826) +- Refactor backend integration with Kedro by replacing bootstrap_project with configure_project. (#1796) # Release 8.0.1 From eba1c30811f0ed0a5db5333cdb662ad1a74e9fb2 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 4 Apr 2024 14:04:12 -0500 Subject: [PATCH 07/13] revert permission change Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/data_access/managers.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 package/kedro_viz/data_access/managers.py diff --git a/package/kedro_viz/data_access/managers.py b/package/kedro_viz/data_access/managers.py old mode 100755 new mode 100644 From 27fc5b685fb0d56af25dc306ecb1d02de67eec18 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Mon, 8 Apr 2024 22:03:08 -0500 Subject: [PATCH 08/13] fix PR comments Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/integrations/kedro/data_loader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index 88199cb867..6beb174310 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -11,6 +11,7 @@ from typing import Any, Dict, Optional, Tuple from kedro import __version__ +from kedro.framework.project import configure_project, pipelines from kedro.framework.session import KedroSession from kedro.framework.session.store import BaseSessionStore from kedro.io import DataCatalog @@ -87,8 +88,6 @@ def load_data( A tuple containing the data catalog and the pipeline dictionary and the session store. """ - from kedro.framework.project import configure_project, pipelines - if package_name: configure_project(package_name) else: From 25db25e4c4f617d33ab1da66d84742627c8c64a9 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 18 Apr 2024 14:09:15 -0500 Subject: [PATCH 09/13] working draft with parent project search Signed-off-by: ravi-kumar-pilla --- RELEASE.md | 3 +- .../integrations/kedro/data_loader.py | 5 +- package/kedro_viz/launchers/cli.py | 22 ++++++- package/kedro_viz/launchers/utils.py | 24 +++++++- package/tests/test_launchers/test_cli.py | 59 ++++++++++++++++++- package/tests/test_launchers/test_utils.py | 50 ++++++++++++++++ 6 files changed, 152 insertions(+), 11 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index aa7228960c..e9dc534e79 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,7 +10,9 @@ Please follow the established format: ## Major features and improvements ## Bug fixes and other changes + - Upgrade the gitpod workspace-full to a newer version which includes both Node 18 and Python 3.11.5. (#1862) +- Refactor backend integration with Kedro by replacing bootstrap_project with configure_project. (#1796) # Release 9.0.0 @@ -27,7 +29,6 @@ Please follow the established format: - Increase Kedro-Viz timeout. (#1803) - Remove demo data source and update feature hints. (#1804) - Add markdown support for backticks in the pop-up reminder. (#1826) -- Refactor backend integration with Kedro by replacing bootstrap_project with configure_project. (#1796) - Fix posix path conversion on Windows in DatasetStatsHook. (#1843) - Add `pydantic` pin to requirements. (#1861) - Fix TRANSCODING_SEPARATOR import error. (#1866) diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index 6beb174310..4662951781 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -3,7 +3,7 @@ load data from projects created in a range of Kedro versions. """ -# pylint: disable=import-outside-toplevel, protected-access +# pylint: disable=protected-access import json import logging @@ -14,6 +14,7 @@ from kedro.framework.project import configure_project, pipelines from kedro.framework.session import KedroSession from kedro.framework.session.store import BaseSessionStore +from kedro.framework.startup import bootstrap_project from kedro.io import DataCatalog from kedro.pipeline import Pipeline @@ -91,8 +92,6 @@ def load_data( if package_name: configure_project(package_name) else: - from kedro.framework.startup import bootstrap_project - # bootstrap project when viz is run in dev mode bootstrap_project(project_path) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index 87fa5b2676..caebdf502a 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -23,7 +23,9 @@ from kedro_viz.integrations.deployment.deployer_factory import DeployerFactory from kedro_viz.integrations.pypi import get_latest_version, is_running_outdated_version from kedro_viz.launchers.utils import ( + _PYPROJECT, _check_viz_up, + _find_kedro_project, _start_browser, _wait_for, viz_deploy_progress_timer, @@ -129,6 +131,21 @@ def run( """Launch local Kedro Viz instance""" from kedro_viz.server import run_server + kedro_project_path = _find_kedro_project(Path.cwd()) + + if kedro_project_path is None: + display_cli_message( + "ERROR: Could not find the project configuration " + f"file '{_PYPROJECT}' at '{Path.cwd()}'. " + f"If you have created your project with Kedro " + f"version <0.17.0, make sure to update your project template. " + f"See https://github.com/kedro-org/kedro/blob/main/RELEASE.md" + f"#migration-guide-from-kedro-016-to-kedro-0170 " + f"for how to migrate your Kedro project.", + "red", + ) + return + installed_version = parse(__version__) latest_version = get_latest_version() if is_running_outdated_version(installed_version, latest_version): @@ -152,16 +169,15 @@ def run( "save_file": save_file, "pipeline_name": pipeline, "env": env, + "project_path": kedro_project_path, "autoreload": autoreload, "include_hooks": include_hooks, "package_name": PACKAGE_NAME, "extra_params": params, } if autoreload: - project_path = Path.cwd() - run_server_kwargs["project_path"] = project_path run_process_kwargs = { - "path": project_path, + "path": kedro_project_path, "target": run_server, "kwargs": run_server_kwargs, "watcher_cls": RegExpWatcher, diff --git a/package/kedro_viz/launchers/utils.py b/package/kedro_viz/launchers/utils.py index 1ade2b426c..bf79602349 100644 --- a/package/kedro_viz/launchers/utils.py +++ b/package/kedro_viz/launchers/utils.py @@ -3,12 +3,14 @@ import logging import webbrowser +from pathlib import Path from time import sleep, time -from typing import Any, Callable +from typing import Any, Callable, Union import requests logger = logging.getLogger(__name__) +_PYPROJECT = "pyproject.toml" class WaitForException(Exception): @@ -105,3 +107,23 @@ def viz_deploy_progress_timer(process_completed, timeout): ) sleep(1) elapsed_time += 1 + + +def _is_project(project_path: Union[str, Path]) -> bool: + metadata_file = Path(project_path).expanduser().resolve() / _PYPROJECT + if not metadata_file.is_file(): + return False + + try: + return "[tool.kedro]" in metadata_file.read_text(encoding="utf-8") + # pylint: disable=broad-exception-caught + except Exception: + return False + + +def _find_kedro_project(current_dir: Path) -> Any: + paths_to_check = [current_dir] + list(current_dir.parents) + for project_dir in paths_to_check: + if _is_project(project_dir): + return project_dir + return None diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index df53b70ca5..73a3501596 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -9,6 +9,7 @@ from kedro_viz import __version__ from kedro_viz.constants import SHAREABLEVIZ_SUPPORTED_PLATFORMS, VIZ_DEPLOY_TIME_LIMIT from kedro_viz.launchers import cli +from kedro_viz.launchers.utils import _PYPROJECT from kedro_viz.server import run_server @@ -85,6 +86,7 @@ def mock_project_path(mocker): "save_file": None, "pipeline_name": None, "env": None, + "project_path": "testPath", "autoreload": False, "include_hooks": False, "package_name": None, @@ -100,6 +102,7 @@ def mock_project_path(mocker): "save_file": None, "pipeline_name": None, "env": None, + "project_path": "testPath", "autoreload": False, "include_hooks": False, "package_name": None, @@ -120,6 +123,7 @@ def mock_project_path(mocker): "save_file": None, "pipeline_name": None, "env": None, + "project_path": "testPath", "autoreload": False, "include_hooks": False, "package_name": None, @@ -151,6 +155,7 @@ def mock_project_path(mocker): "save_file": "save_dir", "pipeline_name": "data_science", "env": "local", + "project_path": "testPath", "autoreload": False, "include_hooks": False, "package_name": None, @@ -166,6 +171,7 @@ def mock_project_path(mocker): "save_file": None, "pipeline_name": None, "env": None, + "project_path": "testPath", "autoreload": False, "include_hooks": True, "package_name": None, @@ -185,6 +191,11 @@ def test_kedro_viz_command_run_server( runner = CliRunner() # Reduce the timeout argument from 600 to 1 to make test run faster. mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.cli._find_kedro_project", + return_value=run_server_args["project_path"], + ) with runner.isolated_filesystem(): runner.invoke(cli.viz_cli, command_options) @@ -195,8 +206,34 @@ def test_kedro_viz_command_run_server( assert run_server_args["port"] in cli._VIZ_PROCESSES +def test_kedro_viz_command_should_log_project_not_found( + mocker, mock_project_path, mock_click_echo +): + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch("kedro_viz.launchers.cli._find_kedro_project", return_value=None) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(cli.viz_cli, ["viz", "run"]) + + mock_click_echo_calls = [ + call( + "\x1b[31mERROR: Could not find the project configuration " + f"file '{_PYPROJECT}' at '{mock_project_path}'. " + f"If you have created your project with Kedro " + f"version <0.17.0, make sure to update your project template. " + f"See https://github.com/kedro-org/kedro/blob/main/RELEASE.md" + f"#migration-guide-from-kedro-016-to-kedro-0170 " + f"for how to migrate your Kedro project.\x1b[0m" + ) + ] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_kedro_viz_command_should_log_outdated_version( - mocker, mock_http_response, mock_click_echo + mocker, mock_http_response, mock_click_echo, mock_project_path ): installed_version = parse(__version__) mock_version = f"{installed_version.major + 1}.0.0" @@ -209,6 +246,10 @@ def test_kedro_viz_command_should_log_outdated_version( # Reduce the timeout argument from 600 to 1 to make test run faster. mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path + ) runner = CliRunner() with runner.isolated_filesystem(): runner.invoke(cli.viz_cli, ["viz", "run"]) @@ -228,7 +269,7 @@ def test_kedro_viz_command_should_log_outdated_version( def test_kedro_viz_command_should_not_log_latest_version( - mocker, mock_http_response, mock_click_echo + mocker, mock_http_response, mock_click_echo, mock_project_path ): requests_get = mocker.patch("requests.get") requests_get.return_value = mock_http_response( @@ -238,6 +279,10 @@ def test_kedro_viz_command_should_not_log_latest_version( mocker.patch("kedro_viz.server.run_server") # Reduce the timeout argument from 600 to 1 to make test run faster. mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path + ) runner = CliRunner() with runner.isolated_filesystem(): runner.invoke(cli.viz_cli, ["viz", "run"]) @@ -248,7 +293,7 @@ def test_kedro_viz_command_should_not_log_latest_version( def test_kedro_viz_command_should_not_log_if_pypi_is_down( - mocker, mock_http_response, mock_click_echo + mocker, mock_http_response, mock_click_echo, mock_project_path ): requests_get = mocker.patch("requests.get") requests_get.side_effect = requests.exceptions.RequestException("PyPI is down") @@ -256,6 +301,10 @@ def test_kedro_viz_command_should_not_log_if_pypi_is_down( mocker.patch("kedro_viz.server.run_server") # Reduce the timeout argument from 600 to 1 to make test run faster. mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path + ) runner = CliRunner() with runner.isolated_filesystem(): runner.invoke(cli.viz_cli, ["viz", "run"]) @@ -272,6 +321,10 @@ def test_kedro_viz_command_with_autoreload( # Reduce the timeout argument from 600 to 1 to make test run faster. mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path + ) runner = CliRunner() with runner.isolated_filesystem(): runner.invoke(cli.viz_cli, ["viz", "run", "--autoreload"]) diff --git a/package/tests/test_launchers/test_utils.py b/package/tests/test_launchers/test_utils.py index e9dead2f79..04425cfd09 100644 --- a/package/tests/test_launchers/test_utils.py +++ b/package/tests/test_launchers/test_utils.py @@ -1,3 +1,4 @@ +from pathlib import Path from unittest import mock from unittest.mock import Mock, call, patch @@ -7,6 +8,8 @@ from kedro_viz.constants import VIZ_DEPLOY_TIME_LIMIT from kedro_viz.launchers.utils import ( _check_viz_up, + _find_kedro_project, + _is_project, _start_browser, viz_deploy_progress_timer, ) @@ -69,3 +72,50 @@ def test_viz_deploy_progress_timer(capsys): for second in range(1, VIZ_DEPLOY_TIME_LIMIT + 1): expected_output = f"...Creating your build/deploy Kedro-Viz ({second}s)" assert expected_output in captured.out + + +class TestIsProject: + project_path = Path.cwd() + + def test_no_metadata_file(self, mocker): + mocker.patch.object(Path, "is_file", return_value=False) + + assert not _is_project(self.project_path) + + def test_toml_invalid_format(self, tmp_path): + """Test for loading context from an invalid path.""" + toml_path = tmp_path / "pyproject.toml" + toml_path.write_text("!!") # Invalid TOML + + assert not _is_project(tmp_path) + + def test_non_kedro_project(self, mocker): + mocker.patch.object(Path, "is_file", return_value=True) + mocker.patch.object(Path, "read_text", return_value="[tool]") + + assert not _is_project(self.project_path) + + def test_valid_toml_file(self, mocker): + mocker.patch.object(Path, "is_file", return_value=True) + pyproject_toml_payload = "[tool.kedro]" # \nproject_name = 'proj'" + mocker.patch.object(Path, "read_text", return_value=pyproject_toml_payload) + + assert _is_project(self.project_path) + + def test_toml_bad_encoding(self, mocker): + mocker.patch.object(Path, "is_file", return_value=True) + mocker.patch.object(Path, "read_text", side_effect=UnicodeDecodeError) + + assert not _is_project(self.project_path) + + +@pytest.mark.parametrize( + "project_dir, is_project_found, expected", + [ + ("/path/to/valid/project", True, Path("/path/to/valid/project")), + ("/path/to/nonexistent/project", False, None), + ], +) +def test_find_kedro_project(project_dir, is_project_found, expected, mocker): + mocker.patch("kedro_viz.launchers.utils._is_project", return_value=is_project_found) + assert _find_kedro_project(Path(project_dir)) == expected From dede026d45d640d1a79f2ea3c55a7805b8ee0d89 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 18 Apr 2024 15:04:20 -0500 Subject: [PATCH 10/13] update error message to be consistent with other viz commands Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/launchers/cli.py | 5 +++-- package/tests/test_launchers/test_cli.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index caebdf502a..840320d0e2 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -135,8 +135,9 @@ def run( if kedro_project_path is None: display_cli_message( - "ERROR: Could not find the project configuration " - f"file '{_PYPROJECT}' at '{Path.cwd()}'. " + "ERROR: Failed to start Kedro-Viz : " + "Could not find the project configuration " + f"file '{_PYPROJECT}' in '{Path.cwd()}'. " f"If you have created your project with Kedro " f"version <0.17.0, make sure to update your project template. " f"See https://github.com/kedro-org/kedro/blob/main/RELEASE.md" diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index 73a3501596..84e806c582 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -219,8 +219,9 @@ def test_kedro_viz_command_should_log_project_not_found( mock_click_echo_calls = [ call( - "\x1b[31mERROR: Could not find the project configuration " - f"file '{_PYPROJECT}' at '{mock_project_path}'. " + "\x1b[31mERROR: Failed to start Kedro-Viz : " + "Could not find the project configuration " + f"file '{_PYPROJECT}' in '{mock_project_path}'. " f"If you have created your project with Kedro " f"version <0.17.0, make sure to update your project template. " f"See https://github.com/kedro-org/kedro/blob/main/RELEASE.md" From a68fcb743e96ed7fa532b4b530b40f7d9254de82 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Mon, 22 Apr 2024 12:19:30 -0500 Subject: [PATCH 11/13] update release note: Signed-off-by: ravi-kumar-pilla --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index e9dc534e79..7fee404902 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -13,6 +13,7 @@ Please follow the established format: - Upgrade the gitpod workspace-full to a newer version which includes both Node 18 and Python 3.11.5. (#1862) - Refactor backend integration with Kedro by replacing bootstrap_project with configure_project. (#1796) +- Fix Kedro Viz waiting for valid Kedro project. (#1871) # Release 9.0.0 From 01eb943b7eac96dfe4bbb7896af9469eaa0d0530 Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Wed, 24 Apr 2024 11:58:55 -0500 Subject: [PATCH 12/13] update release note Signed-off-by: ravi-kumar-pilla --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 7fee404902..2010bdf855 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -13,7 +13,7 @@ Please follow the established format: - Upgrade the gitpod workspace-full to a newer version which includes both Node 18 and Python 3.11.5. (#1862) - Refactor backend integration with Kedro by replacing bootstrap_project with configure_project. (#1796) -- Fix Kedro Viz waiting for valid Kedro project. (#1871) +- Fix Kedro-Viz waiting for valid Kedro project. (#1871) # Release 9.0.0 From f16816b4530136f32c4dfe70ff210d88e091c27b Mon Sep 17 00:00:00 2001 From: ravi-kumar-pilla Date: Thu, 25 Apr 2024 11:53:42 -0500 Subject: [PATCH 13/13] update error message Signed-off-by: ravi-kumar-pilla --- package/kedro_viz/launchers/cli.py | 7 +------ package/tests/test_launchers/test_cli.py | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py index 840320d0e2..5c2bcac75f 100644 --- a/package/kedro_viz/launchers/cli.py +++ b/package/kedro_viz/launchers/cli.py @@ -137,12 +137,7 @@ def run( display_cli_message( "ERROR: Failed to start Kedro-Viz : " "Could not find the project configuration " - f"file '{_PYPROJECT}' in '{Path.cwd()}'. " - f"If you have created your project with Kedro " - f"version <0.17.0, make sure to update your project template. " - f"See https://github.com/kedro-org/kedro/blob/main/RELEASE.md" - f"#migration-guide-from-kedro-016-to-kedro-0170 " - f"for how to migrate your Kedro project.", + f"file '{_PYPROJECT}' at '{Path.cwd()}'. ", "red", ) return diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py index 84e806c582..4ad1b6dd86 100755 --- a/package/tests/test_launchers/test_cli.py +++ b/package/tests/test_launchers/test_cli.py @@ -221,12 +221,7 @@ def test_kedro_viz_command_should_log_project_not_found( call( "\x1b[31mERROR: Failed to start Kedro-Viz : " "Could not find the project configuration " - f"file '{_PYPROJECT}' in '{mock_project_path}'. " - f"If you have created your project with Kedro " - f"version <0.17.0, make sure to update your project template. " - f"See https://github.com/kedro-org/kedro/blob/main/RELEASE.md" - f"#migration-guide-from-kedro-016-to-kedro-0170 " - f"for how to migrate your Kedro project.\x1b[0m" + f"file '{_PYPROJECT}' at '{mock_project_path}'. \x1b[0m" ) ]