From 5f018acb3b197ae426aa4f207912dcbe686fb0a3 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Fri, 9 Sep 2022 10:58:13 +0100 Subject: [PATCH 1/5] Alias Kedro IPython extension to kedro.ipython (#1837) --- MANIFEST.in | 2 +- RELEASE.md | 4 +- docs/source/deployment/databricks.md | 4 +- docs/source/tools_integration/ipython.md | 10 +- kedro/extras/extensions/ipython.py | 157 +++--------------- kedro/framework/cli/jupyter.py | 10 +- kedro/framework/cli/pipeline.py | 1 - kedro/framework/cli/project.py | 2 +- kedro/ipython/__init__.py | 134 +++++++++++++++ .../extensions => ipython}/logo-32x32.png | Bin .../extensions => ipython}/logo-64x64.png | Bin pyproject.toml | 2 +- tests/framework/cli/test_jupyter.py | 2 +- tests/framework/cli/test_project.py | 2 +- .../extensions => ipython}/__init__.py | 0 .../extensions => ipython}/test_ipython.py | 112 ++++++------- 16 files changed, 227 insertions(+), 215 deletions(-) create mode 100644 kedro/ipython/__init__.py rename kedro/{extras/extensions => ipython}/logo-32x32.png (100%) rename kedro/{extras/extensions => ipython}/logo-64x64.png (100%) rename tests/{extras/extensions => ipython}/__init__.py (100%) rename tests/{extras/extensions => ipython}/test_ipython.py (70%) diff --git a/MANIFEST.in b/MANIFEST.in index 59a7c64ab2..97d23372d5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,5 +3,5 @@ include LICENSE.md include requirements.txt include test_requirements.txt include kedro/framework/project/default_logging.yml -include kedro/extras/extensions/*.png +include kedro/ipython/*.png recursive-include templates * diff --git a/RELEASE.md b/RELEASE.md index 41473964d8..81a2e34bd5 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,6 +12,7 @@ # Upcoming Release 0.18.3 ## Major features and improvements +* The Kedro IPython extension should now be loaded with `%load_ext kedro.ipython`. * The line magic `%reload_kedro` now accepts keywords arguments, e.g. `%reload_kedro --env=prod`. ## Bug fixes and other changes @@ -29,6 +30,7 @@ * Relaxed `delta-spark` upper bound to allow compatibility with Spark 3.1.x and 3.2.x. ## Upcoming deprecations for Kedro 0.19.0 +* The Kedro IPython extension will no longer be available as `%load_ext kedro.extras.extensions.ipython`; use `%load_ext kedro.ipython` instead. # Release 0.18.2 @@ -129,7 +131,7 @@ main( * Added `save_args` to `feather.FeatherDataSet`. ### Jupyter and IPython integration -* The [only recommended way to work with Kedro in Jupyter or IPython is now the Kedro IPython extension](https://kedro.readthedocs.io/en/0.18.0/tools_integration/ipython.html). Managed Jupyter instances should load this via `%load_ext kedro.extras.extensions.ipython` and use the line magic `%reload_kedro`. +* The [only recommended way to work with Kedro in Jupyter or IPython is now the Kedro IPython extension](https://kedro.readthedocs.io/en/0.18.0/tools_integration/ipython.html). Managed Jupyter instances should load this via `%load_ext kedro.ipython` and use the line magic `%reload_kedro`. * `kedro ipython` launches an IPython session that preloads the Kedro IPython extension. * `kedro jupyter notebook/lab` creates a custom Jupyter kernel that preloads the Kedro IPython extension and launches a notebook with that kernel selected. There is no longer a need to specify `--all-kernels` to show all available kernels. diff --git a/docs/source/deployment/databricks.md b/docs/source/deployment/databricks.md index 21a07dab84..7ee533b68b 100644 --- a/docs/source/deployment/databricks.md +++ b/docs/source/deployment/databricks.md @@ -230,7 +230,7 @@ Your complete notebook should look similar to this (the results are hidden): ### 9. Using the Kedro IPython Extension -You can interact with Kedro in Databricks through the Kedro [IPython extension](https://ipython.readthedocs.io/en/stable/config/extensions/index.html), `kedro.extras.extensions.ipython`. +You can interact with Kedro in Databricks through the Kedro [IPython extension](https://ipython.readthedocs.io/en/stable/config/extensions/index.html), `kedro.ipython`. The Kedro IPython extension launches a [Kedro session](../kedro_project_setup/session.md) and makes available the useful Kedro variables `catalog`, `context`, `pipelines` and `session`. It also provides the `%reload_kedro` [line magic](https://ipython.readthedocs.io/en/stable/interactive/magics.html) that reloads these variables (for example, if you need to update `catalog` following changes to your Data Catalog). @@ -239,7 +239,7 @@ The IPython extension can be used in a Databricks notebook in a similar way to h If you encounter a `ContextualVersionConflictError`, it is likely caused by Databricks using an old version of `pip`. Hence there's one additional step you need to do in the Databricks notebook to make use of the IPython extension. After you load the IPython extension using the below command: ```ipython -In [1]: %load_ext kedro.extras.extensions.ipython +In [1]: %load_ext kedro.ipython ``` You must explicitly upgrade your `pip` version by doing the below: diff --git a/docs/source/tools_integration/ipython.md b/docs/source/tools_integration/ipython.md index feaa0122a2..00d6698cc4 100644 --- a/docs/source/tools_integration/ipython.md +++ b/docs/source/tools_integration/ipython.md @@ -13,12 +13,12 @@ There are reasons why you may want to use a Notebook, although in general, the p ## Kedro IPython extension -The recommended way to interact with Kedro in IPython and Jupyter is through the Kedro [IPython extension](https://ipython.readthedocs.io/en/stable/config/extensions/index.html), `kedro.extras.extensions.ipython`. An [IPython extension](https://ipython.readthedocs.io/en/stable/config/extensions/) is an importable Python module that has a couple of special functions to load and unload it. +The recommended way to interact with Kedro in IPython and Jupyter is through the Kedro [IPython extension](https://ipython.readthedocs.io/en/stable/config/extensions/index.html), `kedro.ipython`. An [IPython extension](https://ipython.readthedocs.io/en/stable/config/extensions/) is an importable Python module that has a couple of special functions to load and unload it. The Kedro IPython extension launches a [Kedro session](../kedro_project_setup/session.md) and makes available the useful Kedro variables `catalog`, `context`, `pipelines` and `session`. It also provides the `%reload_kedro` [line magic](https://ipython.readthedocs.io/en/stable/interactive/magics.html) that reloads these variables (for example, if you need to update `catalog` following changes to your Data Catalog). The simplest way to make use of the Kedro IPython extension is through the following commands: -* `kedro ipython`. This launches an IPython shell with the extension already loaded and is equivalent to the command `ipython --ext kedro.extras.extensions.ipython`. +* `kedro ipython`. This launches an IPython shell with the extension already loaded and is equivalent to the command `ipython --ext kedro.ipython`. * `kedro jupyter notebook`. This creates a custom Jupyter kernel that automatically loads the extension and launches Jupyter Notebook with this kernel selected. * `kedro jupyter lab`. This creates a custom Jupyter kernel that automatically loads the extension and launches JupyterLab with this kernel selected. @@ -32,7 +32,7 @@ If these variables are not available then Kedro has not been able to load your p If the above commands are not available to you (e.g. you work in a managed Jupyter service such as a Databricks Notebook) then equivalent behaviour can be achieved by explicitly loading the Kedro IPython extension with the `%load_ext` line magic: ```ipython -In [1]: %load_ext kedro.extras.extensions.ipython +In [1]: %load_ext kedro.ipython ``` If your IPython or Jupyter instance was launched from outside your Kedro project then you will need to run a second line magic to set the project path so that Kedro can load the `catalog`, `context`, `pipelines` and `session` variables: @@ -42,7 +42,7 @@ In [2]: %reload_kedro The Kedro IPython extension remembers the project path so that subsequent calls to `%reload_kedro` do not need to specify it: ```ipython -In [1]: %load_ext kedro.extras.extensions.ipython +In [1]: %load_ext kedro.ipython In [2]: %reload_kedro In [3]: %reload_kedro ``` @@ -204,7 +204,7 @@ If you are not able to execute `kedro jupyter notebook` or `kedro jupyter lab` t ### Manage Jupyter kernels -Behind the scenes, the `kedro jupyter notebook` and `kedro jupyter lab` commands create a Jupyter kernel named `kedro_`. This kernel is identical to the [default IPython kernel](https://ipython.readthedocs.io/en/stable/install/kernel_install.html) but with a slightly customised [kernel specification](https://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs) that automatically loads `kedro.extras.extensions.ipython` when the kernel is started. The kernel specification is installed at a user level rather than system-wide. +Behind the scenes, the `kedro jupyter notebook` and `kedro jupyter lab` commands create a Jupyter kernel named `kedro_`. This kernel is identical to the [default IPython kernel](https://ipython.readthedocs.io/en/stable/install/kernel_install.html) but with a slightly customised [kernel specification](https://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs) that automatically loads `kedro.ipython` when the kernel is started. The kernel specification is installed at a user level rather than system-wide. ```{note} If a Jupyter kernel with the name `kedro_` already exists then it is replaced. This ensures that the kernel always points to the correct Python executable. For example, if you change conda environment in a Kedro project then you should re-run `kedro jupyter notebook/lab` to replace the kernel specification with one that points to the new environment. diff --git a/kedro/extras/extensions/ipython.py b/kedro/extras/extensions/ipython.py index c5f6f701a2..2e38dc3772 100644 --- a/kedro/extras/extensions/ipython.py +++ b/kedro/extras/extensions/ipython.py @@ -1,139 +1,22 @@ -# pylint: disable=import-outside-toplevel,global-statement,invalid-name,too-many-locals """ -This script creates an IPython extension to load Kedro-related variables in -local scope. +This file and directory exists purely for backwards compatibility of the following: +%load_ext kedro.extras.extensions.ipython +from kedro.extras.extensions.ipython import reload_kedro + +Any modifications to the IPython extension should now be made in kedro/ipython/. +The Kedro IPython extension should always be loaded as %load_ext kedro.ipython. +Line magics such as reload_kedro should always be called as line magics rather than +importing the underlying Python functions. """ -import logging -import sys -from pathlib import Path -from typing import Any, Dict - -from kedro.framework.cli.project import PARAMS_ARG_HELP -from kedro.framework.cli.utils import ENV_HELP, _split_params - -logger = logging.getLogger(__name__) -default_project_path = Path.cwd() - - -def _remove_cached_modules(package_name): - to_remove = [mod for mod in sys.modules if mod.startswith(package_name)] - # `del` is used instead of `reload()` because: If the new version of a module does not - # define a name that was defined by the old version, the old definition remains. - for module in to_remove: - del sys.modules[module] # pragma: no cover - - -def _find_kedro_project(current_dir: Path): # pragma: no cover - from kedro.framework.startup import _is_project - - while current_dir != current_dir.parent: - if _is_project(current_dir): - return current_dir - current_dir = current_dir.parent - - return None - - -def reload_kedro( - path: str = None, env: str = None, extra_params: Dict[str, Any] = None -): - """Line magic which reloads all Kedro default variables. - Setting the path will also make it default for subsequent calls. - """ - from IPython import get_ipython - from IPython.core.magic import needs_local_scope, register_line_magic - - from kedro.framework.cli import load_entry_points - from kedro.framework.project import LOGGING # noqa # pylint:disable=unused-import - from kedro.framework.project import configure_project, pipelines - from kedro.framework.session import KedroSession - from kedro.framework.startup import bootstrap_project - - # If a path is provided, set it as default for subsequent calls - global default_project_path - if path: - default_project_path = Path(path).expanduser().resolve() - logger.info("Updated path to Kedro project: %s", default_project_path) - else: - logger.info("No path argument was provided. Using: %s", default_project_path) - - metadata = bootstrap_project(default_project_path) - _remove_cached_modules(metadata.package_name) - configure_project(metadata.package_name) - - session = KedroSession.create( - metadata.package_name, default_project_path, env=env, extra_params=extra_params - ) - context = session.load_context() - catalog = context.catalog - - get_ipython().push( - variables={ - "context": context, - "catalog": catalog, - "session": session, - "pipelines": pipelines, - } - ) - - logger.info("Kedro project %s", str(metadata.project_name)) - logger.info( - "Defined global variable 'context', 'session', 'catalog' and 'pipelines'" - ) - - for line_magic in load_entry_points("line_magic"): - register_line_magic(needs_local_scope(line_magic)) - logger.info("Registered line magic '%s'", line_magic.__name__) # type: ignore - - -def load_ipython_extension(ipython): - """ - Main entry point when %load_ext is executed. - IPython will look for this function specifically. - See https://ipython.readthedocs.io/en/stable/config/extensions/index.html - - This function is called when users do `%load_ext kedro.extras.extensions.ipython`. - When user use `kedro jupyter notebook` or `jupyter ipython`, this extension is - loaded automatically. - """ - from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring - - @magic_arguments() - @argument( - "path", - type=str, - help=( - "Path to the project root directory. If not given, use the previously set" - "project root." - ), - nargs="?", - default=None, - ) - @argument("-e", "--env", type=str, default=None, help=ENV_HELP) - @argument( - "--params", - type=lambda value: _split_params(None, None, value), - default=None, - help=PARAMS_ARG_HELP, - ) - def magic_reload_kedro(line: str): - """ - The `%reload_kedro` IPython line magic. See - https://kedro.readthedocs.io/en/stable/tools_integration/ipython.html for more. - """ - args = parse_argstring(magic_reload_kedro, line) - reload_kedro(args.path, args.env, args.params) - - global default_project_path - - ipython.register_magic_function(magic_reload_kedro, magic_name="reload_kedro") - default_project_path = _find_kedro_project(Path.cwd()) - - if default_project_path is None: - logger.warning( - "Kedro extension was registered but couldn't find a Kedro project. " - "Make sure you run '%reload_kedro '." - ) - return - - reload_kedro(default_project_path) +import warnings + +from ...ipython import ( # noqa # pylint: disable=unused-import + load_ipython_extension, + reload_kedro, +) + +warnings.warn( + "kedro.extras.extensions.ipython should be accessed only using the alias " + "kedro.ipython. The unaliased name will be removed in Kedro 0.19.0.", + DeprecationWarning, +) diff --git a/kedro/framework/cli/jupyter.py b/kedro/framework/cli/jupyter.py index 9a0ee3e028..1d4b5b54ad 100644 --- a/kedro/framework/cli/jupyter.py +++ b/kedro/framework/cli/jupyter.py @@ -117,7 +117,7 @@ def _create_kernel(kernel_name: str, display_name: str) -> None: "-f", "{connection_file}", "--ext", - "kedro.extras.extensions.ipython" + "kedro.ipython" ], "display_name": "Kedro (spaceflights)", "language": "python", @@ -151,14 +151,14 @@ def _create_kernel(kernel_name: str, display_name: str) -> None: kernel_json = Path(kernel_path) / "kernel.json" kernel_spec = json.loads(kernel_json.read_text(encoding="utf-8")) - kernel_spec["argv"].extend(["--ext", "kedro.extras.extensions.ipython"]) + kernel_spec["argv"].extend(["--ext", "kedro.ipython"]) # indent=1 is to match the default ipykernel style (see # ipykernel.write_kernel_spec). kernel_json.write_text(json.dumps(kernel_spec, indent=1), encoding="utf-8") - kedro_extensions_dir = Path(__file__).parents[2] / "extras" / "extensions" - shutil.copy(kedro_extensions_dir / "logo-32x32.png", kernel_path) - shutil.copy(kedro_extensions_dir / "logo-64x64.png", kernel_path) + kedro_ipython_dir = Path(__file__).parents[2] / "ipython" + shutil.copy(kedro_ipython_dir / "logo-32x32.png", kernel_path) + shutil.copy(kedro_ipython_dir / "logo-64x64.png", kernel_path) except Exception as exc: raise KedroCliError( f"Cannot setup kedro kernel for Jupyter.\nError: {exc}" diff --git a/kedro/framework/cli/pipeline.py b/kedro/framework/cli/pipeline.py index 062ffcdf05..b648e6608b 100644 --- a/kedro/framework/cli/pipeline.py +++ b/kedro/framework/cli/pipeline.py @@ -223,7 +223,6 @@ def _create_pipeline(name: str, output_dir: Path) -> Path: return result_path -# pylint: disable=missing-raises-doc def _sync_dirs(source: Path, target: Path, prefix: str = "", overwrite: bool = False): """Recursively copies `source` directory (or file) into `target` directory without overwriting any existing files/directories in the target using the following diff --git a/kedro/framework/cli/project.py b/kedro/framework/cli/project.py index 59c483b208..c0b0fc11f0 100644 --- a/kedro/framework/cli/project.py +++ b/kedro/framework/cli/project.py @@ -124,7 +124,7 @@ def ipython( if env: os.environ["KEDRO_ENV"] = env - call(["ipython", "--ext", "kedro.extras.extensions.ipython"] + list(args)) + call(["ipython", "--ext", "kedro.ipython"] + list(args)) @project_group.command() diff --git a/kedro/ipython/__init__.py b/kedro/ipython/__init__.py new file mode 100644 index 0000000000..a98b5cfb86 --- /dev/null +++ b/kedro/ipython/__init__.py @@ -0,0 +1,134 @@ +# pylint: disable=global-statement,invalid-name +""" +This script creates an IPython extension to load Kedro-related variables in +local scope. +""" +import logging +import sys +from pathlib import Path +from typing import Any, Dict + +from IPython import get_ipython +from IPython.core.magic import needs_local_scope, register_line_magic +from IPython.core.magic_arguments import argument, magic_arguments, parse_argstring + +from kedro.framework.cli import load_entry_points +from kedro.framework.cli.project import PARAMS_ARG_HELP +from kedro.framework.cli.utils import ENV_HELP, _split_params +from kedro.framework.project import LOGGING # noqa +from kedro.framework.project import configure_project, pipelines +from kedro.framework.session import KedroSession +from kedro.framework.startup import _is_project, bootstrap_project + +logger = logging.getLogger(__name__) +default_project_path = Path.cwd() + + +def _remove_cached_modules(package_name): # pragma: no cover + to_remove = [mod for mod in sys.modules if mod.startswith(package_name)] + # `del` is used instead of `reload()` because: If the new version of a module does not + # define a name that was defined by the old version, the old definition remains. + for module in to_remove: + del sys.modules[module] + + +def _find_kedro_project(current_dir: Path): # pragma: no cover + while current_dir != current_dir.parent: + if _is_project(current_dir): + return current_dir + current_dir = current_dir.parent + + return None + + +def reload_kedro( + path: str = None, env: str = None, extra_params: Dict[str, Any] = None +): # pragma: no cover + """Function that underlies the %reload_kedro Line magic. This should not be imported + or run directly but instead invoked through %reload_kedro.""" + + # If a path is provided, set it as default for subsequent calls + global default_project_path + if path: + default_project_path = Path(path).expanduser().resolve() + logger.info("Updated path to Kedro project: %s", default_project_path) + else: + logger.info("No path argument was provided. Using: %s", default_project_path) + + metadata = bootstrap_project(default_project_path) + _remove_cached_modules(metadata.package_name) + configure_project(metadata.package_name) + + session = KedroSession.create( + metadata.package_name, default_project_path, env=env, extra_params=extra_params + ) + context = session.load_context() + catalog = context.catalog + + get_ipython().push( + variables={ + "context": context, + "catalog": catalog, + "session": session, + "pipelines": pipelines, + } + ) + + logger.info("Kedro project %s", str(metadata.project_name)) + logger.info( + "Defined global variable 'context', 'session', 'catalog' and 'pipelines'" + ) + + for line_magic in load_entry_points("line_magic"): + register_line_magic(needs_local_scope(line_magic)) + logger.info("Registered line magic '%s'", line_magic.__name__) # type: ignore + + +@magic_arguments() +@argument( + "path", + type=str, + help=( + "Path to the project root directory. If not given, use the previously set" + "project root." + ), + nargs="?", + default=None, +) +@argument("-e", "--env", type=str, default=None, help=ENV_HELP) +@argument( + "--params", + type=lambda value: _split_params(None, None, value), + default=None, + help=PARAMS_ARG_HELP, +) +def magic_reload_kedro(line: str): + """ + The `%reload_kedro` IPython line magic. See + https://kedro.readthedocs.io/en/stable/tools_integration/ipython.html for more. + """ + args = parse_argstring(magic_reload_kedro, line) + reload_kedro(args.path, args.env, args.params) + + +def load_ipython_extension(ipython): + """ + Main entry point when %load_ext kedro.ipython is executed, either manually or + automatically through `kedro ipython` or `kedro jupyter lab/notebook`. + IPython will look for this function specifically. + See https://ipython.readthedocs.io/en/stable/config/extensions/index.html + """ + + global default_project_path + + ipython.register_magic_function(magic_reload_kedro, magic_name="reload_kedro") + default_project_path = _find_kedro_project(Path.cwd()) + + if default_project_path is None: + logger.warning( + "Kedro extension was registered but couldn't find a Kedro project. " + "Make sure you run '%reload_kedro '." + ) + return + + reload_kedro(default_project_path) diff --git a/kedro/extras/extensions/logo-32x32.png b/kedro/ipython/logo-32x32.png similarity index 100% rename from kedro/extras/extensions/logo-32x32.png rename to kedro/ipython/logo-32x32.png diff --git a/kedro/extras/extensions/logo-64x64.png b/kedro/ipython/logo-64x64.png similarity index 100% rename from kedro/extras/extensions/logo-64x64.png rename to kedro/ipython/logo-64x64.png diff --git a/pyproject.toml b/pyproject.toml index ff54547961..9c3e01124a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ min-public-methods = 1 [tool.coverage.report] fail_under = 100 show_missing = true -omit = ["kedro/templates/*", "kedro/extras/logging/color_logger.py", "kedro/framework/cli/hooks/specs.py", "kedro/framework/hooks/specs.py", "kedro/extras/datasets/tensorflow/*", "kedro/extras/datasets/holoviews/*", "tests/*"] +omit = ["kedro/templates/*", "kedro/extras/logging/color_logger.py", "kedro/extras/extensions/ipython.py", "kedro/framework/cli/hooks/specs.py", "kedro/framework/hooks/specs.py", "kedro/extras/datasets/tensorflow/*", "kedro/extras/datasets/holoviews/*", "tests/*"] exclude_lines = ["pragma: no cover", "raise NotImplementedError"] [tool.pytest.ini_options] diff --git a/tests/framework/cli/test_jupyter.py b/tests/framework/cli/test_jupyter.py index 8082fc3154..a1466fd5e1 100644 --- a/tests/framework/cli/test_jupyter.py +++ b/tests/framework/cli/test_jupyter.py @@ -139,7 +139,7 @@ def test_create_new_kernel(self): kernel_spec = get_kernel_spec("my_kernel_name") assert kernel_spec.display_name == "My display name" assert kernel_spec.language == "python" - assert kernel_spec.argv[-2:] == ["--ext", "kedro.extras.extensions.ipython"] + assert kernel_spec.argv[-2:] == ["--ext", "kedro.ipython"] kernel_files = {file.name for file in Path(kernel_spec.resource_dir).iterdir()} assert kernel_files == {"logo-32x32.png", "logo-64x64.png", "kernel.json"} diff --git a/tests/framework/cli/test_project.py b/tests/framework/cli/test_project.py index f12e01c3b5..888f142114 100644 --- a/tests/framework/cli/test_project.py +++ b/tests/framework/cli/test_project.py @@ -249,7 +249,7 @@ def test_happy_path( [ "ipython", "--ext", - "kedro.extras.extensions.ipython", + "kedro.ipython", "--random-arg", "value", ] diff --git a/tests/extras/extensions/__init__.py b/tests/ipython/__init__.py similarity index 100% rename from tests/extras/extensions/__init__.py rename to tests/ipython/__init__.py diff --git a/tests/extras/extensions/test_ipython.py b/tests/ipython/test_ipython.py similarity index 70% rename from tests/extras/extensions/test_ipython.py rename to tests/ipython/test_ipython.py index f17f348835..e8c7178bf9 100644 --- a/tests/extras/extensions/test_ipython.py +++ b/tests/ipython/test_ipython.py @@ -3,15 +3,15 @@ from IPython.core.error import UsageError from IPython.testing.globalipapp import get_ipython -from kedro.extras.extensions.ipython import load_ipython_extension, reload_kedro from kedro.framework.startup import ProjectMetadata +from kedro.ipython import load_ipython_extension, reload_kedro from kedro.pipeline import Pipeline @pytest.fixture(autouse=True) def project_path(mocker, tmp_path): path = tmp_path - mocker.patch("kedro.extras.extensions.ipython.default_project_path", path) + mocker.patch("kedro.ipython.default_project_path", path) @pytest.fixture(autouse=True) @@ -29,28 +29,29 @@ def ipython(): return ipython -PACKAGE_NAME = "fake_page_name" +PACKAGE_NAME = "fake_package_name" PROJECT_NAME = "fake_project_name" PROJECT_VERSION = "0.1" +@pytest.mark.skip() class TestLoadKedroObjects: def test_load_kedro_objects( self, tmp_path, mocker, caplog ): # pylint: disable=too-many-locals - from kedro.extras.extensions.ipython import default_project_path + from kedro.ipython import default_project_path assert default_project_path == tmp_path kedro_path = tmp_path / "here" fake_metadata = ProjectMetadata( - source_dir=tmp_path / "src", # default - config_file=tmp_path / "pyproject.toml", + source_dir=kedro_path / "src", # default + config_file=kedro_path / "pyproject.toml", package_name=PACKAGE_NAME, project_name=PROJECT_NAME, project_version=PROJECT_VERSION, - project_path=tmp_path, + project_path=kedro_path, ) my_pipelines = {"ds": Pipeline([])} @@ -62,21 +63,13 @@ def my_register_pipeline(): return_value=my_register_pipeline, ) mocker.patch("kedro.framework.startup.configure_project") - mocker.patch( - "kedro.framework.startup.bootstrap_project", return_value=fake_metadata - ) + mocker.patch("kedro.ipython.bootstrap_project", return_value=fake_metadata) mock_line_magic = mocker.Mock() mock_line_magic.__name__ = "abc" - mocker.patch( - "kedro.framework.cli.load_entry_points", return_value=[mock_line_magic] - ) - mock_register_line_magic = mocker.patch( - "IPython.core.magic.register_line_magic" - ) - mock_session_create = mocker.patch( - "kedro.framework.session.KedroSession.create" - ) - mock_ipython = mocker.patch("IPython.get_ipython") + mocker.patch("kedro.ipython.load_entry_points", return_value=[mock_line_magic]) + mock_register_line_magic = mocker.patch("kedro.ipython.register_line_magic") + mock_session_create = mocker.patch("kedro.ipython.KedroSession.create") + mock_ipython = mocker.patch("kedro.ipython.get_ipython") reload_kedro(kedro_path) @@ -98,7 +91,7 @@ def my_register_pipeline(): log_messages = [record.getMessage() for record in caplog.records] assert expected_message in log_messages - from kedro.extras.extensions.ipython import default_project_path + from kedro.ipython import default_project_path # make sure global variable updated assert default_project_path == expected_path @@ -112,22 +105,24 @@ def test_load_kedro_objects_extra_args(self, tmp_path, mocker): project_version=PROJECT_VERSION, project_path=tmp_path, ) - mocker.patch("kedro.framework.project.configure_project") + + my_pipelines = {"ds": Pipeline([])} + + def my_register_pipeline(): + return my_pipelines + mocker.patch( - "kedro.framework.startup.bootstrap_project", return_value=fake_metadata + "kedro.framework.project._ProjectPipelines._get_pipelines_registry_callable", + return_value=my_register_pipeline, ) + mocker.patch("kedro.ipython.configure_project") + mocker.patch("kedro.ipython.bootstrap_project", return_value=fake_metadata) mock_line_magic = mocker.Mock() mock_line_magic.__name__ = "abc" - mocker.patch( - "kedro.framework.cli.load_entry_points", return_value=[mock_line_magic] - ) - mock_register_line_magic = mocker.patch( - "IPython.core.magic.register_line_magic" - ) - mock_session_create = mocker.patch( - "kedro.framework.session.KedroSession.create" - ) - mock_ipython = mocker.patch("IPython.get_ipython") + mocker.patch("kedro.ipython.load_entry_points", return_value=[mock_line_magic]) + mock_register_line_magic = mocker.patch("kedro.ipython.register_line_magic") + mock_session_create = mocker.patch("kedro.ipython.KedroSession.create") + mock_ipython = mocker.patch("kedro.ipython.get_ipython") reload_kedro(tmp_path, env="env1", extra_params={"key": "val"}) @@ -147,7 +142,7 @@ def test_load_kedro_objects_extra_args(self, tmp_path, mocker): def test_load_kedro_objects_no_path( self, tmp_path, caplog, mocker, ipython ): # pylint: disable=unused-argument - from kedro.extras.extensions.ipython import default_project_path + from kedro.ipython import default_project_path assert default_project_path == tmp_path @@ -159,6 +154,7 @@ def test_load_kedro_objects_no_path( project_version=PROJECT_VERSION, project_path=tmp_path, ) + my_pipelines = {"ds": Pipeline([])} def my_register_pipeline(): @@ -168,19 +164,15 @@ def my_register_pipeline(): "kedro.framework.project._ProjectPipelines._get_pipelines_registry_callable", return_value=my_register_pipeline, ) - mocker.patch("kedro.framework.project.settings.configure") - mocker.patch("kedro.framework.session.session.validate_settings") - mocker.patch("kedro.framework.session.KedroSession._setup_logging") - mocker.patch( - "kedro.framework.startup.bootstrap_project", return_value=fake_metadata - ) + + mocker.patch("kedro.ipython.configure_project") + mocker.patch("kedro.ipython.bootstrap_project", return_value=fake_metadata) mock_line_magic = mocker.Mock() mock_line_magic.__name__ = "abc" - mocker.patch( - "kedro.framework.cli.load_entry_points", return_value=[mock_line_magic] - ) - mocker.patch("IPython.core.magic.register_line_magic") - mocker.patch("kedro.framework.session.KedroSession.load_context") + mocker.patch("kedro.ipython.load_entry_points", return_value=[mock_line_magic]) + mocker.patch("kedro.ipython.register_line_magic") + mocker.patch("kedro.ipython.KedroSession.create") + mocker.patch("kedro.ipython.get_ipython") reload_kedro() @@ -188,19 +180,23 @@ def my_register_pipeline(): log_messages = [record.getMessage() for record in caplog.records] assert expected_message in log_messages - from kedro.extras.extensions.ipython import default_project_path + from kedro.ipython import default_project_path # make sure global variable stayed the same assert default_project_path == tmp_path class TestLoadIPythonExtension: + def test_load_ipython_extension(self, ipython): + ipython.magic("load_ext kedro.ipython") + + def test_load_ipython_extension_old_location(self, ipython): + ipython.magic("load_ext kedro.ipython") + def test_load_extension_missing_dependency(self, mocker): + mocker.patch("kedro.ipython.reload_kedro", side_effect=ImportError) mocker.patch( - "kedro.extras.extensions.ipython.reload_kedro", side_effect=ImportError - ) - mocker.patch( - "kedro.extras.extensions.ipython._find_kedro_project", + "kedro.ipython._find_kedro_project", return_value=mocker.Mock(), ) mocker.patch("IPython.core.magic.register_line_magic") @@ -215,9 +211,7 @@ def test_load_extension_missing_dependency(self, mocker): assert not mock_ipython().push.called def test_load_extension_not_in_kedro_project(self, mocker, caplog): - mocker.patch( - "kedro.extras.extensions.ipython._find_kedro_project", return_value=None - ) + mocker.patch("kedro.ipython._find_kedro_project", return_value=None) mocker.patch("IPython.core.magic.register_line_magic") mocker.patch("IPython.core.magic_arguments.magic_arguments") mocker.patch("IPython.core.magic_arguments.argument") @@ -237,8 +231,8 @@ def test_load_extension_not_in_kedro_project(self, mocker, caplog): def test_load_extension_register_line_magic(self, mocker, ipython): - mocker.patch("kedro.extras.extensions.ipython._find_kedro_project") - mock_reload_kedro = mocker.patch("kedro.extras.extensions.ipython.reload_kedro") + mocker.patch("kedro.ipython._find_kedro_project") + mock_reload_kedro = mocker.patch("kedro.ipython.reload_kedro") load_ipython_extension(ipython) mock_reload_kedro.assert_called_once() @@ -258,14 +252,14 @@ def test_load_extension_register_line_magic(self, mocker, ipython): ], ) def test_line_magic_with_valid_arguments(self, mocker, args, ipython): - mocker.patch("kedro.extras.extensions.ipython._find_kedro_project") - mocker.patch("kedro.extras.extensions.ipython.reload_kedro") + mocker.patch("kedro.ipython._find_kedro_project") + mocker.patch("kedro.ipython.reload_kedro") ipython.magic(f"reload_kedro {args}") def test_line_magic_with_invalid_arguments(self, mocker, ipython): - mocker.patch("kedro.extras.extensions.ipython._find_kedro_project") - mocker.patch("kedro.extras.extensions.ipython.reload_kedro") + mocker.patch("kedro.ipython._find_kedro_project") + mocker.patch("kedro.ipython.reload_kedro") load_ipython_extension(ipython) with pytest.raises( From c06c8398880b4b430c8380075c66a9a3f24d055e Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Fri, 9 Sep 2022 11:03:30 +0100 Subject: [PATCH 2/5] Delete kedro/extras/extensions directory --- kedro/extras/extensions/__init__.py | 3 --- kedro/extras/extensions/ipython.py | 22 ---------------------- 2 files changed, 25 deletions(-) delete mode 100644 kedro/extras/extensions/__init__.py delete mode 100644 kedro/extras/extensions/ipython.py diff --git a/kedro/extras/extensions/__init__.py b/kedro/extras/extensions/__init__.py deleted file mode 100644 index 8cad7f44a1..0000000000 --- a/kedro/extras/extensions/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -This module contains an IPython extension. -""" diff --git a/kedro/extras/extensions/ipython.py b/kedro/extras/extensions/ipython.py deleted file mode 100644 index 2e38dc3772..0000000000 --- a/kedro/extras/extensions/ipython.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -This file and directory exists purely for backwards compatibility of the following: -%load_ext kedro.extras.extensions.ipython -from kedro.extras.extensions.ipython import reload_kedro - -Any modifications to the IPython extension should now be made in kedro/ipython/. -The Kedro IPython extension should always be loaded as %load_ext kedro.ipython. -Line magics such as reload_kedro should always be called as line magics rather than -importing the underlying Python functions. -""" -import warnings - -from ...ipython import ( # noqa # pylint: disable=unused-import - load_ipython_extension, - reload_kedro, -) - -warnings.warn( - "kedro.extras.extensions.ipython should be accessed only using the alias " - "kedro.ipython. The unaliased name will be removed in Kedro 0.19.0.", - DeprecationWarning, -) From 63578a2c120344871e10cb5764687bebaa5c2a0a Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Fri, 9 Sep 2022 11:04:26 +0100 Subject: [PATCH 3/5] Update test_ipython.py --- tests/ipython/test_ipython.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/ipython/test_ipython.py b/tests/ipython/test_ipython.py index e8c7178bf9..6363e27fac 100644 --- a/tests/ipython/test_ipython.py +++ b/tests/ipython/test_ipython.py @@ -190,8 +190,6 @@ class TestLoadIPythonExtension: def test_load_ipython_extension(self, ipython): ipython.magic("load_ext kedro.ipython") - def test_load_ipython_extension_old_location(self, ipython): - ipython.magic("load_ext kedro.ipython") def test_load_extension_missing_dependency(self, mocker): mocker.patch("kedro.ipython.reload_kedro", side_effect=ImportError) From f2afa4166b9b58c4e19a5f84922158d62cebb0d8 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Fri, 9 Sep 2022 16:39:35 +0100 Subject: [PATCH 4/5] Update RELEASE.md --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index af5a2b7e2d..a686e5b746 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -11,10 +11,10 @@ ### Other * Removed deprecated `kedro.extras.ColorHandler`. +* The Kedro IPython extension is no longer available as `%load_ext kedro.extras.extensions.ipython`; use `%load_ext kedro.ipython` instead. ## Migration guide from Kedro 0.18.* to 0.19.* - # Upcoming Release 0.18.3 ## Major features and improvements From 5ae719ed9a51e1e8dba9153c2f5aaf2b08080717 Mon Sep 17 00:00:00 2001 From: Antony Milne <49395058+AntonyMilneQB@users.noreply.github.com> Date: Fri, 9 Sep 2022 16:44:30 +0100 Subject: [PATCH 5/5] Update test_ipython.py --- tests/ipython/test_ipython.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ipython/test_ipython.py b/tests/ipython/test_ipython.py index 6363e27fac..fddd506f41 100644 --- a/tests/ipython/test_ipython.py +++ b/tests/ipython/test_ipython.py @@ -190,7 +190,6 @@ class TestLoadIPythonExtension: def test_load_ipython_extension(self, ipython): ipython.magic("load_ext kedro.ipython") - def test_load_extension_missing_dependency(self, mocker): mocker.patch("kedro.ipython.reload_kedro", side_effect=ImportError) mocker.patch(