From 8ff3c0d6dca87faed2bea706eb1f7a52be5e951b Mon Sep 17 00:00:00 2001 From: Nok Date: Thu, 18 Jan 2024 18:45:40 +0000 Subject: [PATCH 01/11] e2e draft Signed-off-by: Nok --- features/steps/cli_steps.py | 38 +++++++ features/steps/jupyter_magic.feature | 9 ++ features/steps/test_notebook/load_node.ipynb | 100 +++++++++++++++++++ features/steps/util.py | 27 +++++ 4 files changed, 174 insertions(+) create mode 100644 features/steps/jupyter_magic.feature create mode 100644 features/steps/test_notebook/load_node.ipynb diff --git a/features/steps/cli_steps.py b/features/steps/cli_steps.py index d3d4f9afd1..934dd33015 100644 --- a/features/steps/cli_steps.py +++ b/features/steps/cli_steps.py @@ -304,6 +304,10 @@ def create_project_with_starter(context, starter): env=context.env, cwd=context.temp_dir, ) + + # prevent telemetry from prompting for input during e2e tests + telemetry_file = context.root_project_dir / ".telemetry" + telemetry_file.write_text("consent: false", encoding="utf-8") assert res.returncode == OK_EXIT_CODE, res @@ -732,3 +736,37 @@ def add_micropkg_to_pyproject_toml(context: behave.runner.Context): ) with pyproject_toml_path.open(mode="a") as file: file.write(project_toml_str) + + +@given('I have executed the magic command "{command}"') +@when('I execute the magic command "{command}"') +def exec_magic_command(context, command): + """Execute Kedro target.""" + # split_command = command.split() + # cmd = [context.kedro] + split_command + curr_dir = Path(__file__).parent + run_notebook = context.root_project_dir / "notebooks" / "load_node.ipynb" + shutil.copyfile( + str(curr_dir / "test_notebook" / "load_node.ipynb"), str(run_notebook) + ) + + print(run_notebook.exists()) + util.execute_notebook(str(run_notebook)) + with open(run_notebook) as f: + context.notebook = f.read() + + +@then('the notebook should show "{pattern}"') +def match_pattern_in_notebook(context, pattern): + """This is a very simple function to match pattern in an exeucted notebook. + Noted that the proper way to read a notebook is using `nbformat.read` and parse + the output corresponding. For now the test is simple and it tries to match a + specific keyword which is unlikely to collide. If you need to match generic + keywords it is better to parse the output as a JSON file instead of one big + string. + """ + notebook = context.notebook + + if pattern in notebook: + return True + raise ValueError(f"{pattern} is not found.") diff --git a/features/steps/jupyter_magic.feature b/features/steps/jupyter_magic.feature new file mode 100644 index 0000000000..952814be13 --- /dev/null +++ b/features/steps/jupyter_magic.feature @@ -0,0 +1,9 @@ +Feature: Jupyter targets in new project + + Background: + Given I have prepared a config file + And I have run a non-interactive kedro new with starter "spaceflights-pandas" + + Scenario: Execute jupyter load_node magic + When I execute the magic command "load_node" + Then the notebook should show "split_data_node is not found" diff --git a/features/steps/test_notebook/load_node.ipynb b/features/steps/test_notebook/load_node.ipynb new file mode 100644 index 0000000000..48947e9f50 --- /dev/null +++ b/features/steps/test_notebook/load_node.ipynb @@ -0,0 +1,100 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pwd" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_node.ipynb\n" + ] + } + ], + "source": [ + "ls" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
[01/18/24 18:43:30] WARNING  Kedro extension was registered but couldn't find a Kedro project. Make  __init__.py:41\n",
+       "                             sure you run '%reload_kedro <project_root>'.                                          \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[2;36m[01/18/24 18:43:30]\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m Kedro extension was registered but couldn't find a Kedro project. Make \u001b]8;id=708242;file:///workspace/kedro/kedro/ipython/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=251392;file:///workspace/kedro/kedro/ipython/__init__.py#41\u001b\\\u001b[2m41\u001b[0m\u001b]8;;\u001b\\\n", + "\u001b[2;36m \u001b[0m sure you run \u001b[32m'%reload_kedro \u001b[0m\u001b[32m<\u001b[0m\u001b[32mproject_root\u001b[0m\u001b[32m>\u001b[0m\u001b[32m'\u001b[0m. \u001b[2m \u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%load_ext kedro.ipython\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "UsageError: Line magic function `%load_node` not found.\n" + ] + } + ], + "source": [ + "%load_node split_data_node" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.15" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/features/steps/util.py b/features/steps/util.py index 74031232f1..5f803a0fcf 100644 --- a/features/steps/util.py +++ b/features/steps/util.py @@ -9,6 +9,9 @@ from time import sleep, time from typing import Any, Callable, Iterator +import nbformat +from nbconvert.preprocessors import ExecutePreprocessor + @contextmanager def chdir(path: Path) -> Iterator: @@ -83,3 +86,27 @@ def parse_csv(text: str) -> list[str]: List of string tokens """ return re.findall(r"\"(.+?)\"\s*,?", text) + + +def execute_notebook(notebook_filename): + with open(notebook_filename) as f: + nb = nbformat.read(f, as_version=4) + + ep = ExecutePreprocessor(timeout=600, kernel_name="python3") + print(f"Executing Notebook {notebook_filename}") + try: + # Execute the notebook from the same directory + ep.preprocess( + nb, {"metadata": {"path": Path(notebook_filename).parent.resolve()}} + ) + except Exception as e: + print(e) + + with open(notebook_filename, "w", encoding="utf-8") as f: + nbformat.write(nb, f) + + with open( + "/Users/Nok_Lam_Chan/dev/kedro/nok_debug.ipynb", "w", encoding="utf-8" + ) as f: + nbformat.write(nb, f) + print(f"Executed Notebook saved at {notebook_filename}") From a7c52cfc7eb370b4a3b623e00525a4ad265078cb Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Tue, 9 Jul 2024 17:19:40 +0100 Subject: [PATCH 02/11] Fix name collision Signed-off-by: Nok Lam Chan --- kedro/ipython/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kedro/ipython/__init__.py b/kedro/ipython/__init__.py index ee487582e8..ce71eb8980 100644 --- a/kedro/ipython/__init__.py +++ b/kedro/ipython/__init__.py @@ -163,7 +163,8 @@ def _resolve_project_path( if path: project_path = Path(path).expanduser().resolve() else: - if local_namespace and "context" in local_namespace: + if local_namespace.get("context") and hasattr(local_namespace["context"], "project_path"): + # breakpoint() project_path = local_namespace["context"].project_path else: project_path = _find_kedro_project(Path.cwd()) @@ -177,7 +178,8 @@ def _resolve_project_path( if ( project_path and local_namespace - and "context" in local_namespace + and local_namespace.get("context") + and hasattr(local_namespace["context"], "project_path") # Avoid name collision and project_path != local_namespace["context"].project_path ): logger.info("Updating path to Kedro project: %s...", project_path) From bcd9a77ddcd6f460df7c49457eaefccbefbf57fa Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Tue, 9 Jul 2024 17:20:06 +0100 Subject: [PATCH 03/11] Add e2e test Signed-off-by: Nok Lam Chan --- features/load_node.feature | 9 ++++ features/steps/cli_steps.py | 53 ++++++++----------- features/steps/ipython_script.py | 9 ++++ features/steps/jupyter_magic.feature | 9 ---- .../pipelines/data_engineering/nodes.py | 1 + .../pipelines/data_engineering/pipeline.py | 1 + features/steps/util.py | 1 + 7 files changed, 44 insertions(+), 39 deletions(-) create mode 100644 features/load_node.feature create mode 100644 features/steps/ipython_script.py delete mode 100644 features/steps/jupyter_magic.feature diff --git a/features/load_node.feature b/features/load_node.feature new file mode 100644 index 0000000000..87b50a28c5 --- /dev/null +++ b/features/load_node.feature @@ -0,0 +1,9 @@ +Feature: load_node in new project + + Background: + Given I have prepared a config file + And I have run a non-interactive kedro new with starter "default" + + Scenario: Execute jupyter load_node magic + When I execute the load_node magic command + # Then the logs should show that load_node executed successfully diff --git a/features/steps/cli_steps.py b/features/steps/cli_steps.py index 2463958579..a4965c0fbc 100644 --- a/features/steps/cli_steps.py +++ b/features/steps/cli_steps.py @@ -564,6 +564,15 @@ def check_correct_nodes_run(context, node): f"{expected_log_line},\nbut got {stdout}" ) +@then('the logs should show that load_node executed successfully') +def check_load_node_run(context): + expected_log_line = "load_node executed successfully" + stdout = context.result.stdout + clean_logs = util.clean_up_log(stdout) + assert expected_log_line in clean_logs, ( + "Expected the following message segment to be printed on stdout: " + f"{expected_log_line},\nbut got {stdout}" + ) @then("I should get a successful exit code") def check_status_code(context): @@ -728,38 +737,22 @@ def add_micropkg_to_pyproject_toml(context: behave.runner.Context): file.write(project_toml_str) -@given('I have executed the magic command "{command}"') -@when('I execute the magic command "{command}"') -def exec_magic_command(context, command): +@given("I have executed the load_node magic command") +@when("I execute the load_node magic command") +def exec_magic_command(context): """Execute Kedro target.""" - # split_command = command.split() - # cmd = [context.kedro] + split_command - curr_dir = Path(__file__).parent - run_notebook = context.root_project_dir / "notebooks" / "load_node.ipynb" - shutil.copyfile( - str(curr_dir / "test_notebook" / "load_node.ipynb"), str(run_notebook) - ) - - print(run_notebook.exists()) - util.execute_notebook(str(run_notebook)) - with open(run_notebook) as f: - context.notebook = f.read() - - -@then('the notebook should show "{pattern}"') -def match_pattern_in_notebook(context, pattern): - """This is a very simple function to match pattern in an exeucted notebook. - Noted that the proper way to read a notebook is using `nbformat.read` and parse - the output corresponding. For now the test is simple and it tries to match a - specific keyword which is unlikely to collide. If you need to match generic - keywords it is better to parse the output as a JSON file instead of one big - string. - """ - notebook = context.notebook + print("DEBUG***************") + project_path = context.root_project_dir + # context.result = run(context.python, env=context.env, cwd=str(context.root_project_dir)) + + # ip = get_ipython() + # ip.run_line_magic("load_ext", "kedro.ipython") + # ip.run_line_magic("reload_kedro", str(project_path.absolute())) + # ip.run_line_magic("load_node", "split_data_node") + # # ip.rl_next_input is what you see in the terminal input + # ip.run_cell(ip.rl_next_input) + # There is a print statement inserted in the function so we can assert from the log - if pattern in notebook: - return True - raise ValueError(f"{pattern} is not found.") @given('I have changed the current working directory to "{dir}"') def change_dir(context, dir): diff --git a/features/steps/ipython_script.py b/features/steps/ipython_script.py new file mode 100644 index 0000000000..5bb28fb61f --- /dev/null +++ b/features/steps/ipython_script.py @@ -0,0 +1,9 @@ + +from IPython.testing.globalipapp import get_ipython +ip = get_ipython() +ip.run_line_magic("load_ext", "kedro.ipython") +# Assume cwd is project root +ip.run_line_magic("reload_kedro", "") +ip.run_line_magic("load_node", "split_data_node") +# ip.rl_next_input is what you see in the terminal input +ip.run_cell(ip.rl_next_input) \ No newline at end of file diff --git a/features/steps/jupyter_magic.feature b/features/steps/jupyter_magic.feature deleted file mode 100644 index 952814be13..0000000000 --- a/features/steps/jupyter_magic.feature +++ /dev/null @@ -1,9 +0,0 @@ -Feature: Jupyter targets in new project - - Background: - Given I have prepared a config file - And I have run a non-interactive kedro new with starter "spaceflights-pandas" - - Scenario: Execute jupyter load_node magic - When I execute the magic command "load_node" - Then the notebook should show "split_data_node is not found" diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_engineering/nodes.py b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_engineering/nodes.py index 3f9c8e1337..024ea394ed 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_engineering/nodes.py +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_engineering/nodes.py @@ -17,6 +17,7 @@ def split_data(data: pd.DataFrame, example_test_data_ratio: float) -> dict[str, The data and the parameters will be loaded and provided to your function automatically when the pipeline is executed and it is time to run this node. """ + print("load_node executed successfully") data.columns = [ "sepal_length", "sepal_width", diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_engineering/pipeline.py b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_engineering/pipeline.py index dee78699fd..18aba801f7 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_engineering/pipeline.py +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/{{ cookiecutter.python_package }}/pipelines/data_engineering/pipeline.py @@ -21,6 +21,7 @@ def create_pipeline(**kwargs): test_x="example_test_x", test_y="example_test_y", ), + name="split_data_node" ) ] ) diff --git a/features/steps/util.py b/features/steps/util.py index 056c071e81..5f99321967 100644 --- a/features/steps/util.py +++ b/features/steps/util.py @@ -111,6 +111,7 @@ def execute_notebook(notebook_filename): nbformat.write(nb, f) print(f"Executed Notebook saved at {notebook_filename}") + def clean_up_log(stdout: str) -> str: """ Cleans up log output by removing duplicate lines, extra whitespaces, From 1674e4c9d637f15c7dbce10fbb0c85e0f6fff8dd Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Tue, 9 Jul 2024 17:21:33 +0100 Subject: [PATCH 04/11] protect script under __name__ Signed-off-by: Nok Lam Chan --- features/steps/ipython_script.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/features/steps/ipython_script.py b/features/steps/ipython_script.py index 5bb28fb61f..da67c4844e 100644 --- a/features/steps/ipython_script.py +++ b/features/steps/ipython_script.py @@ -1,9 +1,9 @@ - -from IPython.testing.globalipapp import get_ipython -ip = get_ipython() -ip.run_line_magic("load_ext", "kedro.ipython") -# Assume cwd is project root -ip.run_line_magic("reload_kedro", "") -ip.run_line_magic("load_node", "split_data_node") -# ip.rl_next_input is what you see in the terminal input -ip.run_cell(ip.rl_next_input) \ No newline at end of file +if __name__ == "__main__": + from IPython.testing.globalipapp import get_ipython + ip = get_ipython() + ip.run_line_magic("load_ext", "kedro.ipython") + # Assume cwd is project root + ip.run_line_magic("reload_kedro", "") + ip.run_line_magic("load_node", "split_data_node") + # ip.rl_next_input is what you see in the terminal input + ip.run_cell(ip.rl_next_input) \ No newline at end of file From 6abb8744f009715275acd41d36f1ac891571f18c Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Tue, 9 Jul 2024 17:30:12 +0100 Subject: [PATCH 05/11] Fix test Signed-off-by: Nok Lam Chan --- features/load_node.feature | 2 +- features/steps/cli_steps.py | 19 ++++++++----------- .../ipython_script.py | 3 ++- kedro/ipython/__init__.py | 6 ++++-- 4 files changed, 15 insertions(+), 15 deletions(-) rename features/steps/{ => test_starter/{{ cookiecutter.repo_name }}}/ipython_script.py (84%) diff --git a/features/load_node.feature b/features/load_node.feature index 87b50a28c5..5e28c432da 100644 --- a/features/load_node.feature +++ b/features/load_node.feature @@ -6,4 +6,4 @@ Feature: load_node in new project Scenario: Execute jupyter load_node magic When I execute the load_node magic command - # Then the logs should show that load_node executed successfully + Then the logs should show that load_node executed successfully diff --git a/features/steps/cli_steps.py b/features/steps/cli_steps.py index a4965c0fbc..139467f42a 100644 --- a/features/steps/cli_steps.py +++ b/features/steps/cli_steps.py @@ -564,7 +564,8 @@ def check_correct_nodes_run(context, node): f"{expected_log_line},\nbut got {stdout}" ) -@then('the logs should show that load_node executed successfully') + +@then("the logs should show that load_node executed successfully") def check_load_node_run(context): expected_log_line = "load_node executed successfully" stdout = context.result.stdout @@ -574,6 +575,7 @@ def check_load_node_run(context): f"{expected_log_line},\nbut got {stdout}" ) + @then("I should get a successful exit code") def check_status_code(context): if context.result.returncode != OK_EXIT_CODE: @@ -741,17 +743,12 @@ def add_micropkg_to_pyproject_toml(context: behave.runner.Context): @when("I execute the load_node magic command") def exec_magic_command(context): """Execute Kedro target.""" - print("DEBUG***************") project_path = context.root_project_dir - # context.result = run(context.python, env=context.env, cwd=str(context.root_project_dir)) - - # ip = get_ipython() - # ip.run_line_magic("load_ext", "kedro.ipython") - # ip.run_line_magic("reload_kedro", str(project_path.absolute())) - # ip.run_line_magic("load_node", "split_data_node") - # # ip.rl_next_input is what you see in the terminal input - # ip.run_cell(ip.rl_next_input) - # There is a print statement inserted in the function so we can assert from the log + cmd = [context.python, "ipython_script.py"] + # breakpoint() + context.result = run( + cmd, env=context.env, cwd=str(context.root_project_dir), print_output=True + ) @given('I have changed the current working directory to "{dir}"') diff --git a/features/steps/ipython_script.py b/features/steps/test_starter/{{ cookiecutter.repo_name }}/ipython_script.py similarity index 84% rename from features/steps/ipython_script.py rename to features/steps/test_starter/{{ cookiecutter.repo_name }}/ipython_script.py index da67c4844e..558e76fad5 100644 --- a/features/steps/ipython_script.py +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/ipython_script.py @@ -1,9 +1,10 @@ if __name__ == "__main__": from IPython.testing.globalipapp import get_ipython + print("!!!****!!!****") ip = get_ipython() ip.run_line_magic("load_ext", "kedro.ipython") # Assume cwd is project root ip.run_line_magic("reload_kedro", "") ip.run_line_magic("load_node", "split_data_node") # ip.rl_next_input is what you see in the terminal input - ip.run_cell(ip.rl_next_input) \ No newline at end of file + ip.run_cell(ip.rl_next_input) diff --git a/kedro/ipython/__init__.py b/kedro/ipython/__init__.py index ce71eb8980..e19d6fafa3 100644 --- a/kedro/ipython/__init__.py +++ b/kedro/ipython/__init__.py @@ -163,7 +163,9 @@ def _resolve_project_path( if path: project_path = Path(path).expanduser().resolve() else: - if local_namespace.get("context") and hasattr(local_namespace["context"], "project_path"): + if local_namespace.get("context") and hasattr( + local_namespace["context"], "project_path" + ): # breakpoint() project_path = local_namespace["context"].project_path else: @@ -179,7 +181,7 @@ def _resolve_project_path( project_path and local_namespace and local_namespace.get("context") - and hasattr(local_namespace["context"], "project_path") # Avoid name collision + and hasattr(local_namespace["context"], "project_path") # Avoid name collision and project_path != local_namespace["context"].project_path ): logger.info("Updating path to Kedro project: %s...", project_path) From be53f7724447ef5de03add7c1c98e8e788e25120 Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Tue, 9 Jul 2024 17:31:03 +0100 Subject: [PATCH 06/11] remove test notebook Signed-off-by: Nok Lam Chan --- features/steps/test_notebook/load_node.ipynb | 100 ------------------- 1 file changed, 100 deletions(-) delete mode 100644 features/steps/test_notebook/load_node.ipynb diff --git a/features/steps/test_notebook/load_node.ipynb b/features/steps/test_notebook/load_node.ipynb deleted file mode 100644 index 48947e9f50..0000000000 --- a/features/steps/test_notebook/load_node.ipynb +++ /dev/null @@ -1,100 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pwd" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "load_node.ipynb\n" - ] - } - ], - "source": [ - "ls" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
[01/18/24 18:43:30] WARNING  Kedro extension was registered but couldn't find a Kedro project. Make  __init__.py:41\n",
-       "                             sure you run '%reload_kedro <project_root>'.                                          \n",
-       "
\n" - ], - "text/plain": [ - "\u001b[2;36m[01/18/24 18:43:30]\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m Kedro extension was registered but couldn't find a Kedro project. Make \u001b]8;id=708242;file:///workspace/kedro/kedro/ipython/__init__.py\u001b\\\u001b[2m__init__.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=251392;file:///workspace/kedro/kedro/ipython/__init__.py#41\u001b\\\u001b[2m41\u001b[0m\u001b]8;;\u001b\\\n", - "\u001b[2;36m \u001b[0m sure you run \u001b[32m'%reload_kedro \u001b[0m\u001b[32m<\u001b[0m\u001b[32mproject_root\u001b[0m\u001b[32m>\u001b[0m\u001b[32m'\u001b[0m. \u001b[2m \u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "%load_ext kedro.ipython\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "UsageError: Line magic function `%load_node` not found.\n" - ] - } - ], - "source": [ - "%load_node split_data_node" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.15" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 3bfaeb338c33260f3b3f69088904cb623c985fc0 Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Tue, 9 Jul 2024 17:34:00 +0100 Subject: [PATCH 07/11] remove debug statement Signed-off-by: Nok Lam Chan --- features/steps/cli_steps.py | 2 -- kedro/ipython/__init__.py | 1 - 2 files changed, 3 deletions(-) diff --git a/features/steps/cli_steps.py b/features/steps/cli_steps.py index 139467f42a..98dfd42a49 100644 --- a/features/steps/cli_steps.py +++ b/features/steps/cli_steps.py @@ -743,9 +743,7 @@ def add_micropkg_to_pyproject_toml(context: behave.runner.Context): @when("I execute the load_node magic command") def exec_magic_command(context): """Execute Kedro target.""" - project_path = context.root_project_dir cmd = [context.python, "ipython_script.py"] - # breakpoint() context.result = run( cmd, env=context.env, cwd=str(context.root_project_dir), print_output=True ) diff --git a/kedro/ipython/__init__.py b/kedro/ipython/__init__.py index e19d6fafa3..d40aa2749b 100644 --- a/kedro/ipython/__init__.py +++ b/kedro/ipython/__init__.py @@ -166,7 +166,6 @@ def _resolve_project_path( if local_namespace.get("context") and hasattr( local_namespace["context"], "project_path" ): - # breakpoint() project_path = local_namespace["context"].project_path else: project_path = _find_kedro_project(Path.cwd()) From 77d59bef238b24c55600bd08b995cdb98f658789 Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Tue, 9 Jul 2024 17:35:33 +0100 Subject: [PATCH 08/11] revert changes Signed-off-by: Nok Lam Chan --- features/steps/cli_steps.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/features/steps/cli_steps.py b/features/steps/cli_steps.py index 98dfd42a49..3c8a5da056 100644 --- a/features/steps/cli_steps.py +++ b/features/steps/cli_steps.py @@ -303,9 +303,6 @@ def create_project_with_starter(context, starter): cwd=context.temp_dir, ) - # prevent telemetry from prompting for input during e2e tests - telemetry_file = context.root_project_dir / ".telemetry" - telemetry_file.write_text("consent: false", encoding="utf-8") assert res.returncode == OK_EXIT_CODE, res From 70ae03f26690b4486f9b0e5bee40214a472f40b9 Mon Sep 17 00:00:00 2001 From: Nok Lam Chan Date: Tue, 9 Jul 2024 17:47:10 +0100 Subject: [PATCH 09/11] fix test Signed-off-by: Nok Lam Chan --- kedro/ipython/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kedro/ipython/__init__.py b/kedro/ipython/__init__.py index d40aa2749b..13b4b7a20a 100644 --- a/kedro/ipython/__init__.py +++ b/kedro/ipython/__init__.py @@ -163,8 +163,10 @@ def _resolve_project_path( if path: project_path = Path(path).expanduser().resolve() else: - if local_namespace.get("context") and hasattr( - local_namespace["context"], "project_path" + if ( + local_namespace + and local_namespace.get("context") + and hasattr(local_namespace["context"], "project_path") ): project_path = local_namespace["context"].project_path else: From 08632b2988bc7c291b6168487a63db5c802f16b6 Mon Sep 17 00:00:00 2001 From: Nok Date: Wed, 10 Jul 2024 11:34:06 +0000 Subject: [PATCH 10/11] clean up notebook e2etest Signed-off-by: Nok --- features/load_node.feature | 2 +- .../ipython_script.py | 1 - features/steps/util.py | 27 ------------------- 3 files changed, 1 insertion(+), 29 deletions(-) diff --git a/features/load_node.feature b/features/load_node.feature index 5e28c432da..fbc5a65a07 100644 --- a/features/load_node.feature +++ b/features/load_node.feature @@ -4,6 +4,6 @@ Feature: load_node in new project Given I have prepared a config file And I have run a non-interactive kedro new with starter "default" - Scenario: Execute jupyter load_node magic + Scenario: Execute ipython load_node magic When I execute the load_node magic command Then the logs should show that load_node executed successfully diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/ipython_script.py b/features/steps/test_starter/{{ cookiecutter.repo_name }}/ipython_script.py index 558e76fad5..87a7c9c20e 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/ipython_script.py +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/ipython_script.py @@ -1,6 +1,5 @@ if __name__ == "__main__": from IPython.testing.globalipapp import get_ipython - print("!!!****!!!****") ip = get_ipython() ip.run_line_magic("load_ext", "kedro.ipython") # Assume cwd is project root diff --git a/features/steps/util.py b/features/steps/util.py index 5f99321967..f9c7b2c4e2 100644 --- a/features/steps/util.py +++ b/features/steps/util.py @@ -9,9 +9,6 @@ from time import sleep, time from typing import Any, Callable, Iterator -import nbformat -from nbconvert.preprocessors import ExecutePreprocessor - @contextmanager def chdir(path: Path) -> Iterator: @@ -88,30 +85,6 @@ def parse_csv(text: str) -> list[str]: return re.findall(r"\"(.+?)\"\s*,?", text) -def execute_notebook(notebook_filename): - with open(notebook_filename) as f: - nb = nbformat.read(f, as_version=4) - - ep = ExecutePreprocessor(timeout=600, kernel_name="python3") - print(f"Executing Notebook {notebook_filename}") - try: - # Execute the notebook from the same directory - ep.preprocess( - nb, {"metadata": {"path": Path(notebook_filename).parent.resolve()}} - ) - except Exception as e: - print(e) - - with open(notebook_filename, "w", encoding="utf-8") as f: - nbformat.write(nb, f) - - with open( - "/Users/Nok_Lam_Chan/dev/kedro/nok_debug.ipynb", "w", encoding="utf-8" - ) as f: - nbformat.write(nb, f) - print(f"Executed Notebook saved at {notebook_filename}") - - def clean_up_log(stdout: str) -> str: """ Cleans up log output by removing duplicate lines, extra whitespaces, From 262d01b0458cd19ca483fb14fcef8261b8d04ce3 Mon Sep 17 00:00:00 2001 From: Nok Date: Wed, 10 Jul 2024 11:44:28 +0000 Subject: [PATCH 11/11] fix test after node rename Signed-off-by: Nok --- tests/framework/cli/test_registry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/framework/cli/test_registry.py b/tests/framework/cli/test_registry.py index 3c84efa999..fdb43416f4 100644 --- a/tests/framework/cli/test_registry.py +++ b/tests/framework/cli/test_registry.py @@ -10,13 +10,13 @@ def yaml_dump_mock(mocker): @pytest.fixture def pipelines_dict(): pipelines = { - "data_engineering": ["split_data (split_data)"], + "data_engineering": ["split_data_node (split_data)"], "data_science": [ "train_model (train_model)", "predict (predict)", "report_accuracy (report_accuracy)", ], - "data_processing": ["data_processing.split_data (split_data)"], + "data_processing": ["data_processing.split_data_node (split_data)"], } pipelines["__default__"] = pipelines["data_engineering"] + pipelines["data_science"] return pipelines