diff --git a/docs/source/development/commands_reference.md b/docs/source/development/commands_reference.md index 16c2e4a1cd..2b2d64c69a 100644 --- a/docs/source/development/commands_reference.md +++ b/docs/source/development/commands_reference.md @@ -57,18 +57,13 @@ Here is a list of Kedro CLI commands, as a shortcut to the descriptions below. P * [`kedro new`](#create-a-new-kedro-project) * Project-specific Kedro commands - * [`kedro activate-nbstripout`](#strip-output-cells)(deprecated from version 0.19.0) - * [`kedro build-docs`](#build-the-project-documentation) (deprecated from version 0.19.0) - * [`kedro build-reqs`](#build-the-projects-dependency-tree) (deprecated from version 0.19.0) * [`kedro catalog list`](#list-datasets-per-pipeline-per-type) * [`kedro catalog resolve`](#resolve-dataset-factories-in-the-catalog) * [`kedro catalog rank`](#rank-dataset-factories-in-the-catalog) * [`kedro catalog create`](#create-a-data-catalog-yaml-configuration-file) * [`kedro ipython`](#notebooks) - * [`kedro jupyter convert`](#copy-tagged-cells) (deprecated from version 0.19.0) * [`kedro jupyter lab`](#notebooks) * [`kedro jupyter notebook`](#notebooks) - * [`kedro lint`](#lint-your-project) (deprecated from version 0.19.0) * [`kedro micropkg package `](#package-a-micro-package) * [`kedro micropkg pull `](#pull-a-micro-package) * [`kedro package`](#deploy-the-project) @@ -77,7 +72,6 @@ Here is a list of Kedro CLI commands, as a shortcut to the descriptions below. P * [`kedro registry describe `](#describe-a-registered-pipeline) * [`kedro registry list`](#list-all-registered-pipelines-in-your-project) * [`kedro run`](#run-the-project) - * [`kedro test`](#test-your-project) (deprecated from version 0.19.0) ## Global Kedro commands @@ -278,21 +272,6 @@ def run( ### Project setup -#### Build the project's dependency tree - -```{note} -_This command will be deprecated from Kedro version 0.19.0._ -``` -```bash -kedro build-reqs -``` - -This command runs [`pip-compile`](https://github.com/jazzband/pip-tools#example-usage-for-pip-compile) on the project's `src/requirements.txt` file and will create `src/requirements.lock` with the compiled requirements. - -`kedro build-reqs` has two optional arguments to specify which file to compile the requirements from and where to save the compiled requirements to. These arguments are `--input-file` and `--output-file` respectively. - -`kedro build-reqs` also accepts and passes through CLI options accepted by `pip-compile`. For example, `kedro build-reqs --generate-hashes` will call `pip-compile --output-file=src/requirements.lock --generate-hashes src/requirements.txt`. - #### Install all package dependencies The following runs [`pip`](https://github.com/pypa/pip) to install all package dependencies specified in `src/requirements.txt`: @@ -381,44 +360,6 @@ The above command will take the bundled `.tar.gz` file and do the following: ### Project quality -#### Build the project documentation - -```{note} -_This command will be deprecated from Kedro version 0.19.0._ -``` - -```bash -kedro build-docs -``` - -The `build-docs` command builds [project documentation](../tutorial/package_a_project.md#add-documentation-to-a-kedro-project) using the [Sphinx](https://www.sphinx-doc.org) framework. To further customise your documentation, please refer to `docs/source/conf.py` and the [Sphinx documentation](http://www.sphinx-doc.org/en/master/usage/configuration.html). - - -#### Lint your project - -```{note} -_This command will be deprecated from Kedro version 0.19.0._. We still recommend to (../development/linting.md) and you can find more help here -``` - -```bash -kedro lint -``` - -Your project is linted with [`black`](https://github.com/psf/black), [`flake8`](https://github.com/PyCQA/flake8) and [`isort`](https://github.com/PyCQA/isort). - - -#### Test your project - -```{note} -_This command will be deprecated from Kedro version 0.19.0._ -``` - -The following runs all `pytest` unit tests found in `src/tests`, including coverage (see the file `.coveragerc`): - -```bash -kedro test -``` - ### Project development #### Modular pipelines @@ -544,29 +485,3 @@ The [Kedro IPython extension](../notebooks_and_ipython/kedro_and_notebooks.md#a- * `session` (type `KedroSession`): [Kedro session](../kedro_project_setup/session.md) that orchestrates a pipeline run To reload these variables (e.g. if you updated `catalog.yml`) use the `%reload_kedro` line magic, which can also be used to see the error message if any of the variables above are undefined. - -##### Copy tagged cells - -```{note} -_This command will be deprecated from Kedro version 0.19.0._ -``` - -To copy the code from [cells tagged](https://jupyter-notebook.readthedocs.io/en/stable/changelog.html#cell-tags) with a `node` tag into Python files under `src//nodes/` in a Kedro project: - -```bash -kedro jupyter convert --all -``` - -##### Strip output cells - -```{note} -_This command will be deprecated from Kedro version 0.19.0._ -``` - -Output cells of Jupyter Notebook should not be tracked by git, especially if they contain sensitive information. To strip them out: - -```bash -kedro activate-nbstripout -``` - -This command adds a `git hook` which clears all notebook output cells before committing anything to `git`. It needs to run only once per local repository. diff --git a/docs/source/development/set_up_pycharm.md b/docs/source/development/set_up_pycharm.md index 28d936bf22..6496cb3ad3 100644 --- a/docs/source/development/set_up_pycharm.md +++ b/docs/source/development/set_up_pycharm.md @@ -78,7 +78,7 @@ You may also select **Run** from the toolbar and execute from there.
![](../meta/images/pycharm_conf_run_dropdown.png) -For other `kedro` commands, follow same steps but replace `run` in the `Parameters` field with the other commands that are to be used (e.g., `test`, `package`, `build-docs` etc.). +For other `kedro` commands, follow same steps but replace `run` in the `Parameters` field with the other commands that are to be used (e.g., `jupyter`, `package`, `registry` etc.). ## Debugging diff --git a/docs/source/faq/faq.md b/docs/source/faq/faq.md index 69115f5e30..210320dbd3 100644 --- a/docs/source/faq/faq.md +++ b/docs/source/faq/faq.md @@ -8,8 +8,6 @@ This is a growing set of technical FAQs. The [product FAQs on the Kedro website] ## Working with Jupyter -* [How can I convert functions from Jupyter Notebooks into Kedro nodes](../notebooks_and_ipython/kedro_and_notebooks.md#convert-functions-from-jupyter-notebooks-into-kedro-nodes)? - * [How do I connect a Kedro project kernel to other Jupyter clients like JupyterLab](../notebooks_and_ipython/kedro_and_notebooks.md#ipython-jupyterlab-and-other-jupyter-clients)? ## Kedro project development diff --git a/docs/source/get_started/kedro_concepts.md b/docs/source/get_started/kedro_concepts.md index 67f9bf84a9..88995e5077 100644 --- a/docs/source/get_started/kedro_concepts.md +++ b/docs/source/get_started/kedro_concepts.md @@ -104,4 +104,4 @@ The `data` folder contains multiple subfolders to store project data. We recomme ### `src` -This subfolder contains the project's source code in one subfolder and another folder that you can use to add unit tests for your project. Projects are preconfigured to run tests using `pytest` when you call `kedro test` from the project's root directory. +This subfolder contains the project's source code. diff --git a/docs/source/notebooks_and_ipython/kedro_and_notebooks.md b/docs/source/notebooks_and_ipython/kedro_and_notebooks.md index 0cd509b32c..133e4a13b9 100644 --- a/docs/source/notebooks_and_ipython/kedro_and_notebooks.md +++ b/docs/source/notebooks_and_ipython/kedro_and_notebooks.md @@ -192,32 +192,6 @@ For more details, run `%reload_kedro?`. If you have [Kedro-Viz](https://github.com/kedro-org/kedro-viz) installed for the project you can display an interactive visualisation of your pipeline directly in your Notebook using the [line magic](https://ipython.readthedocs.io/en/stable/interactive/magics.html) `%run_viz`. - -## Convert functions from Jupyter Notebooks into Kedro nodes - -If you are writing experimental code in your Notebook and later want to convert functions you've written to Kedro nodes, you can do this using tags. - -Say you have the following code in your Notebook: - -```ipython -def some_action(): - print("This function came from `notebooks/my_notebook.ipynb`") -``` - -1. Enable tags toolbar: `View` menu -> `Cell Toolbar` -> `Tags` -![Enable the tags toolbar graphic](../meta/images/jupyter_notebook_workflow_activating_tags.png) - -2. Add the `node` tag to the cell containing your function -![Add the node tag graphic](../meta/images/jupyter_notebook_workflow_tagging_nodes.png) - -```{note} -The Notebook can contain multiple functions tagged as `node`, each of them will be exported into the resulting Python file -``` - -3. Save your Jupyter Notebook to `notebooks/my_notebook.ipynb` -4. From your terminal, run `kedro jupyter convert notebooks/my_notebook.ipynb` from the Kedro project directory. The output is a Python file `src//nodes/my_notebook.py` containing the `some_action` function definition -5. The `some_action` function can now be used in your Kedro pipelines - ## Useful to know... Each Kedro project has its own Jupyter kernel so you can switch between multiple Kedro projects from a single Jupyter instance simply by selecting the appropriate kernel. diff --git a/docs/source/tutorial/package_a_project.md b/docs/source/tutorial/package_a_project.md index 010aed2e6c..b0e22fead1 100644 --- a/docs/source/tutorial/package_a_project.md +++ b/docs/source/tutorial/package_a_project.md @@ -15,13 +15,6 @@ pip install sphinx ``` ### Set up the Sphinx project files - -```{warning} -Currently, Kedro projects are created with a `docs/source` subdirectory, which gets pre-populated with two Sphinx configuration files (`conf.py`, and `index.rst`), needed by the `kedro build-docs` command. This command is deprecated; it will be removed in Kedro version 0.19, along with those dummy files. - -Before proceeding with these instructions, back up the contents of `docs/source/index.rst` and remove both `docs/source/conf.py` and `docs/source/index.rst`. -``` - First, run the following command: ```bash diff --git a/features/activate_nbstripout.feature b/features/activate_nbstripout.feature deleted file mode 100644 index fa221417d4..0000000000 --- a/features/activate_nbstripout.feature +++ /dev/null @@ -1,17 +0,0 @@ -Feature: Activate_nbstripout target in new project - - Scenario: Check nbstripout git post commit hook functionality - Given I have prepared a config file - And I have run a non-interactive kedro new with starter "default" - And I have added a test jupyter notebook - And I have initialized a git repository - And I have added the project directory to staging - And I have committed changes to git - And I have executed the kedro command "activate-nbstripout" - When I execute the test jupyter notebook and save changes - And I add the project directory to staging - And I commit changes to git - And I remove the notebooks directory - And I perform a hard git reset to restore the project to last commit - Then there should be an additional cell in the jupyter notebook - And the output should be empty in all the cells in the jupyter notebook diff --git a/features/build_docs.feature b/features/build_docs.feature deleted file mode 100644 index c9f9307ef1..0000000000 --- a/features/build_docs.feature +++ /dev/null @@ -1,11 +0,0 @@ -Feature: build-docs target in new project - - @fresh_venv - Scenario: Execute build-docs target - Given I have prepared a config file - And I have run a non-interactive kedro new with starter "default" - And I have updated kedro requirements - And I have installed the project dependencies - When I execute the kedro command "build-docs" - Then I should get a successful exit code - And docs should be generated diff --git a/features/build_reqs.feature b/features/build_reqs.feature deleted file mode 100644 index 05bf551961..0000000000 --- a/features/build_reqs.feature +++ /dev/null @@ -1,13 +0,0 @@ -Feature: build-reqs target in new project - - @fresh_venv - Scenario: Execute build-reqs target - Given I have prepared a config file - And I have run a non-interactive kedro new with starter "default" - And I have updated kedro requirements - And I have executed the kedro command "build-reqs --resolver=backtracking" - When I add scrapy>=1.7.3 to the requirements - And I execute the kedro command "build-reqs --resolver=backtracking" - Then I should get a successful exit code - And requirements should be generated - And scrapy should be in the requirements diff --git a/features/jupyter.feature b/features/jupyter.feature index 65b5173442..2769ff8b20 100644 --- a/features/jupyter.feature +++ b/features/jupyter.feature @@ -17,11 +17,3 @@ Feature: Jupyter targets in new project When I execute the kedro jupyter command "lab --no-browser" Then I wait for the jupyter webserver to run for up to "120" seconds Then Jupyter Lab should run on port 8888 - - Scenario: Execute node convert into Python files - Given I have added a test jupyter notebook - When I execute the test jupyter notebook and save changes - And I execute the kedro jupyter command "convert --all" - And Wait until the process is finished for up to "120" seconds - Then I should get a successful exit code - And Code cell with node tag should be converted into kedro node diff --git a/features/package.feature b/features/package.feature index 663ea87c49..e840a1b9d7 100644 --- a/features/package.feature +++ b/features/package.feature @@ -12,14 +12,3 @@ Feature: Package target in new project When I install the project's python package And I execute the installed project package Then I should get a successful exit code - - @fresh_venv - Scenario: Install package after running kedro build-reqs - Given I have updated kedro requirements - When I execute the kedro command "build-reqs --resolver=backtracking" - Then I should get a successful exit code - When I execute the kedro command "package" - Then I should get a successful exit code - When I install the project's python package - And I execute the installed project package - Then I should get a successful exit code diff --git a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/tests/test_run.py b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/tests/test_run.py index ee11dea542..f9355de19b 100644 --- a/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/tests/test_run.py +++ b/features/steps/test_starter/{{ cookiecutter.repo_name }}/src/tests/test_run.py @@ -5,7 +5,7 @@ project's structure, and in files named test_*.py. They are simply functions named ``test_*`` which test a unit of logic. -To run the tests, run ``kedro test`` from the project root directory. +To run the tests, run ``pytest`` from the project root directory. """ from pathlib import Path diff --git a/features/test.feature b/features/test.feature deleted file mode 100644 index 0d42f336e6..0000000000 --- a/features/test.feature +++ /dev/null @@ -1,13 +0,0 @@ -Feature: Test target in new project - - Background: - Given I have prepared a config file - And I have run a non-interactive kedro new with starter "default" - - Scenario: Execute successful test in new project - When I execute the kedro command "test" - Then I should get a successful exit code - - Scenario: Execute successful lint in new project - When I execute the kedro command "lint --check-only" - Then I should get a successful exit code diff --git a/kedro/framework/cli/jupyter.py b/kedro/framework/cli/jupyter.py index e7cfbc166e..d2facef34b 100644 --- a/kedro/framework/cli/jupyter.py +++ b/kedro/framework/cli/jupyter.py @@ -6,20 +6,13 @@ import json import os import shutil -import sys -from collections import Counter -from glob import iglob from pathlib import Path -from typing import Any -from warnings import warn import click -from click import secho from kedro.framework.cli.utils import ( KedroCliError, _check_module_importable, - command_with_verbosity, env_option, forward_command, python_call, @@ -190,119 +183,3 @@ def _create_kernel(kernel_name: str, display_name: str) -> str: f"Cannot setup kedro kernel for Jupyter.\nError: {exc}" ) from exc return kernel_path - - -@command_with_verbosity(jupyter, "convert") -@click.option("--all", "-a", "all_flag", is_flag=True, help=CONVERT_ALL_HELP) -@click.option("-y", "overwrite_flag", is_flag=True, help=OVERWRITE_HELP) -@click.argument( - "filepath", - type=click.Path(exists=True, dir_okay=False, resolve_path=True), - required=False, - nargs=-1, -) -@env_option -@click.pass_obj # this will pass the metadata as first argument -def convert_notebook( - metadata: ProjectMetadata, all_flag, overwrite_flag, filepath, env, **kwargs -): # noqa: unused-argument, too-many-locals - """Convert selected or all notebooks found in a Kedro project - to Kedro code, by exporting code from the appropriately-tagged cells: - Cells tagged as `node` will be copied over to a Python file matching - the name of the notebook, under `//nodes`. - *Note*: Make sure your notebooks have unique names! - FILEPATH: Path(s) to exact notebook file(s) to be converted. Both - relative and absolute paths are accepted. - Should not be provided if --all flag is already present. (DEPRECATED) - """ - - deprecation_message = ( - "DeprecationWarning: Command 'kedro jupyter convert' is deprecated and " - "will not be available from Kedro 0.19.0." - ) - click.secho(deprecation_message, fg="red") - - project_path = metadata.project_path - source_path = metadata.source_dir - package_name = metadata.package_name - - if not filepath and not all_flag: - secho( - "Please specify a notebook filepath " - "or add '--all' to convert all notebooks." - ) - sys.exit(1) - - if all_flag: - # pathlib glob does not ignore hidden directories, - # whereas Python glob does, which is more useful in - # ensuring checkpoints will not be included - pattern = project_path / "**" / "*.ipynb" - notebooks = sorted(Path(p) for p in iglob(str(pattern), recursive=True)) - else: - notebooks = [Path(f) for f in filepath] - - counter = Counter(n.stem for n in notebooks) - non_unique_names = [name for name, counts in counter.items() if counts > 1] - if non_unique_names: - names = ", ".join(non_unique_names) - raise KedroCliError( - f"Found non-unique notebook names! Please rename the following: {names}" - ) - - output_dir = source_path / package_name / "nodes" - if not output_dir.is_dir(): - output_dir.mkdir() - (output_dir / "__init__.py").touch() - - for notebook in notebooks: - secho(f"Converting notebook '{notebook}'...") - output_path = output_dir / f"{notebook.stem}.py" - - if output_path.is_file(): - overwrite = overwrite_flag or click.confirm( - f"Output file {output_path} already exists. Overwrite?", default=False - ) - if overwrite: - _export_nodes(notebook, output_path) - else: - _export_nodes(notebook, output_path) - - secho("Done!", color="green") # type: ignore - - -def _export_nodes(filepath: Path, output_path: Path) -> None: - """Copy code from Jupyter cells into nodes in src//nodes/, - under filename with same name as notebook. - - Args: - filepath: Path to Jupyter notebook file - output_path: Path where notebook cells' source code will be exported - Raises: - KedroCliError: When provided a filepath that cannot be read as a - Jupyer notebook and loaded into json format. - """ - try: - content = json.loads(filepath.read_text()) - except json.JSONDecodeError as exc: - raise KedroCliError( - f"Provided filepath is not a Jupyter notebook: {filepath}" - ) from exc - cells = [ - cell - for cell in content["cells"] - if cell["cell_type"] == "code" and "node" in cell["metadata"].get("tags", {}) - ] - - if cells: - output_path.write_text("") - for cell in cells: - _append_source_code(cell, output_path) - else: - warn(f"Skipping notebook '{filepath}' - no nodes to export.") - - -def _append_source_code(cell: dict[str, Any], path: Path) -> None: - source_code = "".join(cell["source"]).strip() + "\n" - with path.open(mode="a") as file_: - file_.write(source_code) diff --git a/kedro/framework/cli/project.py b/kedro/framework/cli/project.py index f3cf141dfa..e9286e71ad 100644 --- a/kedro/framework/cli/project.py +++ b/kedro/framework/cli/project.py @@ -1,16 +1,12 @@ """A collection of CLI commands for working with Kedro project.""" import os -import shutil -import subprocess import sys -import webbrowser from pathlib import Path import click from kedro.framework.cli.utils import ( - KedroCliError, _check_module_importable, _config_file_callback, _deprecate_options, @@ -19,10 +15,8 @@ _split_load_versions, _split_params, call, - command_with_verbosity, env_option, forward_command, - python_call, split_node_names, split_string, ) @@ -74,63 +68,6 @@ def project_group(): # pragma: no cover pass -@forward_command(project_group, forward_help=True) -@click.pass_obj # this will pass the metadata as first argument -def test(metadata: ProjectMetadata, args, **kwargs): # noqa: ument - """Run the test suite. (DEPRECATED)""" - deprecation_message = ( - "DeprecationWarning: Command 'kedro test' is deprecated and " - "will not be available from Kedro 0.19.0. " - "Use the command 'pytest' instead. " - ) - click.secho(deprecation_message, fg="red") - - try: - _check_module_importable("pytest") - except KedroCliError as exc: - source_path = metadata.source_dir - raise KedroCliError( - NO_DEPENDENCY_MESSAGE.format(module="pytest", src=str(source_path)) - ) from exc - python_call("pytest", args) - - -@command_with_verbosity(project_group) -@click.option("-c", "--check-only", is_flag=True, help=LINT_CHECK_ONLY_HELP) -@click.argument("files", type=click.Path(exists=True), nargs=-1) -@click.pass_obj # this will pass the metadata as first argument -def lint( - metadata: ProjectMetadata, files, check_only, **kwargs -): # noqa: unused-argument - """Run flake8, isort and black. (DEPRECATED)""" - deprecation_message = ( - "DeprecationWarning: Command 'kedro lint' is deprecated and " - "will not be available from Kedro 0.19.0." - ) - click.secho(deprecation_message, fg="red") - - source_path = metadata.source_dir - package_name = metadata.package_name - files = files or (str(source_path / "tests"), str(source_path / package_name)) - - if "PYTHONPATH" not in os.environ: - # isort needs the source path to be in the 'PYTHONPATH' environment - # variable to treat it as a first-party import location - os.environ["PYTHONPATH"] = str(source_path) # pragma: no cover - - for module_name in ("flake8", "isort", "black"): - try: - _check_module_importable(module_name) - except KedroCliError as exc: - raise KedroCliError( - NO_DEPENDENCY_MESSAGE.format(module=module_name, src=str(source_path)) - ) from exc - - python_call("black", ("--check",) + files if check_only else files) - python_call("flake8", files) - python_call("isort", ("--check",) + files if check_only else files) - - @forward_command(project_group, forward_help=True) @env_option @click.pass_obj # this will pass the metadata as first argument @@ -177,145 +114,6 @@ def package(metadata: ProjectMetadata): ) -@project_group.command("build-docs") -@click.option( - "--open", - "-o", - "open_docs", - is_flag=True, - multiple=False, - default=False, - help=OPEN_ARG_HELP, -) -@click.pass_obj # this will pass the metadata as first argument -def build_docs(metadata: ProjectMetadata, open_docs): - """Build the project documentation. (DEPRECATED)""" - deprecation_message = ( - "DeprecationWarning: Command 'kedro build-docs' is deprecated and " - "will not be available from Kedro 0.19.0." - ) - click.secho(deprecation_message, fg="red") - - source_path = metadata.source_dir - package_name = metadata.package_name - - python_call("pip", ["install", str(source_path / "[docs]")]) - python_call("pip", ["install", "-r", str(source_path / "requirements.txt")]) - python_call("ipykernel", ["install", "--user", f"--name={package_name}"]) - shutil.rmtree("docs/build", ignore_errors=True) - call( - [ - "sphinx-apidoc", - "--module-first", - "-o", - "docs/source", - str(source_path / package_name), - ] - ) - call(["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"]) - if open_docs: - docs_page = (Path.cwd() / "docs" / "build" / "html" / "index.html").as_uri() - click.secho(f"Opening {docs_page}") - webbrowser.open(docs_page) - - -@forward_command(project_group, name="build-reqs") -@click.option( - "--input-file", - "input_file", - type=click.Path(exists=True, dir_okay=False, resolve_path=True), - multiple=False, - help=INPUT_FILE_HELP, -) -@click.option( - "--output-file", - "output_file", - multiple=False, - help=OUTPUT_FILE_HELP, -) -@click.pass_obj # this will pass the metadata as first argument -def build_reqs( - metadata: ProjectMetadata, input_file, output_file, args, **kwargs -): # noqa: unused-argument - """Run `pip-compile` on src/requirements.txt or the user defined input file and save - the compiled requirements to src/requirements.lock or the user defined output file. - (DEPRECATED) - """ - deprecation_message = ( - "DeprecationWarning: Command 'kedro build-reqs' is deprecated and " - "will not be available from Kedro 0.19.0." - ) - click.secho(deprecation_message, fg="red") - - source_path = metadata.source_dir - input_file = Path(input_file or source_path / "requirements.txt") - output_file = Path(output_file or source_path / "requirements.lock") - - if input_file.is_file(): - python_call( - "piptools", - [ - "compile", - *args, - str(input_file), - "--output-file", - str(output_file), - ], - ) - - else: - raise FileNotFoundError( - f"File '{input_file}' not found in the project. " - "Please specify another input or create the file and try again." - ) - - click.secho( - f"Requirements built! Please update {input_file.name} " - "if you'd like to make a change in your project's dependencies, " - f"and re-run build-reqs to generate the new {output_file.name}.", - fg="green", - ) - - -@command_with_verbosity(project_group, "activate-nbstripout") -@click.pass_obj # this will pass the metadata as first argument -def activate_nbstripout(metadata: ProjectMetadata, **kwargs): # noqa: unused-argument - """Install the nbstripout git hook to automatically clean notebooks. (DEPRECATED)""" - deprecation_message = ( - "DeprecationWarning: Command 'kedro activate-nbstripout' is deprecated and " - "will not be available from Kedro 0.19.0." - ) - click.secho(deprecation_message, fg="red") - - source_path = metadata.source_dir - click.secho( - ( - "Notebook output cells will be automatically cleared before committing" - " to git." - ), - fg="yellow", - ) - - try: - _check_module_importable("nbstripout") - except KedroCliError as exc: - raise KedroCliError( - NO_DEPENDENCY_MESSAGE.format(module="nbstripout", src=str(source_path)) - ) from exc - - try: - res = subprocess.run( # noqa: subprocess-run-check - ["git", "rev-parse", "--git-dir"], - capture_output=True, - ) - if res.returncode: - raise KedroCliError("Not a git repository. Run 'git init' first.") - except FileNotFoundError as exc: - raise KedroCliError("Git executable not found. Install Git first.") from exc - - call(["nbstripout", "--install"]) - - @project_group.command() @click.option( "--from-inputs", diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md b/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md index 07ae44d46c..2b7e60a610 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/README.md @@ -113,7 +113,7 @@ kedro jupyter convert --all ``` ### How to ignore notebook output cells in `git` -To automatically strip out all output cell contents before committing to `git`, you can run `kedro activate-nbstripout`. This will add a hook in `.git/config` which will run `nbstripout` before anything is committed to `git`. +To automatically strip out all output cell contents before committing to `git`, you can use tools like [`nbstripout`](https://github.com/kynan/nbstripout). For example, you can add a hook in `.git/config` with `nbstripout --install`. This will run `nbstripout` before anything is committed to `git`. > *Note:* Your output cells will be retained locally. diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/tests/test_run.py b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/tests/test_run.py index 785c5a40b9..eb57d1908e 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/src/tests/test_run.py +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/src/tests/test_run.py @@ -5,7 +5,7 @@ project's structure, and in files named test_*.py. They are simply functions named ``test_*`` which test a unit of logic. -To run the tests, run ``kedro test`` from the project root directory. +To run the tests, run ``pytest`` from the project root directory. """ from pathlib import Path diff --git a/tests/framework/cli/test_jupyter.py b/tests/framework/cli/test_jupyter.py index 8f363bac3e..20a5dc9ad0 100644 --- a/tests/framework/cli/test_jupyter.py +++ b/tests/framework/cli/test_jupyter.py @@ -1,7 +1,5 @@ -import json import shutil from pathlib import Path -from tempfile import NamedTemporaryFile import pytest from click.testing import CliRunner @@ -11,7 +9,7 @@ get_kernel_spec, ) -from kedro.framework.cli.jupyter import _create_kernel, _export_nodes +from kedro.framework.cli.jupyter import _create_kernel from kedro.framework.cli.utils import KedroCliError @@ -194,304 +192,3 @@ def cleanup_nodes_dir(fake_package_path): nodes_dir = fake_package_path / "nodes" if nodes_dir.exists(): shutil.rmtree(str(nodes_dir)) - - -@pytest.mark.usefixtures("chdir_to_dummy_project", "cleanup_nodes_dir") -class TestConvertNotebookCommand: - @pytest.fixture - def fake_export_nodes(self, mocker): - return mocker.patch("kedro.framework.cli.jupyter._export_nodes") - - @pytest.fixture - def tmp_file_path(self): - with NamedTemporaryFile() as f: - yield Path(f.name) - - # noqa: too-many-arguments - def test_convert_one_file_overwrite( - self, - mocker, - fake_project_cli, - fake_export_nodes, - tmp_file_path, - fake_package_path, - fake_metadata, - ): - """ - Trying to convert one file, the output file already exists, - overwriting it. - """ - mocker.patch.object(Path, "is_file", return_value=True) - mocker.patch("click.confirm", return_value=True) - output_dir = fake_package_path / "nodes" - assert not output_dir.exists() - - result = CliRunner().invoke( - fake_project_cli, - ["jupyter", "convert", str(tmp_file_path)], - obj=fake_metadata, - ) - assert not result.exit_code, result.stdout - - assert (output_dir / "__init__.py").is_file() - fake_export_nodes.assert_called_once_with( - tmp_file_path.resolve(), output_dir / f"{tmp_file_path.stem}.py" - ) - - def test_convert_one_file_do_not_overwrite( - self, mocker, fake_project_cli, fake_export_nodes, tmp_file_path, fake_metadata - ): - """ - Trying to convert one file, the output file already exists, - user refuses to overwrite it. - """ - mocker.patch.object(Path, "is_file", return_value=True) - mocker.patch("click.confirm", return_value=False) - - result = CliRunner().invoke( - fake_project_cli, - ["jupyter", "convert", str(tmp_file_path)], - obj=fake_metadata, - ) - assert not result.exit_code, result.stdout - - fake_export_nodes.assert_not_called() - - def test_convert_all_files( - self, - mocker, - fake_project_cli, - fake_export_nodes, - fake_package_path, - fake_metadata, - ): - """Trying to convert all files, the output files already exist.""" - mocker.patch.object(Path, "is_file", return_value=True) - mocker.patch("click.confirm", return_value=True) - mocker.patch( - "kedro.framework.cli.jupyter.iglob", return_value=["/path/1", "/path/2"] - ) - output_dir = fake_package_path / "nodes" - assert not output_dir.exists() - - result = CliRunner().invoke( - fake_project_cli, ["jupyter", "convert", "--all"], obj=fake_metadata - ) - assert not result.exit_code, result.stdout - - assert (output_dir / "__init__.py").is_file() - fake_export_nodes.assert_has_calls( - [ - mocker.call(Path("/path/1"), output_dir / "1.py"), - mocker.call(Path("/path/2"), output_dir / "2.py"), - ] - ) - - def test_convert_without_filepath_and_all_flag( - self, fake_project_cli, fake_metadata - ): - """Neither path nor --all flag is provided.""" - result = CliRunner().invoke( - fake_project_cli, ["jupyter", "convert"], obj=fake_metadata - ) - expected_output = ( - "Please specify a notebook filepath or " - "add '--all' to convert all notebooks.\n" - ) - assert result.exit_code - assert expected_output in result.stdout - - def test_non_unique_notebook_names_error( - self, fake_project_cli, mocker, fake_metadata - ): - """Trying to convert notebooks with the same name.""" - mocker.patch( - "kedro.framework.cli.jupyter.iglob", return_value=["/path1/1", "/path2/1"] - ) - - result = CliRunner().invoke( - fake_project_cli, ["jupyter", "convert", "--all"], obj=fake_metadata - ) - - expected_output = ( - "Error: Found non-unique notebook names! Please rename the following: 1\n" - ) - assert result.exit_code - assert expected_output in result.output - - def test_convert_one_file( - self, - fake_project_cli, - fake_export_nodes, - tmp_file_path, - fake_package_path, - fake_metadata, - ): - """Trying to convert one file, the output file doesn't exist.""" - output_dir = fake_package_path / "nodes" - assert not output_dir.exists() - - result = CliRunner().invoke( - fake_project_cli, - ["jupyter", "convert", str(tmp_file_path)], - obj=fake_metadata, - ) - assert not result.exit_code, result.stdout - - assert (output_dir / "__init__.py").is_file() - fake_export_nodes.assert_called_once_with( - tmp_file_path.resolve(), output_dir / f"{tmp_file_path.stem}.py" - ) - - def test_convert_one_file_nodes_directory_exists( - self, - fake_project_cli, - fake_export_nodes, - tmp_file_path, - fake_package_path, - fake_metadata, - ): - """User-created nodes/ directory is used as is.""" - output_dir = fake_package_path / "nodes" - assert not output_dir.exists() - output_dir.mkdir() - - result = CliRunner().invoke( - fake_project_cli, - ["jupyter", "convert", str(tmp_file_path)], - obj=fake_metadata, - ) - assert not result.exit_code, result.stdout - - assert not (output_dir / "__init__.py").is_file() - fake_export_nodes.assert_called_once_with( - tmp_file_path.resolve(), output_dir / f"{tmp_file_path.stem}.py" - ) - - -class TestExportNodes: - @pytest.fixture - def project_path(self, tmp_path): - temp = Path(str(tmp_path)) - return Path(temp / "some/path/to/my_project") - - @pytest.fixture - def nodes_path(self, project_path): - path = project_path / "src/my_project/nodes" - path.mkdir(parents=True) - return path - - def test_export_nodes(self, project_path, nodes_path): - nodes = json.dumps( - { - "cells": [ - { - "cell_type": "code", - "source": "print('hello world')", - "metadata": {"tags": ["node"]}, - }, - { - "cell_type": "code", - "source": "print(10+5)", - "metadata": {"tags": ["node"]}, - }, - {"cell_type": "code", "source": "a = 10", "metadata": {}}, - ] - } - ) - notebook_file = project_path / "notebook.ipynb" - notebook_file.write_text(nodes) - - output_path = nodes_path / f"{notebook_file.stem}.py" - _export_nodes(notebook_file, output_path) - - assert output_path.is_file() - assert output_path.read_text() == "print('hello world')\nprint(10+5)\n" - - def test_export_nodes_different_notebook_paths(self, project_path, nodes_path): - nodes = json.dumps( - { - "cells": [ - { - "cell_type": "code", - "source": "print('hello world')", - "metadata": {"tags": ["node"]}, - } - ] - } - ) - notebook_file1 = project_path / "notebook1.ipynb" - notebook_file1.write_text(nodes) - output_path1 = nodes_path / "notebook1.py" - - notebook_file2 = nodes_path / "notebook2.ipynb" - notebook_file2.write_text(nodes) - output_path2 = nodes_path / "notebook2.py" - - _export_nodes(notebook_file1, output_path1) - _export_nodes(notebook_file2, output_path2) - - assert output_path1.read_text() == "print('hello world')\n" - assert output_path2.read_text() == "print('hello world')\n" - - def test_export_nodes_nothing_to_write(self, project_path, nodes_path): - nodes = json.dumps( - { - "cells": [ - { - "cell_type": "code", - "source": "print('hello world')", - "metadata": {}, - }, - { - "cell_type": "text", - "source": "hello world", - "metadata": {"tags": ["node"]}, - }, - ] - } - ) - notebook_file = project_path / "notebook.iypnb" - notebook_file.write_text(nodes) - - with pytest.warns(UserWarning, match="Skipping notebook"): - output_path = nodes_path / f"{notebook_file.stem}.py" - _export_nodes(notebook_file, output_path) - - output_path = nodes_path / "notebook.py" - assert not output_path.exists() - - def test_export_nodes_overwrite(self, project_path, nodes_path): - existing_nodes = nodes_path / "notebook.py" - existing_nodes.touch() - existing_nodes.write_text("original") - - nodes = json.dumps( - { - "cells": [ - { - "cell_type": "code", - "source": "print('hello world')", - "metadata": {"tags": ["node"]}, - } - ] - } - ) - notebook_file = project_path / "notebook.iypnb" - notebook_file.write_text(nodes) - - output_path = nodes_path / f"{notebook_file.stem}.py" - _export_nodes(notebook_file, output_path) - - assert output_path.is_file() - assert output_path.read_text() == "print('hello world')\n" - - def test_export_nodes_json_error(self, nodes_path): - random_file = nodes_path / "notebook.txt" - random_file.touch() - random_file.write_text("original") - output_path = nodes_path / f"{random_file.stem}.py" - - pattern = "Provided filepath is not a Jupyter notebook" - with pytest.raises(KedroCliError, match=pattern): - _export_nodes(random_file, output_path) diff --git a/tests/framework/cli/test_project.py b/tests/framework/cli/test_project.py index d965113ea8..e8939e5987 100644 --- a/tests/framework/cli/test_project.py +++ b/tests/framework/cli/test_project.py @@ -1,234 +1,20 @@ # pylint: disable=unused-argument import sys -from pathlib import Path import pytest from click.testing import CliRunner -from kedro.framework.cli.project import NO_DEPENDENCY_MESSAGE - @pytest.fixture(autouse=True) def call_mock(mocker): return mocker.patch("kedro.framework.cli.project.call") -@pytest.fixture(autouse=True) -def python_call_mock(mocker): - return mocker.patch("kedro.framework.cli.project.python_call") - - @pytest.fixture def fake_copyfile(mocker): return mocker.patch("shutil.copyfile") -@pytest.mark.usefixtures("chdir_to_dummy_project") -class TestActivateNbstripoutCommand: - @staticmethod - @pytest.fixture() - def fake_nbstripout(): - """ - ``nbstripout`` tries to access ``sys.stdin.buffer.readable`` - on import, but it's patches by pytest. - Let's replace it by the fake! - """ - sys.modules["nbstripout"] = "fake" - yield - del sys.modules["nbstripout"] - - @staticmethod - @pytest.fixture - def fake_git_repo(mocker): - return mocker.patch("subprocess.run", return_value=mocker.Mock(returncode=0)) - - @staticmethod - @pytest.fixture - def without_git_repo(mocker): - return mocker.patch("subprocess.run", return_value=mocker.Mock(returncode=1)) - - def test_install_successfully( - self, fake_project_cli, call_mock, fake_nbstripout, fake_git_repo, fake_metadata - ): - result = CliRunner().invoke( - fake_project_cli, ["activate-nbstripout"], obj=fake_metadata - ) - assert not result.exit_code - - call_mock.assert_called_once_with(["nbstripout", "--install"]) - - fake_git_repo.assert_called_once_with( - ["git", "rev-parse", "--git-dir"], capture_output=True - ) - - def test_nbstripout_not_installed( - self, fake_project_cli, fake_git_repo, mocker, fake_metadata - ): - """ - Run activate-nbstripout target without nbstripout installed - There should be a clear message about it. - """ - mocker.patch.dict("sys.modules", {"nbstripout": None}) - - result = CliRunner().invoke( - fake_project_cli, ["activate-nbstripout"], obj=fake_metadata - ) - assert result.exit_code - assert "nbstripout is not installed" in result.stdout - - def test_no_git_repo( - self, fake_project_cli, fake_nbstripout, without_git_repo, fake_metadata - ): - """ - Run activate-nbstripout target with no git repo available. - There should be a clear message about it. - """ - result = CliRunner().invoke( - fake_project_cli, ["activate-nbstripout"], obj=fake_metadata - ) - - assert result.exit_code - assert "Not a git repository" in result.stdout - - def test_no_git_executable( - self, fake_project_cli, fake_nbstripout, mocker, fake_metadata - ): - mocker.patch("subprocess.run", side_effect=FileNotFoundError) - result = CliRunner().invoke( - fake_project_cli, ["activate-nbstripout"], obj=fake_metadata - ) - - assert result.exit_code - assert "Git executable not found. Install Git first." in result.stdout - - -@pytest.mark.usefixtures("chdir_to_dummy_project") -class TestTestCommand: - def test_happy_path(self, fake_project_cli, python_call_mock): - result = CliRunner().invoke(fake_project_cli, ["test", "--random-arg", "value"]) - assert not result.exit_code - python_call_mock.assert_called_once_with("pytest", ("--random-arg", "value")) - - def test_pytest_not_installed( - self, fake_project_cli, python_call_mock, mocker, fake_repo_path, fake_metadata - ): - mocker.patch.dict("sys.modules", {"pytest": None}) - - result = CliRunner().invoke( - fake_project_cli, ["test", "--random-arg", "value"], obj=fake_metadata - ) - expected_message = NO_DEPENDENCY_MESSAGE.format( - module="pytest", src=str(fake_repo_path / "src") - ) - - assert result.exit_code - assert expected_message in result.stdout - python_call_mock.assert_not_called() - - -@pytest.mark.usefixtures("chdir_to_dummy_project") -class TestLintCommand: - @pytest.mark.parametrize("files", [(), ("src",)]) - def test_lint( - self, - fake_project_cli, - python_call_mock, - files, - mocker, - fake_repo_path, - fake_metadata, - ): - mocker.patch("kedro.framework.cli.project._check_module_importable") - result = CliRunner().invoke( - fake_project_cli, ["lint", *files], obj=fake_metadata - ) - assert not result.exit_code, result.stdout - - expected_files = files or ( - str(fake_repo_path / "src/tests"), - str(fake_repo_path / "src/dummy_package"), - ) - expected_calls = [ - mocker.call("black", expected_files), - mocker.call("flake8", expected_files), - mocker.call("isort", expected_files), - ] - - assert python_call_mock.call_args_list == expected_calls - - @pytest.mark.parametrize( - "check_flag,files", - [ - ("-c", ()), - ("--check-only", ()), - ("-c", ("src",)), - ("--check-only", ("src",)), - ], - ) - def test_lint_check_only( - self, - fake_project_cli, - python_call_mock, - check_flag, - mocker, - files, - fake_repo_path, - fake_metadata, - ): - mocker.patch("kedro.framework.cli.project._check_module_importable") - result = CliRunner().invoke( - fake_project_cli, ["lint", check_flag, *files], obj=fake_metadata - ) - assert not result.exit_code, result.stdout - - expected_files = files or ( - str(fake_repo_path / "src/tests"), - str(fake_repo_path / "src/dummy_package"), - ) - expected_calls = [ - mocker.call("black", ("--check",) + expected_files), - mocker.call("flake8", expected_files), - mocker.call("isort", ("--check",) + expected_files), - ] - - assert python_call_mock.call_args_list == expected_calls - - @pytest.mark.parametrize( - "module_name,side_effects", - [("flake8", [ImportError, None, None]), ("isort", [None, ImportError, None])], - ) - def test_import_not_installed( - self, - fake_project_cli, - python_call_mock, - module_name, - side_effects, - mocker, - fake_repo_path, - fake_metadata, - ): - # pretending we have the other linting dependencies, but not the - mocker.patch( - "kedro.framework.cli.utils.import_module", side_effect=side_effects - ) - - result = CliRunner().invoke(fake_project_cli, ["lint"], obj=fake_metadata) - expected_message = NO_DEPENDENCY_MESSAGE.format( - module=module_name, src=str(fake_repo_path / "src") - ) - - assert result.exit_code, result.stdout - assert expected_message in result.stdout - python_call_mock.assert_not_called() - - def test_pythonpath_env_var( - self, fake_project_cli, mocker, fake_repo_path, fake_metadata - ): - mocked_environ = mocker.patch("os.environ", {}) - CliRunner().invoke(fake_project_cli, ["lint"], obj=fake_metadata) - assert mocked_environ == {"PYTHONPATH": str(fake_repo_path / "src")} - - @pytest.mark.usefixtures("chdir_to_dummy_project") class TestIpythonCommand: def test_happy_path( @@ -313,161 +99,3 @@ def test_happy_path( ), ] ) - - -@pytest.mark.usefixtures("chdir_to_dummy_project") -class TestBuildDocsCommand: - def test_happy_path( - self, - call_mock, - python_call_mock, - fake_project_cli, - mocker, - fake_repo_path, - fake_metadata, - ): - fake_rmtree = mocker.patch("shutil.rmtree") - - result = CliRunner().invoke(fake_project_cli, ["build-docs"], obj=fake_metadata) - assert not result.exit_code, result.stdout - call_mock.assert_has_calls( - [ - mocker.call( - [ - "sphinx-apidoc", - "--module-first", - "-o", - "docs/source", - str(fake_repo_path / "src/dummy_package"), - ] - ), - mocker.call( - ["sphinx-build", "-M", "html", "docs/source", "docs/build", "-a"] - ), - ] - ) - python_call_mock.assert_has_calls( - [ - mocker.call("pip", ["install", str(fake_repo_path / "src/[docs]")]), - mocker.call( - "pip", - ["install", "-r", str(fake_repo_path / "src/requirements.txt")], - ), - mocker.call("ipykernel", ["install", "--user", "--name=dummy_package"]), - ] - ) - fake_rmtree.assert_called_once_with("docs/build", ignore_errors=True) - - @pytest.mark.parametrize("open_flag", ["-o", "--open"]) - def test_open_docs(self, open_flag, fake_project_cli, mocker, fake_metadata): - mocker.patch("shutil.rmtree") - patched_browser = mocker.patch("webbrowser.open") - result = CliRunner().invoke( - fake_project_cli, ["build-docs", open_flag], obj=fake_metadata - ) - assert not result.exit_code, result.stdout - expected_path = (Path.cwd() / "docs" / "build" / "html" / "index.html").as_uri() - patched_browser.assert_called_once_with(expected_path) - - -@pytest.mark.usefixtures("chdir_to_dummy_project", "fake_copyfile") -class TestBuildReqsCommand: - def test_compile_from_requirements_file( - self, - python_call_mock, - fake_project_cli, - mocker, - fake_repo_path, - fake_copyfile, - fake_metadata, - ): - # File exists: - mocker.patch.object(Path, "is_file", return_value=True) - - result = CliRunner().invoke(fake_project_cli, ["build-reqs"], obj=fake_metadata) - assert not result.exit_code, result.stdout - assert "Requirements built!" in result.stdout - - python_call_mock.assert_called_once_with( - "piptools", - [ - "compile", - str(fake_repo_path / "src" / "requirements.txt"), - "--output-file", - str(fake_repo_path / "src" / "requirements.lock"), - ], - ) - - def test_compile_from_input_and_to_output_file( - self, - python_call_mock, - fake_project_cli, - fake_repo_path, - fake_copyfile, - fake_metadata, - ): - # File exists: - input_file = fake_repo_path / "src" / "dev-requirements.txt" - with open(input_file, "a", encoding="utf-8") as file: - file.write("") - output_file = fake_repo_path / "src" / "dev-requirements.lock" - - result = CliRunner().invoke( - fake_project_cli, - [ - "build-reqs", - "--input-file", - str(input_file), - "--output-file", - str(output_file), - ], - obj=fake_metadata, - ) - assert not result.exit_code, result.stdout - assert "Requirements built!" in result.stdout - python_call_mock.assert_called_once_with( - "piptools", - ["compile", str(input_file), "--output-file", str(output_file)], - ) - - @pytest.mark.parametrize( - "extra_args", [["--generate-hashes"], ["-foo", "--bar", "baz"]] - ) - def test_extra_args( - self, - python_call_mock, - fake_project_cli, - fake_repo_path, - extra_args, - fake_metadata, - ): - requirements_txt = fake_repo_path / "src" / "requirements.txt" - - result = CliRunner().invoke( - fake_project_cli, ["build-reqs"] + extra_args, obj=fake_metadata - ) - - assert not result.exit_code, result.stdout - assert "Requirements built!" in result.stdout - - call_args = ( - ["compile"] - + extra_args - + [str(requirements_txt)] - + ["--output-file", str(fake_repo_path / "src" / "requirements.lock")] - ) - python_call_mock.assert_called_once_with("piptools", call_args) - - @pytest.mark.parametrize("os_name", ["posix", "nt"]) - def test_missing_requirements_txt( - self, fake_project_cli, mocker, fake_metadata, os_name, fake_repo_path - ): - """Test error when input file requirements.txt doesn't exists.""" - requirements_txt = fake_repo_path / "src" / "requirements.txt" - - mocker.patch("kedro.framework.cli.project.os").name = os_name - mocker.patch.object(Path, "is_file", return_value=False) - result = CliRunner().invoke(fake_project_cli, ["build-reqs"], obj=fake_metadata) - assert result.exit_code # Error expected - assert isinstance(result.exception, FileNotFoundError) - assert f"File '{requirements_txt}' not found" in str(result.exception) diff --git a/tests/tools/test_cli.py b/tests/tools/test_cli.py index cf3ce71d1c..346d7af6e6 100644 --- a/tests/tools/test_cli.py +++ b/tests/tools/test_cli.py @@ -13,13 +13,9 @@ REPO_NAME = "cli_tools_dummy_project" PACKAGE_NAME = "cli_tools_dummy_package" DEFAULT_KEDRO_COMMANDS = [ - "activate-nbstripout", - "build-docs", - "build-reqs", "catalog", "ipython", "jupyter", - "lint", "new", "package", "pipeline", @@ -27,7 +23,6 @@ "registry", "run", "starter", - "test", ]