From 0c7026c8262b3e8a049573b09953c6f99c86bf3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 29 May 2023 16:05:10 +0200 Subject: [PATCH] Improve error when `micropkg pull` does not find sdist Fix gh-2542. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- .../nodes_and_pipelines/micro_packaging.md | 4 ++-- kedro/framework/cli/micropkg.py | 20 ++++++++++++---- .../cli/micropkg/test_micropkg_pull.py | 24 +++++++++++++++++-- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/docs/source/nodes_and_pipelines/micro_packaging.md b/docs/source/nodes_and_pipelines/micro_packaging.md index f034b14204..d72a0c3b4f 100644 --- a/docs/source/nodes_and_pipelines/micro_packaging.md +++ b/docs/source/nodes_and_pipelines/micro_packaging.md @@ -6,7 +6,7 @@ Micro-packaging allows users to share Kedro micro-packages across codebases, org You can package a micro-package by executing: `kedro micropkg package ` -* This will generate a new [tar](https://docs.python.org/3/distutils/sourcedist.html) file for this micro-package. +* This will generate a new [source distribution](https://docs.python.org/3/distutils/sourcedist.html) for this micro-package. * By default, the tar file will be saved into `dist/` directory inside your project. * You can customise the target with the `--destination` (`-d`) option. @@ -64,7 +64,7 @@ The examples above apply to any generic Python package, modular pipelines fall u You can pull a micro-package from a tar file by executing `kedro micropkg pull `. -* The `` must either be a package name on PyPI or a path to the tar file. +* The `` must either be a package name on PyPI or a path to the source distribution file. * Kedro will unpack the tar file, and install the files in following locations in your Kedro project: * All the micro-package code in `src///` * Configuration files in `conf//parameters/.yml`, where `` defaults to `base`. diff --git a/kedro/framework/cli/micropkg.py b/kedro/framework/cli/micropkg.py index ce7d28fb8b..8820ce71de 100644 --- a/kedro/framework/cli/micropkg.py +++ b/kedro/framework/cli/micropkg.py @@ -324,15 +324,25 @@ def _unpack_sdist(location: str, destination: Path, fs_args: str | None) -> None safe_extract(tar_file, destination) else: python_call( - "pip", ["download", "--no-deps", "--dest", str(destination), location] + "pip", + [ + "download", + "--no-deps", + "--no-binary", # the micropackaging expects an sdist, + ":all:", # wheels are not supported + "--dest", + str(destination), + location, + ], ) sdist_file = list(destination.glob("*.tar.gz")) - # `--no-deps` should fetch only one source distribution file, and CLI should fail if that's - # not the case. - if len(sdist_file) != 1: + # `--no-deps --no-binary :all:` should fetch only one source distribution file, + # and CLI should fail if that's not the case. + # No need to check for zero sdists, in that case the previous command would fail. + if len(sdist_file) > 1: file_names = [sf.name for sf in sdist_file] raise KedroCliError( - f"More than 1 or no sdist files found: {file_names}. " + f"More than 1 sdist files found: {file_names}. " f"There has to be exactly one source distribution file." ) with tarfile.open(sdist_file[0], "r:gz") as fs_file: diff --git a/tests/framework/cli/micropkg/test_micropkg_pull.py b/tests/framework/cli/micropkg/test_micropkg_pull.py index 05552fb74b..cb6b2b1a8e 100644 --- a/tests/framework/cli/micropkg/test_micropkg_pull.py +++ b/tests/framework/cli/micropkg/test_micropkg_pull.py @@ -645,6 +645,8 @@ def test_pull_from_pypi( [ "download", "--no-deps", + "--no-binary", + ":all", "--dest", str(tmp_path), package_name, @@ -695,7 +697,16 @@ def test_invalid_pull_from_pypi( assert result.exit_code python_call_mock.assert_called_once_with( - "pip", ["download", "--no-deps", "--dest", str(tmp_path), invalid_pypi_name] + "pip", + [ + "download", + "--no-deps", + "--no-binary", + ":all:", + "--dest", + str(tmp_path), + invalid_pypi_name, + ], ) assert pypi_error_message in result.stdout @@ -750,7 +761,16 @@ def test_pull_unsupported_protocol_by_fsspec( assert result.exit_code filesystem_mock.assert_called_once_with(protocol) python_call_mock.assert_called_once_with( - "pip", ["download", "--no-deps", "--dest", str(tmp_path), package_path] + "pip", + [ + "download", + "--no-deps", + "--no-binary", + ":all", + "--dest", + str(tmp_path), + package_path, + ], ) assert exception_message in result.output assert "Trying to use 'pip download'..." in result.output