Skip to content

Commit

Permalink
Improve error when micropkg pull does not find sdist
Browse files Browse the repository at this point in the history
Fix gh-2542.

Signed-off-by: Juan Luis Cano Rodríguez <juan_luis_cano@mckinsey.com>
  • Loading branch information
astrojuanlu committed May 29, 2023
1 parent 2f248a4 commit b427bc1
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 9 deletions.
4 changes: 2 additions & 2 deletions docs/source/nodes_and_pipelines/micro_packaging.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Micro-packaging allows users to share Kedro micro-packages across codebases, org

You can package a micro-package by executing: `kedro micropkg package <micropkg_name>`

* This will generate a new [tar](https://docs.python.org/3/distutils/sourcedist.html) file for this micro-package.
* This will generate a new [source distribution](https://docs.python.org/3/distutils/sourcedist.html) for this micro-package.
* By default, the tar file will be saved into `dist/` directory inside your project.
* You can customise the target with the `--destination` (`-d`) option.

Expand Down Expand Up @@ -64,7 +64,7 @@ The examples above apply to any generic Python package, modular pipelines fall u

You can pull a micro-package from a tar file by executing `kedro micropkg pull <package_name>`.

* The `<package_name>` must either be a package name on PyPI or a path to the tar file.
* The `<package_name>` must either be a package name on PyPI or a path to the source distribution file.
* Kedro will unpack the tar file, and install the files in following locations in your Kedro project:
* All the micro-package code in `src/<package_name>/<micropkg_name>/`
* Configuration files in `conf/<env>/parameters/<micropkg_name>.yml`, where `<env>` defaults to `base`.
Expand Down
20 changes: 15 additions & 5 deletions kedro/framework/cli/micropkg.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,15 +324,25 @@ def _unpack_sdist(location: str, destination: Path, fs_args: str | None) -> None
safe_extract(tar_file, destination)
else:
python_call(
"pip", ["download", "--no-deps", "--dest", str(destination), location]
"pip",
[
"download",
"--no-deps",
"--no-binary", # the micropackaging expects an sdist,
":all:", # wheels are not supported
"--dest",
str(destination),
location,
],
)
sdist_file = list(destination.glob("*.tar.gz"))
# `--no-deps` should fetch only one source distribution file, and CLI should fail if that's
# not the case.
if len(sdist_file) != 1:
# `--no-deps --no-binary :all:` should fetch only one source distribution file,
# and CLI should fail if that's not the case.
# No need to check for zero sdists, in that case the previous command would fail.
if len(sdist_file) > 1:
file_names = [sf.name for sf in sdist_file]
raise KedroCliError(
f"More than 1 or no sdist files found: {file_names}. "
f"More than 1 sdist files found: {file_names}. "
f"There has to be exactly one source distribution file."
)
with tarfile.open(sdist_file[0], "r:gz") as fs_file:
Expand Down
24 changes: 22 additions & 2 deletions tests/framework/cli/micropkg/test_micropkg_pull.py
Original file line number Diff line number Diff line change
Expand Up @@ -645,6 +645,8 @@ def test_pull_from_pypi(
[
"download",
"--no-deps",
"--no-binary",
":all:",
"--dest",
str(tmp_path),
package_name,
Expand Down Expand Up @@ -695,7 +697,16 @@ def test_invalid_pull_from_pypi(
assert result.exit_code

python_call_mock.assert_called_once_with(
"pip", ["download", "--no-deps", "--dest", str(tmp_path), invalid_pypi_name]
"pip",
[
"download",
"--no-deps",
"--no-binary",
":all:",
"--dest",
str(tmp_path),
invalid_pypi_name,
],
)

assert pypi_error_message in result.stdout
Expand Down Expand Up @@ -750,7 +761,16 @@ def test_pull_unsupported_protocol_by_fsspec(
assert result.exit_code
filesystem_mock.assert_called_once_with(protocol)
python_call_mock.assert_called_once_with(
"pip", ["download", "--no-deps", "--dest", str(tmp_path), package_path]
"pip",
[
"download",
"--no-deps",
"--no-binary",
":all:",
"--dest",
str(tmp_path),
package_path,
],
)
assert exception_message in result.output
assert "Trying to use 'pip download'..." in result.output
Expand Down

0 comments on commit b427bc1

Please sign in to comment.