From f4a9eb76109f468e6ed259c3204864381cb2385b Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 20 Oct 2022 00:58:07 -0500 Subject: [PATCH 01/19] Add option to skip installing path dependencies --- src/poetry/console/commands/install.py | 9 +++++++++ src/poetry/installation/installer.py | 8 ++++++++ src/poetry/puzzle/transaction.py | 4 ++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/poetry/console/commands/install.py b/src/poetry/console/commands/install.py index 844ec37176f..8e678a59cb0 100644 --- a/src/poetry/console/commands/install.py +++ b/src/poetry/console/commands/install.py @@ -30,6 +30,14 @@ class InstallCommand(InstallerCommand): option( "no-root", None, "Do not install the root package (the current project)." ), + option( + "no-path", + None, + "Do not install any path dependencies " + "(useful to install dependencies without source code, e.g. for caching)", + flag=True, + multiple=False, + ), option( "dry-run", None, @@ -148,6 +156,7 @@ def handle(self) -> int: with_synchronization = True self.installer.only_groups(self.activated_groups) + self.installer.skip_path(self.option("no-path")) self.installer.dry_run(self.option("dry-run")) self.installer.requires_synchronization(with_synchronization) self.installer.executor.enable_bytecode_compilation(self.option("compile")) diff --git a/src/poetry/installation/installer.py b/src/poetry/installation/installer.py index b5aa54f7072..412341ea004 100644 --- a/src/poetry/installation/installer.py +++ b/src/poetry/installation/installer.py @@ -3,6 +3,7 @@ import warnings from typing import TYPE_CHECKING +from unittest import skip from cleo.io.null_io import NullIO from packaging.utils import canonicalize_name @@ -59,6 +60,7 @@ def __init__( self._verbose = False self._write_lock = True self._groups: Iterable[str] | None = None + self._skip_path = False self._execute_operations = True self._lock = False @@ -149,6 +151,11 @@ def update(self, update: bool = True) -> Installer: self._update = update return self + + def skip_path(self, skip_path: bool = False) -> Installer: + self._skip_path = skip_path + + return self def lock(self, update: bool = True) -> Installer: """ @@ -336,6 +343,7 @@ def _do_install(self) -> int: ops = solver.solve(use_latest=self._whitelist).calculate_operations( with_uninstalls=self._requires_synchronization, synchronize=self._requires_synchronization, + skip_path=self._skip_path ) if not self._requires_synchronization: diff --git a/src/poetry/puzzle/transaction.py b/src/poetry/puzzle/transaction.py index 74d0d6c5e61..c9f89e0f2ef 100644 --- a/src/poetry/puzzle/transaction.py +++ b/src/poetry/puzzle/transaction.py @@ -27,7 +27,7 @@ def __init__( self._root_package = root_package def calculate_operations( - self, with_uninstalls: bool = True, synchronize: bool = False + self, with_uninstalls: bool = True, synchronize: bool = False, skip_path: bool = False ) -> list[Operation]: from poetry.installation.operations import Install from poetry.installation.operations import Uninstall @@ -70,7 +70,7 @@ def calculate_operations( break - if not installed: + if not installed and (not skip_path or result_package.source_type != "directory"): operations.append(Install(result_package, priority=priority)) if with_uninstalls: From 990bc2df9b2164af66dc516316745e3ca7cc21bd Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 20 Oct 2022 01:18:57 -0500 Subject: [PATCH 02/19] Update installer.py --- src/poetry/installation/installer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/poetry/installation/installer.py b/src/poetry/installation/installer.py index 412341ea004..ca0bcfa7873 100644 --- a/src/poetry/installation/installer.py +++ b/src/poetry/installation/installer.py @@ -3,7 +3,6 @@ import warnings from typing import TYPE_CHECKING -from unittest import skip from cleo.io.null_io import NullIO from packaging.utils import canonicalize_name From e5e6e5af00e9a081d7b059aa93f7ae4e9fc5c3da Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 20 Oct 2022 20:28:15 -0500 Subject: [PATCH 03/19] lint --- src/poetry/installation/installer.py | 4 ++-- src/poetry/puzzle/transaction.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/poetry/installation/installer.py b/src/poetry/installation/installer.py index ca0bcfa7873..bb39f93fc70 100644 --- a/src/poetry/installation/installer.py +++ b/src/poetry/installation/installer.py @@ -150,7 +150,7 @@ def update(self, update: bool = True) -> Installer: self._update = update return self - + def skip_path(self, skip_path: bool = False) -> Installer: self._skip_path = skip_path @@ -342,7 +342,7 @@ def _do_install(self) -> int: ops = solver.solve(use_latest=self._whitelist).calculate_operations( with_uninstalls=self._requires_synchronization, synchronize=self._requires_synchronization, - skip_path=self._skip_path + skip_path=self._skip_path, ) if not self._requires_synchronization: diff --git a/src/poetry/puzzle/transaction.py b/src/poetry/puzzle/transaction.py index c9f89e0f2ef..947c0af5798 100644 --- a/src/poetry/puzzle/transaction.py +++ b/src/poetry/puzzle/transaction.py @@ -27,7 +27,10 @@ def __init__( self._root_package = root_package def calculate_operations( - self, with_uninstalls: bool = True, synchronize: bool = False, skip_path: bool = False + self, + with_uninstalls: bool = True, + synchronize: bool = False, + skip_path: bool = False, ) -> list[Operation]: from poetry.installation.operations import Install from poetry.installation.operations import Uninstall @@ -70,7 +73,9 @@ def calculate_operations( break - if not installed and (not skip_path or result_package.source_type != "directory"): + if not installed and ( + not skip_path or result_package.source_type != "directory" + ): operations.append(Install(result_package, priority=priority)) if with_uninstalls: From 76dd7e24e608fc957794293f8bf94ad466a4903d Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 26 Oct 2022 13:42:40 -0500 Subject: [PATCH 04/19] add tests and docs --- docs/cli.md | 7 ++++ docs/faq.md | 34 +++++++++++++++++++ tests/console/commands/test_install.py | 12 +++++++ tests/installation/test_installer.py | 47 ++++++++++++++++++++++++-- 4 files changed, 98 insertions(+), 2 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index f09ffb97273..073d975fee9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -225,6 +225,11 @@ If you want to skip this installation, use the `--no-root` option. poetry install --no-root ``` +Similar to `--no-root` you can use `--no-path` to skip path dependencies: + +```bash +poetry install --no-path + By default `poetry` does not compile Python source files to bytecode during installation. This speeds up the installation process, but the first execution may take a little more time because Python then compiles source files to bytecode automatically. @@ -239,6 +244,7 @@ poetry install --compile The `--compile` option has no effect if `installer.modern-installation` is set to `false` because the old installer always compiles source files to bytecode. {{% /note %}} +``` ### Options @@ -248,6 +254,7 @@ is set to `false` because the old installer always compiles source files to byte * `--only-root`: Install only the root project, exclude all dependencies. * `--sync`: Synchronize the environment with the locked packages and the specified groups. * `--no-root`: Do not install the root package (your project). +* `--no-path`: Skip all path dependencies (including transitive ones). * `--dry-run`: Output the operations but do not execute anything (implicitly enables --verbose). * `--extras (-E)`: Features to install (multiple values allowed). * `--all-extras`: Install all extra features (conflicts with --extras). diff --git a/docs/faq.md b/docs/faq.md index 5280072037b..1bca6226ef9 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -152,3 +152,37 @@ This is done so to be compliant with the broader Python ecosystem. For example, if Poetry builds a distribution for a project that uses a version that is not valid according to [PEP 440](https://peps.python.org/pep-0440), third party tools will be unable to parse the version correctly. + + +### Poetry busts my Docker cache because it requires me to COPY my source files in before installing 3rd party dependencies + +By default running `poetry install ...` requires you to have your source files present (both the "root" package and any `path` dependencies you might have). +This interacts poorly with Docker's caching mechanisms because any change to the source file will make any layers (subsequent commands in your Dockerfile) re-run. +For example, you might have a Dockerfile that looks something like this: + +```text +FROM python +COPY pyproject.toml poetry.lock . +COPY src/ ./src +RUN pip install poetry && poetry install --no-dev +``` + +The `RUN` instruction will always re-run, which forces all 3rd party dependencies (likely the slowest step out of these) to re-run if you changed any files in `src/`. + +To avoid this cache busting you can split this into two steps: + +1. Install 3rd party dependencies. +2. Copy over your source code and install just the source code. + +This might look something like this: + +```text +FROM python +COPY pyproject.toml poetry.lock . +RUN pip install poetry && poetry install --no-root --no-path +COPY src/ ./src +RUN poetry install --no-dev +``` + +The two key options we are using here are `--no-root` (skips installing the project source) and `--no-path` (skips installing any local path dependencies, you can skip this if you don't have any). +[More information on the options available for `poetry install`]({{< relref "cli#install" >}}). diff --git a/tests/console/commands/test_install.py b/tests/console/commands/test_install.py index 0200a452bfa..5dbc5787c8e 100644 --- a/tests/console/commands/test_install.py +++ b/tests/console/commands/test_install.py @@ -347,3 +347,15 @@ def test_install_path_dependency_does_not_exist( else: with pytest.raises(ValueError, match="does not exist"): tester.execute(options) + + +def test_no_path_is_passed_to_installer(tester: CommandTester, mocker: MockerFixture): + """ + The --no-root options is passed to the installer. + """ + + mocker.patch.object(tester.command.installer, "run", return_value=1) + + tester.execute("--no-path") + + assert tester.command.installer._skip_path is True diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index 76e11ecd611..6947a5fc863 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -60,12 +60,12 @@ class Executor(BaseExecutor): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._installs: list[DependencyPackage] = [] + self._installs: list[Package] = [] self._updates: list[DependencyPackage] = [] self._uninstalls: list[DependencyPackage] = [] @property - def installations(self) -> list[DependencyPackage]: + def installations(self) -> list[Package]: return self._installs @property @@ -1283,6 +1283,49 @@ def test_run_installs_with_local_poetry_directory_transitive( assert installer.executor.installations_count == 6 +def test_run_installs_with_local_poetry_directory_transitive_no_path( + installer: Installer, + locker: Locker, + repo: Repository, + package: ProjectPackage, + tmpdir: Path, + fixture_dir: FixtureDirGetter, +): + """When we set Installer.skip_path(True) no path dependencies should + be installed (including transitive dependencies) + """ + root_dir = fixture_dir("directory") + package.root_dir = root_dir + locker.set_lock_path(root_dir) + directory = root_dir.joinpath("project_with_transitive_directory_dependencies") + package.add_dependency( + Factory.create_dependency( + "project-with-transitive-directory-dependencies", + {"path": str(directory.relative_to(root_dir))}, + root_dir=root_dir, + ) + ) + + repo.add_package(get_package("pendulum", "1.4.4")) + repo.add_package(get_package("cachy", "0.2.0")) + + installer.skip_path(True) + + installer.run() + + executor: Executor = installer.executor # type: ignore + + expected = fixture("with-directory-dependency-poetry-transitive") + + assert locker.written_data == expected + + directory_installs = [ + p for p in executor.installations if p.source_type == "directory" + ] + + assert not directory_installs, directory_installs + + def test_run_installs_with_local_poetry_file_transitive( installer: Installer, locker: Locker, From d2b9b00c8c79437e4980cb43b728a1003e9119db Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 28 Dec 2022 05:12:44 -0500 Subject: [PATCH 05/19] Clarify cache busting cause --- docs/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index 1bca6226ef9..3ef0b7e8dc4 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -167,7 +167,7 @@ COPY src/ ./src RUN pip install poetry && poetry install --no-dev ``` -The `RUN` instruction will always re-run, which forces all 3rd party dependencies (likely the slowest step out of these) to re-run if you changed any files in `src/`. +As soon as *any* source file changes the cache for the `RUN` layer will be invalidated, which forces all 3rd party dependencies (likely the slowest step out of these) to re-run if you changed any files in `src/`. To avoid this cache busting you can split this into two steps: From d9bd5016e208b7fe43a2de9b68df44b033ea0a8a Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 28 Dec 2022 06:23:48 -0500 Subject: [PATCH 06/19] lint --- tests/console/commands/test_install.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/console/commands/test_install.py b/tests/console/commands/test_install.py index 5dbc5787c8e..b40e670ed9b 100644 --- a/tests/console/commands/test_install.py +++ b/tests/console/commands/test_install.py @@ -285,6 +285,18 @@ def test_dry_run_populates_installer(tester: CommandTester, mocker: MockerFixtur assert tester.command.installer._dry_run is True +def test_no_path_is_passed_to_installer(tester: CommandTester, mocker: MockerFixture): + """ + The --no-root options is passed to the installer. + """ + + mocker.patch.object(tester.command.installer, "run", return_value=1) + + tester.execute("--no-path") + + assert tester.command.installer._skip_path is True + + def test_dry_run_does_not_build(tester: CommandTester, mocker: MockerFixture): mocker.patch.object(tester.command.installer, "run", return_value=0) mocked_editable_builder = mocker.patch( From 7e24af13c970e8d7212a3cea05fc0f8c39da7777 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 01:00:25 +0000 Subject: [PATCH 07/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/poetry/console/commands/install.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/poetry/console/commands/install.py b/src/poetry/console/commands/install.py index 8e678a59cb0..7cde2f468b1 100644 --- a/src/poetry/console/commands/install.py +++ b/src/poetry/console/commands/install.py @@ -33,8 +33,10 @@ class InstallCommand(InstallerCommand): option( "no-path", None, - "Do not install any path dependencies " - "(useful to install dependencies without source code, e.g. for caching)", + ( + "Do not install any path dependencies " + "(useful to install dependencies without source code, e.g. for caching)" + ), flag=True, multiple=False, ), From 97b890ddfe9c4116111b60e0d1e7eac6eb9e9c50 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 6 Mar 2023 16:21:28 -0600 Subject: [PATCH 08/19] Rename --no-path -> --no-directory --- docs/cli.md | 9 +++++---- docs/faq.md | 4 ++-- src/poetry/console/commands/install.py | 9 +++++---- src/poetry/installation/installer.py | 8 ++++---- src/poetry/puzzle/transaction.py | 4 ++-- tests/console/commands/test_install.py | 16 ++-------------- tests/installation/test_installer.py | 4 ++-- 7 files changed, 22 insertions(+), 32 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 073d975fee9..191ce2c2883 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -225,10 +225,11 @@ If you want to skip this installation, use the `--no-root` option. poetry install --no-root ``` -Similar to `--no-root` you can use `--no-path` to skip path dependencies: +Similar to `--no-root` you can use `--no-directory` to skip path dependencies: ```bash -poetry install --no-path +poetry install --no-directory +``` By default `poetry` does not compile Python source files to bytecode during installation. This speeds up the installation process, but the first execution may take a little more @@ -244,7 +245,7 @@ poetry install --compile The `--compile` option has no effect if `installer.modern-installation` is set to `false` because the old installer always compiles source files to bytecode. {{% /note %}} -``` + ### Options @@ -254,7 +255,7 @@ is set to `false` because the old installer always compiles source files to byte * `--only-root`: Install only the root project, exclude all dependencies. * `--sync`: Synchronize the environment with the locked packages and the specified groups. * `--no-root`: Do not install the root package (your project). -* `--no-path`: Skip all path dependencies (including transitive ones). +* `--no-directory`: Skip all directory path dependencies (including transitive ones). * `--dry-run`: Output the operations but do not execute anything (implicitly enables --verbose). * `--extras (-E)`: Features to install (multiple values allowed). * `--all-extras`: Install all extra features (conflicts with --extras). diff --git a/docs/faq.md b/docs/faq.md index 3ef0b7e8dc4..3f5fc8533a5 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -179,10 +179,10 @@ This might look something like this: ```text FROM python COPY pyproject.toml poetry.lock . -RUN pip install poetry && poetry install --no-root --no-path +RUN pip install poetry && poetry install --no-root --no-directory COPY src/ ./src RUN poetry install --no-dev ``` -The two key options we are using here are `--no-root` (skips installing the project source) and `--no-path` (skips installing any local path dependencies, you can skip this if you don't have any). +The two key options we are using here are `--no-root` (skips installing the project source) and `--no-directory` (skips installing any local directory path dependencies, you can skip this if you don't have any). [More information on the options available for `poetry install`]({{< relref "cli#install" >}}). diff --git a/src/poetry/console/commands/install.py b/src/poetry/console/commands/install.py index 7cde2f468b1..9bebfba3ba0 100644 --- a/src/poetry/console/commands/install.py +++ b/src/poetry/console/commands/install.py @@ -31,11 +31,12 @@ class InstallCommand(InstallerCommand): "no-root", None, "Do not install the root package (the current project)." ), option( - "no-path", + "no-directory", None, ( - "Do not install any path dependencies " - "(useful to install dependencies without source code, e.g. for caching)" + "Do not install any directory path dependencies (ones using `package =" + ' { path = "..." }`\'; useful to install dependencies without source' + " code, e.g. for caching of Docker layers)" ), flag=True, multiple=False, @@ -158,7 +159,7 @@ def handle(self) -> int: with_synchronization = True self.installer.only_groups(self.activated_groups) - self.installer.skip_path(self.option("no-path")) + self.installer.skip_directory(self.option("no-directory")) self.installer.dry_run(self.option("dry-run")) self.installer.requires_synchronization(with_synchronization) self.installer.executor.enable_bytecode_compilation(self.option("compile")) diff --git a/src/poetry/installation/installer.py b/src/poetry/installation/installer.py index bb39f93fc70..dcd8ef2df31 100644 --- a/src/poetry/installation/installer.py +++ b/src/poetry/installation/installer.py @@ -59,7 +59,7 @@ def __init__( self._verbose = False self._write_lock = True self._groups: Iterable[str] | None = None - self._skip_path = False + self._skip_directory = False self._execute_operations = True self._lock = False @@ -151,8 +151,8 @@ def update(self, update: bool = True) -> Installer: return self - def skip_path(self, skip_path: bool = False) -> Installer: - self._skip_path = skip_path + def skip_directory(self, skip_directory: bool = False) -> Installer: + self._skip_directory = skip_directory return self @@ -342,7 +342,7 @@ def _do_install(self) -> int: ops = solver.solve(use_latest=self._whitelist).calculate_operations( with_uninstalls=self._requires_synchronization, synchronize=self._requires_synchronization, - skip_path=self._skip_path, + skip_directory=self._skip_directory, ) if not self._requires_synchronization: diff --git a/src/poetry/puzzle/transaction.py b/src/poetry/puzzle/transaction.py index 947c0af5798..679baa98980 100644 --- a/src/poetry/puzzle/transaction.py +++ b/src/poetry/puzzle/transaction.py @@ -30,7 +30,7 @@ def calculate_operations( self, with_uninstalls: bool = True, synchronize: bool = False, - skip_path: bool = False, + skip_directory: bool = False, ) -> list[Operation]: from poetry.installation.operations import Install from poetry.installation.operations import Uninstall @@ -74,7 +74,7 @@ def calculate_operations( break if not installed and ( - not skip_path or result_package.source_type != "directory" + not skip_directory or result_package.source_type != "directory" ): operations.append(Install(result_package, priority=priority)) diff --git a/tests/console/commands/test_install.py b/tests/console/commands/test_install.py index b40e670ed9b..6b7557322c8 100644 --- a/tests/console/commands/test_install.py +++ b/tests/console/commands/test_install.py @@ -285,18 +285,6 @@ def test_dry_run_populates_installer(tester: CommandTester, mocker: MockerFixtur assert tester.command.installer._dry_run is True -def test_no_path_is_passed_to_installer(tester: CommandTester, mocker: MockerFixture): - """ - The --no-root options is passed to the installer. - """ - - mocker.patch.object(tester.command.installer, "run", return_value=1) - - tester.execute("--no-path") - - assert tester.command.installer._skip_path is True - - def test_dry_run_does_not_build(tester: CommandTester, mocker: MockerFixture): mocker.patch.object(tester.command.installer, "run", return_value=0) mocked_editable_builder = mocker.patch( @@ -368,6 +356,6 @@ def test_no_path_is_passed_to_installer(tester: CommandTester, mocker: MockerFix mocker.patch.object(tester.command.installer, "run", return_value=1) - tester.execute("--no-path") + tester.execute("--no-directory") - assert tester.command.installer._skip_path is True + assert tester.command.installer._skip_directory is True diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index 6947a5fc863..9462ff8cb5c 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -1291,7 +1291,7 @@ def test_run_installs_with_local_poetry_directory_transitive_no_path( tmpdir: Path, fixture_dir: FixtureDirGetter, ): - """When we set Installer.skip_path(True) no path dependencies should + """When we set Installer.skip_directory(True) no path dependencies should be installed (including transitive dependencies) """ root_dir = fixture_dir("directory") @@ -1309,7 +1309,7 @@ def test_run_installs_with_local_poetry_directory_transitive_no_path( repo.add_package(get_package("pendulum", "1.4.4")) repo.add_package(get_package("cachy", "0.2.0")) - installer.skip_path(True) + installer.skip_directory(True) installer.run() From 5f8bd7ae8eef28eff1cc9d13d5276d81036de3b5 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 5 Apr 2023 14:43:58 +0200 Subject: [PATCH 09/19] Update docs/faq.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Randy Döring <30527984+radoering@users.noreply.github.com> --- docs/faq.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index d318ee7d4c6..83b7a5c4f32 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -160,7 +160,7 @@ For example, if Poetry builds a distribution for a project that uses a version t ### Poetry busts my Docker cache because it requires me to COPY my source files in before installing 3rd party dependencies -By default running `poetry install ...` requires you to have your source files present (both the "root" package and any `path` dependencies you might have). +By default running `poetry install ...` requires you to have your source files present (both the "root" package and any directory path dependencies you might have). This interacts poorly with Docker's caching mechanisms because any change to the source file will make any layers (subsequent commands in your Dockerfile) re-run. For example, you might have a Dockerfile that looks something like this: From 06332d855fd80d94718f35e12e930f88a8d99929 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 5 Apr 2023 15:04:05 +0200 Subject: [PATCH 10/19] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Randy Döring <30527984+radoering@users.noreply.github.com> --- docs/faq.md | 6 +++--- src/poetry/console/commands/install.py | 3 +-- src/poetry/puzzle/transaction.py | 1 + tests/console/commands/test_install.py | 2 +- tests/installation/test_installer.py | 1 - 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 83b7a5c4f32..b0abd90fee9 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -161,7 +161,7 @@ For example, if Poetry builds a distribution for a project that uses a version t ### Poetry busts my Docker cache because it requires me to COPY my source files in before installing 3rd party dependencies By default running `poetry install ...` requires you to have your source files present (both the "root" package and any directory path dependencies you might have). -This interacts poorly with Docker's caching mechanisms because any change to the source file will make any layers (subsequent commands in your Dockerfile) re-run. +This interacts poorly with Docker's caching mechanisms because any change to a source file will make any layers (subsequent commands in your Dockerfile) re-run. For example, you might have a Dockerfile that looks something like this: ```text @@ -171,7 +171,7 @@ COPY src/ ./src RUN pip install poetry && poetry install --no-dev ``` -As soon as *any* source file changes the cache for the `RUN` layer will be invalidated, which forces all 3rd party dependencies (likely the slowest step out of these) to re-run if you changed any files in `src/`. +As soon as *any* source file changes, the cache for the `RUN` layer will be invalidated, which forces all 3rd party dependencies (likely the slowest step out of these) to be installed again if you changed any files in `src/`. To avoid this cache busting you can split this into two steps: @@ -188,5 +188,5 @@ COPY src/ ./src RUN poetry install --no-dev ``` -The two key options we are using here are `--no-root` (skips installing the project source) and `--no-directory` (skips installing any local directory path dependencies, you can skip this if you don't have any). +The two key options we are using here are `--no-root` (skips installing the project source) and `--no-directory` (skips installing any local directory path dependencies, you can omit this if you don't have any). [More information on the options available for `poetry install`]({{< relref "cli#install" >}}). diff --git a/src/poetry/console/commands/install.py b/src/poetry/console/commands/install.py index 9bebfba3ba0..45bb76e4036 100644 --- a/src/poetry/console/commands/install.py +++ b/src/poetry/console/commands/install.py @@ -34,8 +34,7 @@ class InstallCommand(InstallerCommand): "no-directory", None, ( - "Do not install any directory path dependencies (ones using `package =" - ' { path = "..." }`\'; useful to install dependencies without source' + "Do not install any directory path dependencies; useful to install dependencies without source' " code, e.g. for caching of Docker layers)" ), flag=True, diff --git a/src/poetry/puzzle/transaction.py b/src/poetry/puzzle/transaction.py index 679baa98980..3bbdf91003a 100644 --- a/src/poetry/puzzle/transaction.py +++ b/src/poetry/puzzle/transaction.py @@ -30,6 +30,7 @@ def calculate_operations( self, with_uninstalls: bool = True, synchronize: bool = False, + *, skip_directory: bool = False, ) -> list[Operation]: from poetry.installation.operations import Install diff --git a/tests/console/commands/test_install.py b/tests/console/commands/test_install.py index 6b7557322c8..f99674eb26a 100644 --- a/tests/console/commands/test_install.py +++ b/tests/console/commands/test_install.py @@ -351,7 +351,7 @@ def test_install_path_dependency_does_not_exist( def test_no_path_is_passed_to_installer(tester: CommandTester, mocker: MockerFixture): """ - The --no-root options is passed to the installer. + The --no-directory option is passed to the installer. """ mocker.patch.object(tester.command.installer, "run", return_value=1) diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index 9462ff8cb5c..409777816e5 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -1288,7 +1288,6 @@ def test_run_installs_with_local_poetry_directory_transitive_no_path( locker: Locker, repo: Repository, package: ProjectPackage, - tmpdir: Path, fixture_dir: FixtureDirGetter, ): """When we set Installer.skip_directory(True) no path dependencies should From deb34d5f988b3c229bd564409390c0af508ad5d0 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 5 Apr 2023 15:14:05 +0200 Subject: [PATCH 11/19] add note in cli.md linking to faq.md --- docs/cli.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/cli.md b/docs/cli.md index 191ce2c2883..890c6b9ef7b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -231,6 +231,8 @@ Similar to `--no-root` you can use `--no-directory` to skip path dependencies: poetry install --no-directory ``` +This is mainly useful for caching in CI or when building Docker images. See the [FAQ entry]({{< relref "faq#poetry-busts-my-docker-cache-because-it-requires-me-to-copy-my-source-files-in-before-installing-3rd-party-dependencies" >}}) for more information on this option. + By default `poetry` does not compile Python source files to bytecode during installation. This speeds up the installation process, but the first execution may take a little more time because Python then compiles source files to bytecode automatically. From 528c6874a9a29c6c2da7fb7143345dd2a4787a10 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 5 Apr 2023 16:52:55 +0200 Subject: [PATCH 12/19] run formatting --- src/poetry/console/commands/install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/poetry/console/commands/install.py b/src/poetry/console/commands/install.py index 45bb76e4036..90003d5793e 100644 --- a/src/poetry/console/commands/install.py +++ b/src/poetry/console/commands/install.py @@ -34,8 +34,8 @@ class InstallCommand(InstallerCommand): "no-directory", None, ( - "Do not install any directory path dependencies; useful to install dependencies without source' - " code, e.g. for caching of Docker layers)" + "Do not install any directory path dependencies; useful to install" + " dependencies without source code, e.g. for caching of Docker layers)" ), flag=True, multiple=False, From 017ace11d5bfbec7c9e07f2e0be8cad7f50ef3b2 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 5 Apr 2023 16:54:08 +0200 Subject: [PATCH 13/19] rename test_no_path_is_passed_to_installer -> test_no_directory_is_passed_to_installer --- tests/console/commands/test_install.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/console/commands/test_install.py b/tests/console/commands/test_install.py index f99674eb26a..9390663f8c9 100644 --- a/tests/console/commands/test_install.py +++ b/tests/console/commands/test_install.py @@ -349,7 +349,9 @@ def test_install_path_dependency_does_not_exist( tester.execute(options) -def test_no_path_is_passed_to_installer(tester: CommandTester, mocker: MockerFixture): +def test_no_directory_is_passed_to_installer( + tester: CommandTester, mocker: MockerFixture +): """ The --no-directory option is passed to the installer. """ From 81868bb5d43cb255ee23ebc357a5f334f61ea02e Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 5 Apr 2023 16:56:40 +0200 Subject: [PATCH 14/19] move test_no_directory_is_passed_to_installer and add negative test case --- tests/console/commands/test_install.py | 32 +++++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/console/commands/test_install.py b/tests/console/commands/test_install.py index 9390663f8c9..61bce38f8ba 100644 --- a/tests/console/commands/test_install.py +++ b/tests/console/commands/test_install.py @@ -180,6 +180,24 @@ def test_compile_option_is_passed_to_the_installer( enable_bytecode_compilation_mock.assert_called_once_with(compile) +@pytest.mark.parametrize("skip_directory_cli_value", [True, False]) +def test_no_directory_is_passed_to_installer( + tester: CommandTester, mocker: MockerFixture, skip_directory_cli_value: bool +): + """ + The --no-directory option is passed to the installer. + """ + + mocker.patch.object(tester.command.installer, "run", return_value=1) + + if skip_directory_cli_value is True: + tester.execute("--no-directory") + else: + tester.execute() + + assert tester.command.installer._skip_directory is skip_directory_cli_value + + def test_no_all_extras_doesnt_populate_installer( tester: CommandTester, mocker: MockerFixture ): @@ -347,17 +365,3 @@ def test_install_path_dependency_does_not_exist( else: with pytest.raises(ValueError, match="does not exist"): tester.execute(options) - - -def test_no_directory_is_passed_to_installer( - tester: CommandTester, mocker: MockerFixture -): - """ - The --no-directory option is passed to the installer. - """ - - mocker.patch.object(tester.command.installer, "run", return_value=1) - - tester.execute("--no-directory") - - assert tester.command.installer._skip_directory is True From 23d18d2324757f0431e244c0683f5865c951ebe3 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:01:42 +0200 Subject: [PATCH 15/19] Collapse tests into test_run_installs_with_local_poetry_directory_and_skip_directory_flag --- tests/installation/test_installer.py | 43 ++++++---------------------- 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index 409777816e5..2b3344f3d40 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -1251,44 +1251,14 @@ def test_run_installs_with_local_poetry_directory_and_extras( assert installer.executor.installations_count == 2 -def test_run_installs_with_local_poetry_directory_transitive( - installer: Installer, - locker: Locker, - repo: Repository, - package: ProjectPackage, - tmpdir: Path, - fixture_dir: FixtureDirGetter, -): - root_dir = fixture_dir("directory") - package.root_dir = root_dir - locker.set_lock_path(root_dir) - directory = root_dir.joinpath("project_with_transitive_directory_dependencies") - package.add_dependency( - Factory.create_dependency( - "project-with-transitive-directory-dependencies", - {"path": str(directory.relative_to(root_dir))}, - root_dir=root_dir, - ) - ) - - repo.add_package(get_package("pendulum", "1.4.4")) - repo.add_package(get_package("cachy", "0.2.0")) - - installer.run() - - expected = fixture("with-directory-dependency-poetry-transitive") - - assert locker.written_data == expected - - assert installer.executor.installations_count == 6 - - -def test_run_installs_with_local_poetry_directory_transitive_no_path( +@pytest.mark.parametrize("skip_directory", [True, False]) +def test_run_installs_with_local_poetry_directory_and_skip_directory_flag( installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage, fixture_dir: FixtureDirGetter, + skip_directory: bool, ): """When we set Installer.skip_directory(True) no path dependencies should be installed (including transitive dependencies) @@ -1308,7 +1278,7 @@ def test_run_installs_with_local_poetry_directory_transitive_no_path( repo.add_package(get_package("pendulum", "1.4.4")) repo.add_package(get_package("cachy", "0.2.0")) - installer.skip_directory(True) + installer.skip_directory(skip_directory) installer.run() @@ -1322,7 +1292,10 @@ def test_run_installs_with_local_poetry_directory_transitive_no_path( p for p in executor.installations if p.source_type == "directory" ] - assert not directory_installs, directory_installs + if skip_directory: + assert not directory_installs, directory_installs + else: + assert len(directory_installs) == 4 def test_run_installs_with_local_poetry_file_transitive( From 7edaef68ebddca9a12e4ea32cc7f788bee74ed20 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:02:50 +0200 Subject: [PATCH 16/19] add directory_installs to debug assertion --- tests/installation/test_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index 2b3344f3d40..c2e1076fbc7 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -1295,7 +1295,7 @@ def test_run_installs_with_local_poetry_directory_and_skip_directory_flag( if skip_directory: assert not directory_installs, directory_installs else: - assert len(directory_installs) == 4 + assert len(directory_installs) == 4, directory_installs def test_run_installs_with_local_poetry_file_transitive( From 1d4cafc41c12e12efcb1af194db45943e04dbedb Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:04:39 +0200 Subject: [PATCH 17/19] change another path -> directory in docs --- docs/cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.md b/docs/cli.md index 890c6b9ef7b..9bef8f7a699 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -225,7 +225,7 @@ If you want to skip this installation, use the `--no-root` option. poetry install --no-root ``` -Similar to `--no-root` you can use `--no-directory` to skip path dependencies: +Similar to `--no-root` you can use `--no-directory` to skip directory path dependencies: ```bash poetry install --no-directory From 81ae993bd583921a96970711136f0b0e863c791f Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:45:27 +0200 Subject: [PATCH 18/19] fix test and be more explicit about expected results --- tests/installation/test_installer.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index 0759ab12288..beb121183e5 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -1315,13 +1315,22 @@ def test_run_installs_with_local_poetry_directory_and_skip_directory_flag( assert locker.written_data == expected directory_installs = [ - p for p in executor.installations if p.source_type == "directory" + p.name for p in executor.installations if p.source_type == "directory" ] if skip_directory: assert not directory_installs, directory_installs else: - assert len(directory_installs) == 4, directory_installs + assert directory_installs == [ + "inner-directory-project", + "project-with-extras", + "project-with-transitive-file-dependencies", + "project-with-transitive-directory-dependencies", + "inner-directory-project", + "project-with-extras", + "project-with-transitive-file-dependencies", + "project-with-transitive-directory-dependencies", + ] def test_run_installs_with_local_poetry_file_transitive( From c940cbfc851d1c01d8c8fdda36b03fdc00fcb85a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Randy=20D=C3=B6ring?= <30527984+radoering@users.noreply.github.com> Date: Mon, 10 Apr 2023 11:13:13 +0200 Subject: [PATCH 19/19] fix test, simplify condition --- src/poetry/puzzle/transaction.py | 5 +++-- tests/installation/test_installer.py | 16 ++++------------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/poetry/puzzle/transaction.py b/src/poetry/puzzle/transaction.py index 3bbdf91003a..665093416cb 100644 --- a/src/poetry/puzzle/transaction.py +++ b/src/poetry/puzzle/transaction.py @@ -74,8 +74,9 @@ def calculate_operations( break - if not installed and ( - not skip_directory or result_package.source_type != "directory" + if not ( + installed + or (skip_directory and result_package.source_type == "directory") ): operations.append(Install(result_package, priority=priority)) diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index beb121183e5..8a59e049fe9 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -1285,7 +1285,7 @@ def test_run_installs_with_local_poetry_directory_and_skip_directory_flag( skip_directory: bool, ): """When we set Installer.skip_directory(True) no path dependencies should - be installed (including transitive dependencies) + be installed (including transitive dependencies). """ root_dir = fixture_dir("directory") package.root_dir = root_dir @@ -1304,7 +1304,6 @@ def test_run_installs_with_local_poetry_directory_and_skip_directory_flag( installer.skip_directory(skip_directory) - installer.run() result = installer.run() assert result == 0 @@ -1320,17 +1319,10 @@ def test_run_installs_with_local_poetry_directory_and_skip_directory_flag( if skip_directory: assert not directory_installs, directory_installs + assert installer.executor.installations_count == 2 else: - assert directory_installs == [ - "inner-directory-project", - "project-with-extras", - "project-with-transitive-file-dependencies", - "project-with-transitive-directory-dependencies", - "inner-directory-project", - "project-with-extras", - "project-with-transitive-file-dependencies", - "project-with-transitive-directory-dependencies", - ] + assert directory_installs, directory_installs + assert installer.executor.installations_count == 6 def test_run_installs_with_local_poetry_file_transitive(