diff --git a/docs/cli.md b/docs/cli.md index f09ffb97273..9bef8f7a699 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -225,6 +225,14 @@ If you want to skip this installation, use the `--no-root` option. poetry install --no-root ``` +Similar to `--no-root` you can use `--no-directory` to skip directory path dependencies: + +```bash +poetry install --no-directory +``` + +This is mainly useful for caching in CI or when building Docker images. See the [FAQ entry]({{< relref "faq#poetry-busts-my-docker-cache-because-it-requires-me-to-copy-my-source-files-in-before-installing-3rd-party-dependencies" >}}) for more information on this option. + By default `poetry` does not compile Python source files to bytecode during installation. This speeds up the installation process, but the first execution may take a little more time because Python then compiles source files to bytecode automatically. @@ -240,6 +248,7 @@ The `--compile` option has no effect if `installer.modern-installation` is set to `false` because the old installer always compiles source files to bytecode. {{% /note %}} + ### Options * `--without`: The dependency groups to ignore. @@ -248,6 +257,7 @@ is set to `false` because the old installer always compiles source files to byte * `--only-root`: Install only the root project, exclude all dependencies. * `--sync`: Synchronize the environment with the locked packages and the specified groups. * `--no-root`: Do not install the root package (your project). +* `--no-directory`: Skip all directory path dependencies (including transitive ones). * `--dry-run`: Output the operations but do not execute anything (implicitly enables --verbose). * `--extras (-E)`: Features to install (multiple values allowed). * `--all-extras`: Install all extra features (conflicts with --extras). diff --git a/docs/faq.md b/docs/faq.md index 0a8b97b3075..71ac22c1c2d 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -189,3 +189,37 @@ This is done so to be compliant with the broader Python ecosystem. For example, if Poetry builds a distribution for a project that uses a version that is not valid according to [PEP 440](https://peps.python.org/pep-0440), third party tools will be unable to parse the version correctly. + + +### Poetry busts my Docker cache because it requires me to COPY my source files in before installing 3rd party dependencies + +By default running `poetry install ...` requires you to have your source files present (both the "root" package and any directory path dependencies you might have). +This interacts poorly with Docker's caching mechanisms because any change to a source file will make any layers (subsequent commands in your Dockerfile) re-run. +For example, you might have a Dockerfile that looks something like this: + +```text +FROM python +COPY pyproject.toml poetry.lock . +COPY src/ ./src +RUN pip install poetry && poetry install --no-dev +``` + +As soon as *any* source file changes, the cache for the `RUN` layer will be invalidated, which forces all 3rd party dependencies (likely the slowest step out of these) to be installed again if you changed any files in `src/`. + +To avoid this cache busting you can split this into two steps: + +1. Install 3rd party dependencies. +2. Copy over your source code and install just the source code. + +This might look something like this: + +```text +FROM python +COPY pyproject.toml poetry.lock . +RUN pip install poetry && poetry install --no-root --no-directory +COPY src/ ./src +RUN poetry install --no-dev +``` + +The two key options we are using here are `--no-root` (skips installing the project source) and `--no-directory` (skips installing any local directory path dependencies, you can omit this if you don't have any). +[More information on the options available for `poetry install`]({{< relref "cli#install" >}}). diff --git a/src/poetry/console/commands/install.py b/src/poetry/console/commands/install.py index 844ec37176f..90003d5793e 100644 --- a/src/poetry/console/commands/install.py +++ b/src/poetry/console/commands/install.py @@ -30,6 +30,16 @@ class InstallCommand(InstallerCommand): option( "no-root", None, "Do not install the root package (the current project)." ), + option( + "no-directory", + None, + ( + "Do not install any directory path dependencies; useful to install" + " dependencies without source code, e.g. for caching of Docker layers)" + ), + flag=True, + multiple=False, + ), option( "dry-run", None, @@ -148,6 +158,7 @@ def handle(self) -> int: with_synchronization = True self.installer.only_groups(self.activated_groups) + self.installer.skip_directory(self.option("no-directory")) self.installer.dry_run(self.option("dry-run")) self.installer.requires_synchronization(with_synchronization) self.installer.executor.enable_bytecode_compilation(self.option("compile")) diff --git a/src/poetry/installation/installer.py b/src/poetry/installation/installer.py index ec5911ce8f7..1df9a887853 100644 --- a/src/poetry/installation/installer.py +++ b/src/poetry/installation/installer.py @@ -59,6 +59,7 @@ def __init__( self._verbose = False self._write_lock = True self._groups: Iterable[str] | None = None + self._skip_directory = False self._execute_operations = True self._lock = False @@ -150,6 +151,11 @@ def update(self, update: bool = True) -> Installer: return self + def skip_directory(self, skip_directory: bool = False) -> Installer: + self._skip_directory = skip_directory + + return self + def lock(self, update: bool = True) -> Installer: """ Prepare the installer for locking only. @@ -334,6 +340,7 @@ def _do_install(self) -> int: ops = solver.solve(use_latest=self._whitelist).calculate_operations( with_uninstalls=self._requires_synchronization, synchronize=self._requires_synchronization, + skip_directory=self._skip_directory, ) if not self._requires_synchronization: diff --git a/src/poetry/puzzle/transaction.py b/src/poetry/puzzle/transaction.py index 74d0d6c5e61..665093416cb 100644 --- a/src/poetry/puzzle/transaction.py +++ b/src/poetry/puzzle/transaction.py @@ -27,7 +27,11 @@ def __init__( self._root_package = root_package def calculate_operations( - self, with_uninstalls: bool = True, synchronize: bool = False + self, + with_uninstalls: bool = True, + synchronize: bool = False, + *, + skip_directory: bool = False, ) -> list[Operation]: from poetry.installation.operations import Install from poetry.installation.operations import Uninstall @@ -70,7 +74,10 @@ def calculate_operations( break - if not installed: + if not ( + installed + or (skip_directory and result_package.source_type == "directory") + ): operations.append(Install(result_package, priority=priority)) if with_uninstalls: diff --git a/tests/console/commands/test_install.py b/tests/console/commands/test_install.py index f39081872d0..a1e9379de49 100644 --- a/tests/console/commands/test_install.py +++ b/tests/console/commands/test_install.py @@ -184,6 +184,24 @@ def test_compile_option_is_passed_to_the_installer( enable_bytecode_compilation_mock.assert_called_once_with(compile) +@pytest.mark.parametrize("skip_directory_cli_value", [True, False]) +def test_no_directory_is_passed_to_installer( + tester: CommandTester, mocker: MockerFixture, skip_directory_cli_value: bool +): + """ + The --no-directory option is passed to the installer. + """ + + mocker.patch.object(tester.command.installer, "run", return_value=1) + + if skip_directory_cli_value is True: + tester.execute("--no-directory") + else: + tester.execute() + + assert tester.command.installer._skip_directory is skip_directory_cli_value + + def test_no_all_extras_doesnt_populate_installer( tester: CommandTester, mocker: MockerFixture ): diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index 400278e7798..4d6e975b45a 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -60,12 +60,12 @@ class Executor(BaseExecutor): def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._installs: list[DependencyPackage] = [] + self._installs: list[Package] = [] self._updates: list[DependencyPackage] = [] self._uninstalls: list[DependencyPackage] = [] @property - def installations(self) -> list[DependencyPackage]: + def installations(self) -> list[Package]: return self._installs @property @@ -1276,14 +1276,18 @@ def test_run_installs_with_local_poetry_directory_and_extras( assert installer.executor.installations_count == 2 -def test_run_installs_with_local_poetry_directory_transitive( +@pytest.mark.parametrize("skip_directory", [True, False]) +def test_run_installs_with_local_poetry_directory_and_skip_directory_flag( installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage, - tmpdir: Path, fixture_dir: FixtureDirGetter, + skip_directory: bool, ): + """When we set Installer.skip_directory(True) no path dependencies should + be installed (including transitive dependencies). + """ root_dir = fixture_dir("directory") package.root_dir = root_dir locker.set_lock_path(root_dir) @@ -1299,14 +1303,27 @@ def test_run_installs_with_local_poetry_directory_transitive( repo.add_package(get_package("pendulum", "1.4.4")) repo.add_package(get_package("cachy", "0.2.0")) + installer.skip_directory(skip_directory) + result = installer.run() assert result == 0 + executor: Executor = installer.executor # type: ignore + expected = fixture("with-directory-dependency-poetry-transitive") assert locker.written_data == expected - assert installer.executor.installations_count == 6 + directory_installs = [ + p.name for p in executor.installations if p.source_type == "directory" + ] + + if skip_directory: + assert not directory_installs, directory_installs + assert installer.executor.installations_count == 2 + else: + assert directory_installs, directory_installs + assert installer.executor.installations_count == 6 def test_run_installs_with_local_poetry_file_transitive(