diff --git a/CHANGELOG.md b/CHANGELOG.md index e375aeeffb6..ec01b75abde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Change Log +## [1.1.3] - 2020-10-14 + +### Changed + +- Python version support deprecation warning is now written to `stderr`. ([#3131](https://github.com/python-poetry/poetry/pull/3131)) + +### Fixed + +- Fixed `KeyError` when `PATH` is not defined in environment variables. ([#3159](https://github.com/python-poetry/poetry/pull/3159)) +- Fixed error when using `config` command in a directory with an existing `pyproject.toml` without any Poetry configuration. ([#3172](https://github.com/python-poetry/poetry/pull/3172)) +- Fixed incorrect inspection of package requirements when same dependency is specified multiple times with unique markers. ([#3147](https://github.com/python-poetry/poetry/pull/3147)) +- Fixed `show` command to use already resolved package metadata. ([#3117](https://github.com/python-poetry/poetry/pull/3117)) +- Fixed multiple issues with `export` command output when using `requirements.txt` format. ([#3119](https://github.com/python-poetry/poetry/pull/3119)) + +## [1.1.2] - 2020-10-06 + +### Changed +- Dependency installation of editable packages and all uninstall operations are now performed serially within their corresponding priority groups. ([#3099](https://github.com/python-poetry/poetry/pull/3099)) +- Improved package metadata inspection of nested poetry projects within project path dependencies. ([#3105](https://github.com/python-poetry/poetry/pull/3105)) + +### Fixed + +- Fixed export of `requirements.txt` when project dependency contains git dependencies. ([#3100](https://github.com/python-poetry/poetry/pull/3100)) + +## [1.1.1] - 2020-10-05 + +### Added + +- Added `--no-update` option to `lock` command. ([#3034](https://github.com/python-poetry/poetry/pull/3034)) + +### Fixed + +- Fixed resolution of packages with missing required extras. ([#3035](https://github.com/python-poetry/poetry/pull/3035)) +- Fixed export of `requirements.txt` dependencies to include development dependencies. ([#3024](https://github.com/python-poetry/poetry/pull/3024)) +- Fixed incorrect selection of unsupported binary distribution formats when selecting a package artifact to install. ([#3058](https://github.com/python-poetry/poetry/pull/3058)) +- Fixed incorrect use of system executable when building package distributions via `build` command. ([#3056](https://github.com/python-poetry/poetry/pull/3056)) +- Fixed errors in `init` command when specifying `--dependency` in non-interactive mode when a `pyproject.toml` file already exists. ([#3076](https://github.com/python-poetry/poetry/pull/3076)) +- Fixed incorrect selection of configured source url when a publish repository url configuration with the same name already exists. ([#3047](https://github.com/python-poetry/poetry/pull/3047)) +- Fixed dependency resolution issues when the same package is specified in multiple dependency extras. ([#3046](https://github.com/python-poetry/poetry/pull/3046)) + ## [1.1.0] - 2020-10-01 ### Changed @@ -1023,7 +1063,10 @@ Initial release -[Unreleased]: https://github.com/python-poetry/poetry/compare/1.1.0...master +[Unreleased]: https://github.com/python-poetry/poetry/compare/1.1.3...master +[1.1.3]: https://github.com/python-poetry/poetry/compare/1.1.3 +[1.1.2]: https://github.com/python-poetry/poetry/releases/tag/1.1.2 +[1.1.1]: https://github.com/python-poetry/poetry/releases/tag/1.1.1 [1.1.0]: https://github.com/python-poetry/poetry/releases/tag/1.1.0 [1.1.0rc1]: https://github.com/python-poetry/poetry/releases/tag/1.1.0rc1 [1.1.0b4]: https://github.com/python-poetry/poetry/releases/tag/1.1.0b4 diff --git a/poetry/__version__.py b/poetry/__version__.py index 6849410aae0..bc0be1b6fa5 100644 --- a/poetry/__version__.py +++ b/poetry/__version__.py @@ -1 +1 @@ -__version__ = "1.1.0" +__version__ = "1.2.0a0" diff --git a/poetry/console/commands/show.py b/poetry/console/commands/show.py index 808122e64dc..f6f42f42654 100644 --- a/poetry/console/commands/show.py +++ b/poetry/console/commands/show.py @@ -68,7 +68,7 @@ def handle(self): table = self.table(style="compact") # table.style.line_vc_char = "" locked_packages = locked_repo.packages - pool = Pool() + pool = Pool(ignore_repository_names=True) pool.add_repository(locked_repo) solver = Solver( self.poetry.package, diff --git a/poetry/inspection/info.py b/poetry/inspection/info.py index 73034de16d3..251dac95f91 100644 --- a/poetry/inspection/info.py +++ b/poetry/inspection/info.py @@ -165,6 +165,8 @@ def to_package( package.requires = poetry_package.requires return package + seen_requirements = set() + for req in self.requires_dist or []: try: # Attempt to parse the PEP-508 requirement string @@ -191,8 +193,11 @@ def to_package( package.extras[extra].append(dependency) - if dependency not in package.requires: + req = dependency.to_pep_508(with_extras=True) + + if req not in seen_requirements: package.requires.append(dependency) + seen_requirements.add(req) return package diff --git a/poetry/packages/locker.py b/poetry/packages/locker.py index 2283a2373d2..9dd75e66519 100644 --- a/poetry/packages/locker.py +++ b/poetry/packages/locker.py @@ -1,4 +1,3 @@ -import itertools import json import logging import os @@ -6,9 +5,15 @@ from copy import deepcopy from hashlib import sha256 -from typing import Any +from typing import Dict +from typing import Iterable +from typing import Iterator from typing import List from typing import Optional +from typing import Sequence +from typing import Set +from typing import Tuple +from typing import Union from tomlkit import array from tomlkit import document @@ -25,8 +30,10 @@ from poetry.core.semver.version import Version from poetry.core.toml.file import TOMLFile from poetry.core.version.markers import parse_marker +from poetry.packages import DependencyPackage from poetry.utils._compat import OrderedDict from poetry.utils._compat import Path +from poetry.utils.extras import get_extra_package_names logger = logging.getLogger(__name__) @@ -181,40 +188,118 @@ def locked_repository( return packages - def get_project_dependencies( - self, project_requires, pinned_versions=False, with_nested=False, with_dev=False - ): # type: (List[Dependency], bool, bool, bool) -> Any - packages = self.locked_repository(with_dev).packages + @staticmethod + def __get_locked_package( + _dependency, packages_by_name + ): # type: (Dependency, Dict[str, List[Package]]) -> Optional[Package] + """ + Internal helper to identify corresponding locked package using dependency + version constraints. + """ + for _package in packages_by_name.get(_dependency.name, []): + if _dependency.constraint.allows(_package.version): + return _package + return None + + @classmethod + def __walk_dependency_level( + cls, + dependencies, + level, + pinned_versions, + packages_by_name, + project_level_dependencies, + nested_dependencies, + ): # type: (List[Dependency], int, bool, Dict[str, List[Package]], Set[str], Dict[Tuple[str, str], Dependency]) -> Dict[Tuple[str, str], Dependency] + if not dependencies: + return nested_dependencies + + next_level_dependencies = [] + + for requirement in dependencies: + locked_package = cls.__get_locked_package(requirement, packages_by_name) + + if locked_package: + for require in locked_package.requires: + if require.marker.is_empty(): + require.marker = requirement.marker + else: + require.marker = require.marker.intersect(requirement.marker) + + require.marker = require.marker.intersect(locked_package.marker) + next_level_dependencies.append(require) + + if requirement.name in project_level_dependencies and level == 0: + # project level dependencies take precedence + continue + + if locked_package: + # create dependency from locked package to retain dependency metadata + # if this is not done, we can end-up with incorrect nested dependencies + marker = requirement.marker + requirement = locked_package.to_dependency() + requirement.marker = requirement.marker.intersect(marker) + else: + # we make a copy to avoid any side-effects + requirement = deepcopy(requirement) + + if pinned_versions: + requirement.set_constraint( + cls.__get_locked_package(requirement, packages_by_name) + .to_dependency() + .constraint + ) - # group packages entries by name, this is required because requirement might use - # different constraints + # dependencies use extra to indicate that it was activated via parent + # package's extras, this is not required for nested exports as we assume + # the resolver already selected this dependency + requirement.marker = requirement.marker.without_extras() + + key = (requirement.name, requirement.pretty_constraint) + if key not in nested_dependencies: + nested_dependencies[key] = requirement + else: + nested_dependencies[key].marker = nested_dependencies[ + key + ].marker.intersect(requirement.marker) + + return cls.__walk_dependency_level( + dependencies=next_level_dependencies, + level=level + 1, + pinned_versions=pinned_versions, + packages_by_name=packages_by_name, + project_level_dependencies=project_level_dependencies, + nested_dependencies=nested_dependencies, + ) + + @classmethod + def get_project_dependencies( + cls, project_requires, locked_packages, pinned_versions=False, with_nested=False + ): # type: (List[Dependency], List[Package], bool, bool) -> Iterable[Dependency] + # group packages entries by name, this is required because requirement might use different constraints packages_by_name = {} - for pkg in packages: + for pkg in locked_packages: if pkg.name not in packages_by_name: packages_by_name[pkg.name] = [] packages_by_name[pkg.name].append(pkg) - def __get_locked_package( - _dependency, - ): # type: (Dependency) -> Optional[Package] - """ - Internal helper to identify corresponding locked package using dependency - version constraints. - """ - for _package in packages_by_name.get(_dependency.name, []): - if _dependency.constraint.allows(_package.version): - return _package - return None - project_level_dependencies = set() dependencies = [] for dependency in project_requires: dependency = deepcopy(dependency) - if pinned_versions: - locked_package = __get_locked_package(dependency) - if locked_package: - dependency.set_constraint(locked_package.to_dependency().constraint) + locked_package = cls.__get_locked_package(dependency, packages_by_name) + if locked_package: + locked_dependency = locked_package.to_dependency() + locked_dependency.marker = dependency.marker.intersect( + locked_package.marker + ) + + if not pinned_versions: + locked_dependency.set_constraint(dependency.constraint) + + dependency = locked_dependency + project_level_dependencies.add(dependency.name) dependencies.append(dependency) @@ -222,46 +307,74 @@ def __get_locked_package( # return only with project level dependencies return dependencies - nested_dependencies = list() + nested_dependencies = cls.__walk_dependency_level( + dependencies=dependencies, + level=0, + pinned_versions=pinned_versions, + packages_by_name=packages_by_name, + project_level_dependencies=project_level_dependencies, + nested_dependencies=dict(), + ) - for pkg in packages: # type: Package - for requirement in pkg.requires: # type: Dependency - if requirement.name in project_level_dependencies: - # project level dependencies take precedence - continue + # Merge same dependencies using marker union + for requirement in dependencies: + key = (requirement.name, requirement.pretty_constraint) + if key not in nested_dependencies: + nested_dependencies[key] = requirement + else: + nested_dependencies[key].marker = nested_dependencies[key].marker.union( + requirement.marker + ) - # we make a copy to avoid any side-effects - requirement = deepcopy(requirement) - requirement._category = pkg.category + return sorted(nested_dependencies.values(), key=lambda x: x.name.lower()) - if pinned_versions: - requirement.set_constraint( - __get_locked_package(requirement).to_dependency().constraint - ) + def get_project_dependency_packages( + self, project_requires, dev=False, extras=None + ): # type: (List[Dependency], bool, Optional[Union[bool, Sequence[str]]]) -> Iterator[DependencyPackage] + repository = self.locked_repository(with_dev_reqs=dev) - # dependencies use extra to indicate that it was activated via parent - # package's extras - marker = requirement.marker.without_extras() - for project_requirement in project_requires: - if ( - pkg.name == project_requirement.name - and project_requirement.constraint.allows(pkg.version) - ): - requirement.marker = marker.intersect( - project_requirement.marker - ) - break - else: - # this dependency was not from a project requirement - requirement.marker = marker.intersect(pkg.marker) + # Build a set of all packages required by our selected extras + extra_package_names = ( + None if (isinstance(extras, bool) and extras is True) else () + ) - if requirement not in nested_dependencies: - nested_dependencies.append(requirement) + if extra_package_names is not None: + extra_package_names = set( + get_extra_package_names( + repository.packages, self.lock_data.get("extras", {}), extras or (), + ) + ) - return sorted( - itertools.chain(dependencies, nested_dependencies), - key=lambda x: x.name.lower(), - ) + # If a package is optional and we haven't opted in to it, do not select + selected = [] + for dependency in project_requires: + try: + package = repository.find_packages(dependency=dependency)[0] + except IndexError: + continue + + if extra_package_names is not None and ( + package.optional and package.name not in extra_package_names + ): + # a package is locked as optional, but is not activated via extras + continue + + selected.append(dependency) + + for dependency in self.get_project_dependencies( + project_requires=selected, + locked_packages=repository.packages, + with_nested=True, + ): + try: + package = repository.find_packages(dependency=dependency)[0] + except IndexError: + continue + + for extra in dependency.extras: + package.requires_extras.append(extra) + + yield DependencyPackage(dependency=dependency, package=package) def set_lock_data(self, root, packages): # type: (...) -> bool files = table() diff --git a/poetry/utils/exporter.py b/poetry/utils/exporter.py index 4200abd146d..5f2d6303817 100644 --- a/poetry/utils/exporter.py +++ b/poetry/utils/exporter.py @@ -1,3 +1,5 @@ +from typing import Optional +from typing import Sequence from typing import Union from clikit.api.io import IO @@ -5,7 +7,6 @@ from poetry.poetry import Poetry from poetry.utils._compat import Path from poetry.utils._compat import decode -from poetry.utils.extras import get_extra_package_names class Exporter(object): @@ -30,7 +31,7 @@ def export( dev=False, extras=None, with_credentials=False, - ): # type: (str, Path, Union[IO, str], bool, bool, bool) -> None + ): # type: (str, Path, Union[IO, str], bool, bool, Optional[Union[bool, Sequence[str]]], bool) -> None if fmt not in self.ACCEPTED_FORMATS: raise ValueError("Invalid export format: {}".format(fmt)) @@ -51,38 +52,19 @@ def _export_requirements_txt( dev=False, extras=None, with_credentials=False, - ): # type: (Path, Union[IO, str], bool, bool, bool) -> None + ): # type: (Path, Union[IO, str], bool, bool, Optional[Union[bool, Sequence[str]]], bool) -> None indexes = set() content = "" - repository = self._poetry.locker.locked_repository(dev) - - # Build a set of all packages required by our selected extras - extra_package_names = set( - get_extra_package_names( - repository.packages, - self._poetry.locker.lock_data.get("extras", {}), - extras or (), - ) - ) - dependency_lines = set() - for dependency in self._poetry.locker.get_project_dependencies( - project_requires=self._poetry.package.all_requires, - with_nested=True, - with_dev=dev, + for dependency_package in self._poetry.locker.get_project_dependency_packages( + project_requires=self._poetry.package.all_requires, dev=dev, extras=extras ): - try: - package = repository.find_packages(dependency=dependency)[0] - except IndexError: - continue - - # If a package is optional and we haven't opted in to it, continue - if package.optional and package.name not in extra_package_names: - continue - line = "" + dependency = dependency_package.dependency + package = dependency_package.package + if package.develop: line += "-e " diff --git a/pyproject.toml b/pyproject.toml index f099a4cac80..59c49f74cf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "poetry" -version = "1.1.0" +version = "1.2.0a0" description = "Python dependency management and packaging made easy." authors = [ "Sébastien Eustace " diff --git a/tests/repositories/test_legacy_repository.py b/tests/repositories/test_legacy_repository.py index 7fd131c70e4..47ccc1052f9 100644 --- a/tests/repositories/test_legacy_repository.py +++ b/tests/repositories/test_legacy_repository.py @@ -117,7 +117,7 @@ def test_get_package_information_skips_dependencies_with_invalid_constraints(): package.description == "Python Language Server for the Language Server Protocol" ) - assert 19 == len(package.requires) + assert 25 == len(package.requires) assert sorted( [r for r in package.requires if not r.is_optional()], key=lambda r: r.name ) == [ @@ -216,7 +216,7 @@ def test_get_package_from_both_py2_and_py3_specific_wheels(): assert "ipython" == package.name assert "5.7.0" == package.version.text assert "*" == package.python_versions - assert 26 == len(package.requires) + assert 41 == len(package.requires) expected = [ Dependency("appnope", "*"), diff --git a/tests/repositories/test_pypi_repository.py b/tests/repositories/test_pypi_repository.py index ef094f3f1d5..55afdd39485 100644 --- a/tests/repositories/test_pypi_repository.py +++ b/tests/repositories/test_pypi_repository.py @@ -165,7 +165,7 @@ def test_pypi_repository_supports_reading_bz2_files(): package = repo.package("twisted", "18.9.0") assert package.name == "twisted" - assert 28 == len(package.requires) + assert 71 == len(package.requires) assert sorted( [r for r in package.requires if not r.is_optional()], key=lambda r: r.name ) == [ diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index e26f448f6de..d810bb8b08b 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -2,6 +2,7 @@ import pytest +from poetry.core.packages import dependency_from_pep_508 from poetry.core.toml.file import TOMLFile from poetry.factory import Factory from poetry.packages import Locker as BaseLocker @@ -59,13 +60,18 @@ def poetry(fixture_dir, locker): return p -def set_package_requires(poetry): +def set_package_requires(poetry, skip=None): + skip = skip or set() packages = poetry.locker.locked_repository(with_dev_reqs=True).packages poetry.package.requires = [ - pkg.to_dependency() for pkg in packages if pkg.category == "main" + pkg.to_dependency() + for pkg in packages + if pkg.category == "main" and pkg.name not in skip ] poetry.package.dev_requires = [ - pkg.to_dependency() for pkg in packages if pkg.category == "dev" + pkg.to_dependency() + for pkg in packages + if pkg.category == "dev" and pkg.name not in skip ] @@ -170,6 +176,141 @@ def test_exporter_can_export_requirements_txt_with_standard_packages_and_markers assert expected == content +def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers( + tmp_dir, poetry +): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "a", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "python_version < '3.7'", + "dependencies": {"b": ">=0.0.0", "c": ">=0.0.0"}, + }, + { + "name": "b", + "version": "4.5.6", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "platform_system == 'Windows'", + "dependencies": {"d": ">=0.0.0"}, + }, + { + "name": "c", + "version": "7.8.9", + "category": "main", + "optional": False, + "python-versions": "*", + "marker": "sys_platform == 'win32'", + "dependencies": {"d": ">=0.0.0"}, + }, + { + "name": "d", + "version": "0.0.1", + "category": "main", + "optional": False, + "python-versions": "*", + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"a": [], "b": [], "c": [], "d": []}, + }, + } + ) + set_package_requires(poetry, skip={"b", "c", "d"}) + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + expected = { + "a": dependency_from_pep_508("a==1.2.3; python_version < '3.7'"), + "b": dependency_from_pep_508( + "b==4.5.6; platform_system == 'Windows' and python_version < '3.7'" + ), + "c": dependency_from_pep_508( + "c==7.8.9; sys_platform == 'win32' and python_version < '3.7'" + ), + "d": dependency_from_pep_508( + "d==0.0.1; python_version < '3.7' and platform_system == 'Windows' and sys_platform == 'win32'" + ), + } + + for line in content.strip().split("\n"): + dependency = dependency_from_pep_508(line) + assert dependency.name in expected + expected_dependency = expected.pop(dependency.name) + assert dependency == expected_dependency + assert dependency.marker == expected_dependency.marker + + assert expected == {} + + +@pytest.mark.parametrize( + "dev,lines", + [(False, ['a==1.2.3; python_version < "3.8"']), (True, ["a==1.2.3", "b==4.5.6"])], +) +def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_any( + tmp_dir, poetry, dev, lines +): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "a", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + }, + { + "name": "b", + "version": "4.5.6", + "category": "dev", + "optional": False, + "python-versions": "*", + "dependencies": {"a": ">=1.2.3"}, + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"a": [], "b": []}, + }, + } + ) + + poetry.package.requires = [ + Factory.create_dependency( + name="a", constraint=dict(version="^1.2.3", python="<3.8") + ), + ] + poetry.package.dev_requires = [ + Factory.create_dependency( + name="b", constraint=dict(version="^4.5.6"), category="dev" + ), + ] + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=dev) + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + assert content.strip() == "\n".join(lines) + + def test_exporter_can_export_requirements_txt_with_standard_packages_and_hashes( tmp_dir, poetry ): @@ -399,8 +540,17 @@ def test_exporter_exports_requirements_txt_without_optional_packages(tmp_dir, po assert expected == content -def test_exporter_exports_requirements_txt_with_optional_packages_if_opted_in( - tmp_dir, poetry +@pytest.mark.parametrize( + "extras,lines", + [ + (None, ["foo==1.2.3"]), + (False, ["foo==1.2.3"]), + (True, ["bar==4.5.6", "foo==1.2.3", "spam==0.1.0"]), + (["feature_bar"], ["bar==4.5.6", "foo==1.2.3", "spam==0.1.0"]), + ], +) +def test_exporter_exports_requirements_txt_with_optional_packages( + tmp_dir, poetry, extras, lines ): poetry.locker.mock_lock_data( { @@ -445,22 +595,16 @@ def test_exporter_exports_requirements_txt_with_optional_packages_if_opted_in( Path(tmp_dir), "requirements.txt", dev=True, - extras=["feature_bar"], + with_hashes=False, + extras=extras, ) with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: content = f.read() - expected = """\ -bar==4.5.6 \\ - --hash=sha256:67890 -foo==1.2.3 \\ - --hash=sha256:12345 -spam==0.1.0 \\ - --hash=sha256:abcde -""" + expected = "\n".join(lines) - assert expected == content + assert content.strip() == expected def test_exporter_can_export_requirements_txt_with_git_packages(tmp_dir, poetry): @@ -503,6 +647,55 @@ def test_exporter_can_export_requirements_txt_with_git_packages(tmp_dir, poetry) assert expected == content +def test_exporter_can_export_requirements_txt_with_nested_packages(tmp_dir, poetry): + poetry.locker.mock_lock_data( + { + "package": [ + { + "name": "foo", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "*", + "source": { + "type": "git", + "url": "https://github.com/foo/foo.git", + "reference": "123456", + }, + }, + { + "name": "bar", + "version": "4.5.6", + "category": "main", + "optional": False, + "python-versions": "*", + "dependencies": {"foo": "rev 123456"}, + }, + ], + "metadata": { + "python-versions": "*", + "content-hash": "123456789", + "hashes": {"foo": [], "bar": []}, + }, + } + ) + set_package_requires(poetry, skip={"foo"}) + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt") + + with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f: + content = f.read() + + expected = """\ +bar==4.5.6 +foo @ git+https://github.com/foo/foo.git@123456 +""" + + assert expected == content + + def test_exporter_can_export_requirements_txt_with_git_packages_and_markers( tmp_dir, poetry ):