From e7d38ee74f6272a6845e7c47b1df1036451da2f9 Mon Sep 17 00:00:00 2001 From: David Hotham Date: Sat, 5 Feb 2022 13:56:42 +0000 Subject: [PATCH] Refactor poetry export Introduce a new root_package section in poetry.lock, and miscellaneous fixes --- src/poetry/packages/locker.py | 350 +++++++++--------- src/poetry/repositories/base_repository.py | 1 + src/poetry/utils/exporter.py | 44 ++- tests/console/commands/test_export.py | 20 +- .../fixtures/extras-with-dependencies.test | 15 + tests/installation/fixtures/extras.test | 16 + .../installation/fixtures/install-no-dev.test | 9 + .../fixtures/no-dependencies.test | 9 + tests/installation/fixtures/old-lock.test | 9 + tests/installation/fixtures/remove.test | 12 + .../fixtures/update-with-lock.test | 12 + .../fixtures/update-with-locked-extras.test | 13 + .../fixtures/with-category-change.test | 12 + .../fixtures/with-conditional-dependency.test | 9 + .../fixtures/with-dependencies-extras.test | 13 + .../with-dependencies-nested-extras.test | 12 + .../fixtures/with-dependencies.test | 13 + ...irectory-dependency-poetry-transitive.test | 12 + .../with-directory-dependency-poetry.test | 12 + .../with-directory-dependency-setuptools.test | 12 + .../with-duplicate-dependencies-update.test | 12 + .../fixtures/with-duplicate-dependencies.test | 12 + .../with-file-dependency-transitive.test | 12 + .../fixtures/with-file-dependency.test | 12 + .../fixtures/with-multiple-updates.test | 12 + .../fixtures/with-optional-dependencies.test | 15 + .../fixtures/with-platform-dependencies.test | 15 + .../fixtures/with-prereleases.test | 13 + .../fixtures/with-pypi-repository.test | 9 + .../fixtures/with-python-versions.test | 14 + .../fixtures/with-sub-dependencies.test | 13 + .../fixtures/with-url-dependency.test | 12 + ...ith-wheel-dependency-no-requires-dist.test | 12 + tests/installation/test_installer.py | 28 +- tests/installation/test_installer_old.py | 30 +- tests/packages/test_locker.py | 66 +++- tests/utils/test_exporter.py | 132 ++++++- 37 files changed, 808 insertions(+), 216 deletions(-) diff --git a/src/poetry/packages/locker.py b/src/poetry/packages/locker.py index 4cf56b9ae93..311385f0772 100644 --- a/src/poetry/packages/locker.py +++ b/src/poetry/packages/locker.py @@ -7,6 +7,7 @@ from hashlib import sha256 from pathlib import Path from typing import TYPE_CHECKING +from typing import Any from typing import Dict from typing import Iterable from typing import Iterator @@ -87,18 +88,114 @@ def is_fresh(self) -> bool: return False + def read_package(self, info: Dict[str, Any]) -> Package: + """ + Constructs a Package from information read from the locker. + """ + from poetry.factory import Factory + + lock_data = self.lock_data + + source = info.get("source", {}) + source_type = source.get("type") + url = source.get("url") + if source_type in ["directory", "file"]: + url = self._lock.path.parent.joinpath(url).resolve().as_posix() + + package = Package( + info["name"], + info["version"], + info["version"], + source_type=source_type, + source_url=url, + source_reference=source.get("reference"), + source_resolved_reference=source.get("resolved_reference"), + ) + package.description = info.get("description", "") + package.category = info.get("category", "main") + package.groups = info.get("groups", ["default"]) + package.optional = info["optional"] + if "hashes" in lock_data["metadata"]: + # Old lock so we create dummy files from the hashes + package.files = [ + {"name": h, "hash": h} + for h in lock_data["metadata"]["hashes"].get(info["name"], []) + ] + else: + package.files = lock_data["metadata"]["files"].get(info["name"], []) + + package.python_versions = info["python-versions"] + extras = info.get("extras", {}) + if extras: + for name, deps in extras.items(): + package.extras[name] = [] + + for dep in deps: + try: + dependency = Dependency.create_from_pep_508(dep) + except InvalidRequirement: + # handle lock files with invalid PEP 508 + m = re.match(r"^(.+?)(?:\[(.+?)])?(?:\s+\((.+)\))?$", dep) + dep_name = m.group(1) + extras = m.group(2) or "" + constraint = m.group(3) or "*" + dependency = Dependency( + dep_name, constraint, extras=extras.split(",") + ) + package.extras[name].append(dependency) + + if "marker" in info: + package.marker = parse_marker(info["marker"]) + else: + # Compatibility for old locks + if "requirements" in info: + dep = Dependency("foo", "0.0.0") + for name, value in info["requirements"].items(): + if name == "python": + dep.python_versions = value + elif name == "platform": + dep.platform = value + + split_dep = dep.to_pep_508(False).split(";") + if len(split_dep) > 1: + package.marker = parse_marker(split_dep[1].strip()) + + for dep_name, constraint in info.get("dependencies", {}).items(): + + root_dir = self._lock.path.parent + if package.source_type == "directory": + # root dir should be the source of the package relative to the lock + # path + root_dir = Path(package.source_url) + + if isinstance(constraint, list): + for c in constraint: + package.add_dependency( + Factory.create_dependency(dep_name, c, root_dir=root_dir) + ) + + continue + + package.add_dependency( + Factory.create_dependency(dep_name, constraint, root_dir=root_dir) + ) + + if "develop" in info: + package.develop = info["develop"] + + return package + def locked_repository(self, with_dev_reqs: bool = False) -> "Repository": """ Searches and returns a repository of locked packages. """ - from poetry.factory import Factory from poetry.repositories import Repository if not self.is_locked(): return Repository() lock_data = self.lock_data - packages = Repository() + repository = Repository() if with_dev_reqs: locked_packages = lock_data["package"] @@ -108,165 +205,91 @@ def locked_repository(self, with_dev_reqs: bool = False) -> "Repository": ] if not locked_packages: - return packages + return repository for info in locked_packages: - source = info.get("source", {}) - source_type = source.get("type") - url = source.get("url") - if source_type in ["directory", "file"]: - url = self._lock.path.parent.joinpath(url).resolve().as_posix() - - package = Package( - info["name"], - info["version"], - info["version"], - source_type=source_type, - source_url=url, - source_reference=source.get("reference"), - source_resolved_reference=source.get("resolved_reference"), - ) - package.description = info.get("description", "") - package.category = info.get("category", "main") - package.groups = info.get("groups", ["default"]) - package.optional = info["optional"] - if "hashes" in lock_data["metadata"]: - # Old lock so we create dummy files from the hashes - package.files = [ - {"name": h, "hash": h} - for h in lock_data["metadata"]["hashes"][info["name"]] - ] - else: - package.files = lock_data["metadata"]["files"][info["name"]] - - package.python_versions = info["python-versions"] - extras = info.get("extras", {}) - if extras: - for name, deps in extras.items(): - package.extras[name] = [] - - for dep in deps: - try: - dependency = Dependency.create_from_pep_508(dep) - except InvalidRequirement: - # handle lock files with invalid PEP 508 - m = re.match(r"^(.+?)(?:\[(.+?)])?(?:\s+\((.+)\))?$", dep) - dep_name = m.group(1) - extras = m.group(2) or "" - constraint = m.group(3) or "*" - dependency = Dependency( - dep_name, constraint, extras=extras.split(",") - ) - package.extras[name].append(dependency) - - if "marker" in info: - package.marker = parse_marker(info["marker"]) - else: - # Compatibility for old locks - if "requirements" in info: - dep = Dependency("foo", "0.0.0") - for name, value in info["requirements"].items(): - if name == "python": - dep.python_versions = value - elif name == "platform": - dep.platform = value - - split_dep = dep.to_pep_508(False).split(";") - if len(split_dep) > 1: - package.marker = parse_marker(split_dep[1].strip()) - - for dep_name, constraint in info.get("dependencies", {}).items(): - - root_dir = self._lock.path.parent - if package.source_type == "directory": - # root dir should be the source of the package relative to the lock - # path - root_dir = Path(package.source_url) - - if isinstance(constraint, list): - for c in constraint: - package.add_dependency( - Factory.create_dependency(dep_name, c, root_dir=root_dir) - ) + package = self.read_package(info) + repository.add_package(package) - continue - - package.add_dependency( - Factory.create_dependency(dep_name, constraint, root_dir=root_dir) - ) + root_package = lock_data.get("root_package") + if root_package is not None: + repository.root_package = self.read_package(root_package) - if "develop" in info: - package.develop = info["develop"] - - packages.add_package(package) - - return packages + return repository @staticmethod def __get_locked_package( - _dependency: Dependency, packages_by_name: Dict[str, List[Package]] + dependency: Dependency, + packages_by_name: Dict[str, List[Package]], + decided: Optional[Dict[Package, Dependency]] = None, ) -> Optional[Package]: """ Internal helper to identify corresponding locked package using dependency version constraints. """ - for _package in packages_by_name.get(_dependency.name, []): - if _dependency.constraint.allows(_package.version): - return _package - return None + decided = decided or {} + + # Get the packages that are consistent with this dependency. + packages = [ + package + for package in packages_by_name.get(dependency.name, []) + if package.python_constraint.allows_all(dependency.python_constraint) + and dependency.constraint.allows(package.version) + ] + + # If we've previously made a choice that is compatible with the current + # requirement, stick with it. + for package in packages: + old_decision = decided.get(package) + if ( + old_decision is not None + and not old_decision.marker.intersect(dependency.marker).is_empty() + ): + return package + + return next(iter(packages), None) @classmethod - def __walk_dependency_level( + def __walk_dependencies( cls, dependencies: List[Dependency], - level: int, - pinned_versions: bool, packages_by_name: Dict[str, List[Package]], - project_level_dependencies: Set[str], - nested_dependencies: Dict[Tuple[str, str], Dependency], ) -> Dict[Tuple[str, str], Dependency]: - if not dependencies: - return nested_dependencies - - next_level_dependencies = [] - - for requirement in dependencies: - key = (requirement.name, requirement.pretty_constraint) - locked_package = cls.__get_locked_package(requirement, packages_by_name) - - if locked_package: - # create dependency from locked package to retain dependency metadata - # if this is not done, we can end-up with incorrect nested dependencies - constraint = requirement.constraint - pretty_constraint = requirement.pretty_constraint - marker = requirement.marker - requirement = locked_package.to_dependency() - requirement.marker = requirement.marker.intersect(marker) - - key = (requirement.name, pretty_constraint) + nested_dependencies: Dict[Package, Dependency] = {} - if not pinned_versions: - requirement.set_constraint(constraint) + visited: Set[Tuple[Dependency, Marker]] = set() + while dependencies: + requirement = dependencies.pop(0) + if (requirement, requirement.marker) in visited: + continue + visited.add((requirement, requirement.marker)) - for require in locked_package.requires: - if require.marker.is_empty(): - require.marker = requirement.marker - else: - require.marker = require.marker.intersect(requirement.marker) + locked_package = cls.__get_locked_package( + requirement, packages_by_name, nested_dependencies + ) - require.marker = require.marker.intersect(locked_package.marker) + if not locked_package: + # Should normally be able to satisfy all requirements, but this case is + # permissible eg if we encounter a dev dependency when walking the + # non-dev dependencies. + continue - if key not in nested_dependencies: - next_level_dependencies.append(require) + # create dependency from locked package to retain dependency metadata + # if this is not done, we can end-up with incorrect nested dependencies + constraint = requirement.constraint + marker = requirement.marker + requirement = locked_package.to_dependency() + requirement.marker = requirement.marker.intersect(marker) - if requirement.name in project_level_dependencies and level == 0: - # project level dependencies take precedence - continue + requirement.set_constraint(constraint) - if not locked_package: - # we make a copy to avoid any side-effects - requirement = deepcopy(requirement) + for require in locked_package.requires: + require = deepcopy(require) + require.marker = require.marker.intersect(requirement.marker) + if not require.marker.is_empty(): + dependencies.append(require) + key = locked_package if key not in nested_dependencies: nested_dependencies[key] = requirement else: @@ -274,32 +297,26 @@ def __walk_dependency_level( requirement.marker ) - return cls.__walk_dependency_level( - dependencies=next_level_dependencies, - level=level + 1, - pinned_versions=pinned_versions, - packages_by_name=packages_by_name, - project_level_dependencies=project_level_dependencies, - nested_dependencies=nested_dependencies, - ) + return nested_dependencies @classmethod def get_project_dependencies( cls, project_requires: List[Dependency], locked_packages: List[Package], - pinned_versions: bool = False, - with_nested: bool = False, - ) -> Iterable[Dependency]: + ) -> Iterable[Tuple[Package, Dependency]]: # group packages entries by name, this is required because requirement might use - # different constraints + # different constraints. packages_by_name = {} for pkg in locked_packages: if pkg.name not in packages_by_name: packages_by_name[pkg.name] = [] packages_by_name[pkg.name].append(pkg) - project_level_dependencies = set() + # Put higher versions first so that we prefer them. + for packages in packages_by_name.values(): + packages.sort(key=lambda package: package.version, reverse=True) + dependencies = [] for dependency in project_requires: @@ -311,38 +328,18 @@ def get_project_dependencies( locked_package.marker ) - if not pinned_versions: - locked_dependency.set_constraint(dependency.constraint) + locked_dependency.set_constraint(dependency.constraint) dependency = locked_dependency - project_level_dependencies.add(dependency.name) dependencies.append(dependency) - if not with_nested: - # return only with project level dependencies - return dependencies - - nested_dependencies = cls.__walk_dependency_level( + nested_dependencies = cls.__walk_dependencies( dependencies=dependencies, - level=0, - pinned_versions=pinned_versions, packages_by_name=packages_by_name, - project_level_dependencies=project_level_dependencies, - nested_dependencies={}, ) - # Merge same dependencies using marker union - for requirement in dependencies: - key = (requirement.name, requirement.pretty_constraint) - if key not in nested_dependencies: - nested_dependencies[key] = requirement - else: - nested_dependencies[key].marker = nested_dependencies[key].marker.union( - requirement.marker - ) - - return sorted(nested_dependencies.values(), key=lambda x: x.name.lower()) + return nested_dependencies.items() def get_project_dependency_packages( self, @@ -382,16 +379,10 @@ def get_project_dependency_packages( selected.append(dependency) - for dependency in self.get_project_dependencies( + for package, dependency in self.get_project_dependencies( project_requires=selected, locked_packages=repository.packages, - with_nested=True, ): - try: - package = repository.find_packages(dependency=dependency)[0] - except IndexError: - continue - for extra in dependency.extras: package.requires_extras.append(extra) @@ -418,6 +409,7 @@ def set_lock_data(self, root: Package, packages: List[Package]) -> bool: del package["files"] lock = document() + lock["root_package"] = self._dump_package(root) lock["package"] = packages if root.extras: diff --git a/src/poetry/repositories/base_repository.py b/src/poetry/repositories/base_repository.py index 4aa77fcb2a6..038bd57c941 100644 --- a/src/poetry/repositories/base_repository.py +++ b/src/poetry/repositories/base_repository.py @@ -11,6 +11,7 @@ class BaseRepository: def __init__(self) -> None: self._packages: List["Package"] = [] + self.root_package: Optional["Package"] = None @property def packages(self) -> List["Package"]: diff --git a/src/poetry/utils/exporter.py b/src/poetry/utils/exporter.py index 98070d19b23..dee4227325b 100644 --- a/src/poetry/utils/exporter.py +++ b/src/poetry/utils/exporter.py @@ -1,6 +1,6 @@ -import itertools import urllib.parse +from copy import deepcopy from typing import TYPE_CHECKING from typing import Optional from typing import Sequence @@ -70,21 +70,37 @@ def _export_requirements_txt( content = "" dependency_lines = set() - for package, groups in itertools.groupby( - self._poetry.locker.get_project_dependency_packages( - project_requires=self._poetry.package.all_requires, - dev=dev, - extras=extras, - ), - lambda dependency_package: dependency_package.package, + # If we have a root package then use that to tell us what the top-level + # dependencies are. If we don't (older version of poetry.lock) then try to + # cover all dependencies. + repository = self._poetry.locker.locked_repository(with_dev_reqs=dev) + root_package = repository.root_package + project_requires = ( + self._poetry.package.all_requires + if root_package is None + else root_package.requires + ) + + # If we have a root package then we can also update the marker on each + # dependency to take account of the project-level python version. + if root_package is not None: + restricted_project_requires = [] + for project_require in project_requires: + project_require = deepcopy(project_require) + project_require.marker = project_require.marker.intersect( + root_package.python_marker + ) + restricted_project_requires.append(project_require) + project_requires = restricted_project_requires + + for dependency_package in self._poetry.locker.get_project_dependency_packages( + project_requires=project_requires, + dev=dev, + extras=extras, ): line = "" - dependency_packages = list(groups) - dependency = dependency_packages[0].dependency - marker = dependency.marker - for dep_package in dependency_packages[1:]: - marker = marker.union(dep_package.dependency.marker) - dependency.marker = marker + dependency = dependency_package.dependency + package = dependency_package.package if package.develop: line += "-e " diff --git a/tests/console/commands/test_export.py b/tests/console/commands/test_export.py index 010dfd10617..87f4b6fa2bc 100644 --- a/tests/console/commands/test_export.py +++ b/tests/console/commands/test_export.py @@ -81,9 +81,7 @@ def _export_requirements(tester: "CommandTester", poetry: "Poetry") -> None: assert poetry.locker.lock.exists() - expected = """\ -foo==1.0.0 -""" + expected = 'foo==1.0.0 ; python_version >= "2.7" and python_version < "2.8" or python_version >= "3.4" and python_version < "4.0"\n' # noqa: E501 assert content == expected @@ -110,9 +108,7 @@ def test_export_fails_on_invalid_format(tester: "CommandTester", do_lock: None): def test_export_prints_to_stdout_by_default(tester: "CommandTester", do_lock: None): tester.execute("--format requirements.txt") - expected = """\ -foo==1.0.0 -""" + expected = 'foo==1.0.0 ; python_version >= "2.7" and python_version < "2.8" or python_version >= "3.4" and python_version < "4.0"\n' # noqa: E501 assert tester.io.fetch_output() == expected @@ -120,18 +116,16 @@ def test_export_uses_requirements_txt_format_by_default( tester: "CommandTester", do_lock: None ): tester.execute() - expected = """\ -foo==1.0.0 -""" + expected = 'foo==1.0.0 ; python_version >= "2.7" and python_version < "2.8" or python_version >= "3.4" and python_version < "4.0"\n' # noqa: E501 assert tester.io.fetch_output() == expected def test_export_includes_extras_by_flag(tester: "CommandTester", do_lock: None): tester.execute("--format requirements.txt --extras feature_bar") - expected = """\ -bar==1.1.0 -foo==1.0.0 -""" + expected = ( + 'bar==1.1.0 ; python_version >= "2.7" and python_version < "2.8" or python_version >= "3.4" and python_version < "4.0"\n' # noqa: E501 + 'foo==1.0.0 ; python_version >= "2.7" and python_version < "2.8" or python_version >= "3.4" and python_version < "4.0"\n' # noqa: E501 + ) assert tester.io.fetch_output() == expected diff --git a/tests/installation/fixtures/extras-with-dependencies.test b/tests/installation/fixtures/extras-with-dependencies.test index d0a63bfb879..5a58a6fcc8f 100644 --- a/tests/installation/fixtures/extras-with-dependencies.test +++ b/tests/installation/fixtures/extras-with-dependencies.test @@ -1,3 +1,18 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] +extras = { foo = ['c'] } + +[root_package.dependencies] +A = "^1.0" +B = "^1.0" +C = { optional = true, version = "^1.0" } + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/extras.test b/tests/installation/fixtures/extras.test index 3b4086ab18a..7765713d8c6 100644 --- a/tests/installation/fixtures/extras.test +++ b/tests/installation/fixtures/extras.test @@ -1,3 +1,19 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +extras = { foo = ['d'] } +files = [] + +[root_package.dependencies] +A = "^1.0" +B = "^1.0" +C = "^1.0" +D = { version = "^1.0", optional = true } + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/install-no-dev.test b/tests/installation/fixtures/install-no-dev.test index c966c56baa0..67c41cf2223 100644 --- a/tests/installation/fixtures/install-no-dev.test +++ b/tests/installation/fixtures/install-no-dev.test @@ -1,3 +1,12 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/no-dependencies.test b/tests/installation/fixtures/no-dependencies.test index 61424ceee4b..344a0480a33 100644 --- a/tests/installation/fixtures/no-dependencies.test +++ b/tests/installation/fixtures/no-dependencies.test @@ -1,5 +1,14 @@ package = [] +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + [metadata] python-versions = "*" lock-version = "1.1" diff --git a/tests/installation/fixtures/old-lock.test b/tests/installation/fixtures/old-lock.test index e1898fbd3f1..afc21bfb234 100644 --- a/tests/installation/fixtures/old-lock.test +++ b/tests/installation/fixtures/old-lock.test @@ -1,3 +1,12 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + [[package]] name = "attrs" version = "17.4.0" diff --git a/tests/installation/fixtures/remove.test b/tests/installation/fixtures/remove.test index d0aea7a7b3e..b8a54541380 100644 --- a/tests/installation/fixtures/remove.test +++ b/tests/installation/fixtures/remove.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +A = "~1.0" + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/update-with-lock.test b/tests/installation/fixtures/update-with-lock.test index 4856157ff5e..f758968df60 100644 --- a/tests/installation/fixtures/update-with-lock.test +++ b/tests/installation/fixtures/update-with-lock.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +A = "*" + [[package]] name = "A" version = "1.1" diff --git a/tests/installation/fixtures/update-with-locked-extras.test b/tests/installation/fixtures/update-with-locked-extras.test index 4872311c702..ab5a36c470f 100644 --- a/tests/installation/fixtures/update-with-locked-extras.test +++ b/tests/installation/fixtures/update-with-locked-extras.test @@ -1,3 +1,16 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +A = { version = "^1.0", extras = ['foo'] } +D = "^1.0" + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/with-category-change.test b/tests/installation/fixtures/with-category-change.test index f431f92ba0e..54203ec8e82 100644 --- a/tests/installation/fixtures/with-category-change.test +++ b/tests/installation/fixtures/with-category-change.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +B = "^1.1" + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/with-conditional-dependency.test b/tests/installation/fixtures/with-conditional-dependency.test index 78546a5c96a..76409018286 100644 --- a/tests/installation/fixtures/with-conditional-dependency.test +++ b/tests/installation/fixtures/with-conditional-dependency.test @@ -1,3 +1,12 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + [[package]] name = "A" version = "1.0.0" diff --git a/tests/installation/fixtures/with-dependencies-extras.test b/tests/installation/fixtures/with-dependencies-extras.test index 042e29670e1..b99fc4ac6da 100644 --- a/tests/installation/fixtures/with-dependencies-extras.test +++ b/tests/installation/fixtures/with-dependencies-extras.test @@ -1,3 +1,16 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +A = "^1.0" +B = { version = "^1.0", extras = ['foo'] } + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/with-dependencies-nested-extras.test b/tests/installation/fixtures/with-dependencies-nested-extras.test index 48a22a7c7f3..2f1c199ec3d 100644 --- a/tests/installation/fixtures/with-dependencies-nested-extras.test +++ b/tests/installation/fixtures/with-dependencies-nested-extras.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +A = { version = "^1.0", extras = ['B'] } + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/with-dependencies.test b/tests/installation/fixtures/with-dependencies.test index e9ed46612a0..6973727e5fa 100644 --- a/tests/installation/fixtures/with-dependencies.test +++ b/tests/installation/fixtures/with-dependencies.test @@ -1,3 +1,16 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +A = "~1.0" +B = "^1.0" + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/with-directory-dependency-poetry-transitive.test b/tests/installation/fixtures/with-directory-dependency-poetry-transitive.test index fb10b1accea..b24b86c24a2 100644 --- a/tests/installation/fixtures/with-directory-dependency-poetry-transitive.test +++ b/tests/installation/fixtures/with-directory-dependency-poetry-transitive.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +project-with-transitive-directory-dependencies = { path = "project_with_transitive_directory_dependencies" } + [[package]] category = "main" description = "" diff --git a/tests/installation/fixtures/with-directory-dependency-poetry.test b/tests/installation/fixtures/with-directory-dependency-poetry.test index 12431f62185..513099bcb4a 100644 --- a/tests/installation/fixtures/with-directory-dependency-poetry.test +++ b/tests/installation/fixtures/with-directory-dependency-poetry.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +project-with-extras = { extras = ['extras_a'], path = "tests/fixtures/project_with_extras" } + [[package]] description = "" category = "main" diff --git a/tests/installation/fixtures/with-directory-dependency-setuptools.test b/tests/installation/fixtures/with-directory-dependency-setuptools.test index f02bd82a335..e9c58c32e83 100644 --- a/tests/installation/fixtures/with-directory-dependency-setuptools.test +++ b/tests/installation/fixtures/with-directory-dependency-setuptools.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +project-with-setup = { path = "tests/fixtures/project_with_setup" } + [[package]] name = "cachy" version = "0.2.0" diff --git a/tests/installation/fixtures/with-duplicate-dependencies-update.test b/tests/installation/fixtures/with-duplicate-dependencies-update.test index f29945e71a8..3ee063c1ee2 100644 --- a/tests/installation/fixtures/with-duplicate-dependencies-update.test +++ b/tests/installation/fixtures/with-duplicate-dependencies-update.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +A = "*" + [[package]] name = "A" version = "1.1" diff --git a/tests/installation/fixtures/with-duplicate-dependencies.test b/tests/installation/fixtures/with-duplicate-dependencies.test index 75126501ef8..f8efe5e67d2 100644 --- a/tests/installation/fixtures/with-duplicate-dependencies.test +++ b/tests/installation/fixtures/with-duplicate-dependencies.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +A = "*" + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/with-file-dependency-transitive.test b/tests/installation/fixtures/with-file-dependency-transitive.test index b882f262640..002f6ecf0c2 100644 --- a/tests/installation/fixtures/with-file-dependency-transitive.test +++ b/tests/installation/fixtures/with-file-dependency-transitive.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +project-with-transitive-file-dependencies = { path = "project_with_transitive_file_dependencies" } + [[package]] category = "main" description = "" diff --git a/tests/installation/fixtures/with-file-dependency.test b/tests/installation/fixtures/with-file-dependency.test index 06ad019c0f4..957d4bd1b5a 100644 --- a/tests/installation/fixtures/with-file-dependency.test +++ b/tests/installation/fixtures/with-file-dependency.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +demo = { path = "tests/fixtures/distributions/demo-0.1.0-py2.py3-none-any.whl" } + [[package]] name = "demo" version = "0.1.0" diff --git a/tests/installation/fixtures/with-multiple-updates.test b/tests/installation/fixtures/with-multiple-updates.test index a488ef79631..4b77e560c63 100644 --- a/tests/installation/fixtures/with-multiple-updates.test +++ b/tests/installation/fixtures/with-multiple-updates.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "~2.7 || ^3.4" +files = [] + +[root_package.dependencies] +A = "^1.0" + [[package]] name = "A" version = "1.1" diff --git a/tests/installation/fixtures/with-optional-dependencies.test b/tests/installation/fixtures/with-optional-dependencies.test index 6c172caa078..134643260c1 100644 --- a/tests/installation/fixtures/with-optional-dependencies.test +++ b/tests/installation/fixtures/with-optional-dependencies.test @@ -1,3 +1,18 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "~2.7 || ^3.4" +extras = { foo = ['A (>=1.0,<1.1)'] } +files = [] + +[root_package.dependencies] +A = { version = "~1.0", optional = true } +B = { version = "^1.0", markers = "python_version >= \"2.4\" and python_version < \"2.5\"" } +C = { version = "^1.0", markers = "python_version >= \"2.7\" and python_version < \"2.8\" or python_version >= \"3.4\" and python_version < \"4.0\"" } + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/with-platform-dependencies.test b/tests/installation/fixtures/with-platform-dependencies.test index 9696b74fb9a..aac1bb1644f 100644 --- a/tests/installation/fixtures/with-platform-dependencies.test +++ b/tests/installation/fixtures/with-platform-dependencies.test @@ -1,3 +1,18 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +extras = { foo = ['A (>=1.0,<1.1)'] } +files = [] + +[root_package.dependencies] +A = { version = "~1.0", optional = true } +B = { version = "^1.0", markers = "sys_platform == \"custom\"" } +C = { version = "^1.0", markers = "sys_platform == \"darwin\"" } + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/with-prereleases.test b/tests/installation/fixtures/with-prereleases.test index c164800cb80..7dd89333a29 100644 --- a/tests/installation/fixtures/with-prereleases.test +++ b/tests/installation/fixtures/with-prereleases.test @@ -1,3 +1,16 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +A = "*" +B = "^1.1" + [[package]] name = "A" version = "1.0a2" diff --git a/tests/installation/fixtures/with-pypi-repository.test b/tests/installation/fixtures/with-pypi-repository.test index d1ed1ae55ed..0e498df6dbb 100644 --- a/tests/installation/fixtures/with-pypi-repository.test +++ b/tests/installation/fixtures/with-pypi-repository.test @@ -1,3 +1,12 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + [[package]] name = "attrs" version = "17.4.0" diff --git a/tests/installation/fixtures/with-python-versions.test b/tests/installation/fixtures/with-python-versions.test index 04ea98bc79a..b50bccf6a1f 100644 --- a/tests/installation/fixtures/with-python-versions.test +++ b/tests/installation/fixtures/with-python-versions.test @@ -1,3 +1,17 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "~2.7 || ^3.4" +files = [] + +[root_package.dependencies] +A = "~1.0" +B = "^1.0" +C = "^1.0" + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/with-sub-dependencies.test b/tests/installation/fixtures/with-sub-dependencies.test index 35b133dfb57..a8a286adb9a 100644 --- a/tests/installation/fixtures/with-sub-dependencies.test +++ b/tests/installation/fixtures/with-sub-dependencies.test @@ -1,3 +1,16 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +A = "~1.0" +B = "^1.0" + [[package]] name = "A" version = "1.0" diff --git a/tests/installation/fixtures/with-url-dependency.test b/tests/installation/fixtures/with-url-dependency.test index 2d23f7981aa..f1029f65cdb 100644 --- a/tests/installation/fixtures/with-url-dependency.test +++ b/tests/installation/fixtures/with-url-dependency.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +demo = { url = "https://python-poetry.org/distributions/demo-0.1.0-py2.py3-none-any.whl" } + [[package]] name = "demo" version = "0.1.0" diff --git a/tests/installation/fixtures/with-wheel-dependency-no-requires-dist.test b/tests/installation/fixtures/with-wheel-dependency-no-requires-dist.test index 383e43803eb..28d1045e7b7 100644 --- a/tests/installation/fixtures/with-wheel-dependency-no-requires-dist.test +++ b/tests/installation/fixtures/with-wheel-dependency-no-requires-dist.test @@ -1,3 +1,15 @@ +[root_package] +name = "root" +version = "1.0" +description = "" +category = "dev" +optional = true +python-versions = "*" +files = [] + +[root_package.dependencies] +demo = { path = "tests/fixtures/wheel_with_no_requires_dist/demo-0.1.0-py2.py3-none-any.whl" } + [[package]] name = "demo" version = "0.1.0" diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index fef1ee5c426..bc18d366b49 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -1,5 +1,6 @@ import itertools import json +import os.path from pathlib import Path from typing import TYPE_CHECKING @@ -293,7 +294,7 @@ def test_run_update_after_removing_dependencies( installed.add_package(package_c) package.add_dependency(Factory.create_dependency("A", "~1.0")) - package.add_dependency(Factory.create_dependency("B", "~1.1")) + package.add_dependency(Factory.create_dependency("B", "^1.0")) installer.update(True) installer.run() @@ -1206,6 +1207,12 @@ def test_run_installs_with_local_file( expected = fixture("with-file-dependency") + # Output includes the full path of the local dependency, which varies according to + # where the test is run... + expected["root_package"]["dependencies"]["demo"]["path"] = os.path.realpath( + file_path + ) + assert locker.written_data == expected assert installer.executor.installations_count == 2 @@ -1226,6 +1233,12 @@ def test_run_installs_wheel_with_no_requires_dist( expected = fixture("with-wheel-dependency-no-requires-dist") + # Output includes the full path of the local dependency, which varies according to + # where the test is run... + expected["root_package"]["dependencies"]["demo"]["path"] = os.path.realpath( + file_path + ) + assert locker.written_data == expected assert installer.executor.installations_count == 1 @@ -1251,6 +1264,13 @@ def test_run_installs_with_local_poetry_directory_and_extras( installer.run() expected = fixture("with-directory-dependency-poetry") + + # Output includes the full path of the local dependency, which varies according to + # where the test is run... + expected["root_package"]["dependencies"]["project-with-extras"][ + "path" + ] = os.path.realpath(file_path) + assert locker.written_data == expected assert installer.executor.installations_count == 2 @@ -1342,6 +1362,12 @@ def test_run_installs_with_local_setuptools_directory( expected = fixture("with-directory-dependency-setuptools") + # Output includes the full path of the local dependency, which varies according to + # where the test is run... patch up the written data to account for this. + expected["root_package"]["dependencies"]["project-with-setup"][ + "path" + ] = os.path.realpath(file_path) + assert locker.written_data == expected assert installer.executor.installations_count == 3 diff --git a/tests/installation/test_installer_old.py b/tests/installation/test_installer_old.py index d1478fcccd6..ec837accbfa 100644 --- a/tests/installation/test_installer_old.py +++ b/tests/installation/test_installer_old.py @@ -1,4 +1,5 @@ import itertools +import os.path from pathlib import Path from typing import TYPE_CHECKING @@ -32,6 +33,7 @@ if TYPE_CHECKING: + from poetry.core.toml.file import TOMLDocument from pytest_mock import MockerFixture from poetry.utils.env import Env @@ -145,7 +147,7 @@ def installer( return Installer(NullIO(), env, package, locker, pool, config, installed=installed) -def fixture(name: str) -> str: +def fixture(name: str) -> "TOMLDocument": file = TOMLFile(Path(__file__).parent / "fixtures" / f"{name}.test") return file.read() @@ -234,7 +236,7 @@ def test_run_update_after_removing_dependencies( installed.add_package(package_c) package.add_dependency(Factory.create_dependency("A", "~1.0")) - package.add_dependency(Factory.create_dependency("B", "~1.1")) + package.add_dependency(Factory.create_dependency("B", "^1.0")) installer.update(True) installer.run() @@ -853,6 +855,12 @@ def test_run_installs_with_local_file( expected = fixture("with-file-dependency") + # Output includes the full path of the local dependency, which varies according to + # where the test is run... + expected["root_package"]["dependencies"]["demo"]["path"] = os.path.realpath( + file_path + ) + assert locker.written_data == expected assert len(installer.installer.installs) == 2 @@ -874,6 +882,12 @@ def test_run_installs_wheel_with_no_requires_dist( expected = fixture("with-wheel-dependency-no-requires-dist") + # Output includes the full path of the local dependency, which varies according to + # where the test is run... + expected["root_package"]["dependencies"]["demo"]["path"] = os.path.realpath( + file_path + ) + assert locker.written_data == expected assert len(installer.installer.installs) == 1 @@ -900,6 +914,12 @@ def test_run_installs_with_local_poetry_directory_and_extras( expected = fixture("with-directory-dependency-poetry") + # Output includes the full path of the local dependency, which varies according to + # where the test is run... + expected["root_package"]["dependencies"]["project-with-extras"][ + "path" + ] = os.path.realpath(file_path) + assert locker.written_data == expected assert len(installer.installer.installs) == 2 @@ -989,6 +1009,12 @@ def test_run_installs_with_local_setuptools_directory( expected = fixture("with-directory-dependency-setuptools") + # Output includes the full path of the local dependency, which varies according to + # where the test is run... + expected["root_package"]["dependencies"]["project-with-setup"][ + "path" + ] = os.path.realpath(file_path) + assert locker.written_data == expected assert len(installer.installer.installs) == 3 diff --git a/tests/packages/test_locker.py b/tests/packages/test_locker.py index b6bb1837cc8..6d81abb6c28 100644 --- a/tests/packages/test_locker.py +++ b/tests/packages/test_locker.py @@ -55,7 +55,16 @@ def test_lock_file_data_is_ordered(locker: Locker, root: ProjectPackage): with locker.lock.open(encoding="utf-8") as f: content = f.read() - expected = """[[package]] + expected = """[root_package] +name = "root" +version = "1.2.3" +description = "" +category = "main" +optional = false +python-versions = "*" +files = [] + +[[package]] name = "A" version = "1.0.0" description = "" @@ -290,7 +299,16 @@ def test_lock_packages_with_null_description(locker: Locker, root: ProjectPackag with locker.lock.open(encoding="utf-8") as f: content = f.read() - expected = """[[package]] + expected = """[root_package] +name = "root" +version = "1.2.3" +description = "" +category = "main" +optional = false +python-versions = "*" +files = [] + +[[package]] name = "A" version = "1.0.0" description = "" @@ -321,7 +339,16 @@ def test_lock_file_should_not_have_mixed_types(locker: Locker, root: ProjectPack locker.set_lock_data(root, [package_a]) - expected = """[[package]] + expected = """[root_package] +name = "root" +version = "1.2.3" +description = "" +category = "main" +optional = false +python-versions = "*" +files = [] + +[[package]] name = "A" version = "1.0.0" description = "" @@ -402,7 +429,16 @@ def test_locking_legacy_repository_package_should_include_source_section( with locker.lock.open(encoding="utf-8") as f: content = f.read() - expected = """[[package]] + expected = """[root_package] +name = "root" +version = "1.2.3" +description = "" +category = "main" +optional = false +python-versions = "*" +files = [] + +[[package]] name = "A" version = "1.0.0" description = "" @@ -488,7 +524,16 @@ def test_extras_dependencies_are_ordered(locker: Locker, root: ProjectPackage): locker.set_lock_data(root, [package_a]) - expected = """[[package]] + expected = """[root_package] +name = "root" +version = "1.2.3" +description = "" +category = "main" +optional = false +python-versions = "*" +files = [] + +[[package]] name = "A" version = "1.0.0" description = "" @@ -578,7 +623,16 @@ def test_locker_dumps_dependency_information_correctly( with locker.lock.open(encoding="utf-8") as f: content = f.read() - expected = """[[package]] + expected = """[root_package] +name = "root" +version = "1.2.3" +description = "" +category = "main" +optional = false +python-versions = "*" +files = [] + +[[package]] name = "A" version = "1.0.0" description = "" diff --git a/tests/utils/test_exporter.py b/tests/utils/test_exporter.py index 53dfd0a2282..ac3714d993d 100644 --- a/tests/utils/test_exporter.py +++ b/tests/utils/test_exporter.py @@ -472,7 +472,10 @@ def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers( @pytest.mark.parametrize( ["dev", "lines"], - [(False, ['a==1.2.3 ; python_version < "3.8"']), (True, ["a==1.2.3", "b==4.5.6"])], + [ + (False, ['a==1.2.3 ; python_version < "3.8"']), + (True, ['a==1.2.3 ; python_version < "3.8"', "b==4.5.6"]), + ], ) def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_any( tmp_dir: str, poetry: "Poetry", dev: bool, lines: List[str] @@ -1775,3 +1778,130 @@ def test_exporter_exports_requirements_txt_to_standard_output( """ assert out == expected + + +def test_exporter_doesnt_confuse_repeated_packages( + tmp_dir: str, poetry: "Poetry", capsys: "CaptureFixture" +): + # Testcase derived from . + poetry.locker.mock_lock_data( + { + "root_package": { + "name": "foo", + "version": "1.2.3", + "category": "main", + "optional": False, + "python-versions": "^3.6", + "dependencies": { + "celery": [ + { + "version": ">=5.1.0,<6.0", + "markers": "python_version < '3.7'", + }, + { + "version": ">=5.2.0,<6.0", + "markers": "python_version >= '3.7'", + }, + ] + }, + }, + "package": [ + { + "name": "celery", + "version": "5.1.2", + "category": "main", + "optional": False, + "python-versions": "<3.7", + "dependencies": { + "click": ">=7.0,<8.0", + "click-didyoumean": ">=0.0.3", + "click-plugins": ">=1.1.1", + }, + }, + { + "name": "celery", + "version": "5.2.3", + "category": "main", + "optional": False, + "python-versions": ">=3.7", + "dependencies": { + "click": ">=8.0.3,<9.0", + "click-didyoumean": ">=0.0.3", + "click-plugins": ">=1.1.1", + }, + }, + { + "name": "click", + "version": "7.1.2", + "category": "main", + "optional": False, + "python-versions": ( + ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + ), + }, + { + "name": "click", + "version": "8.0.3", + "category": "main", + "optional": False, + "python-versions": ">=3.6", + "dependencies": {}, + }, + { + "name": "click-didyoumean", + "version": "0.0.3", + "category": "main", + "optional": False, + "python-versions": "*", + "dependencies": {"click": "*"}, + }, + { + "name": "click-didyoumean", + "version": "0.3.0", + "category": "main", + "optional": False, + "python-versions": ">=3.6.2,<4.0.0", + "dependencies": {"click": ">=7"}, + }, + { + "name": "click-plugins", + "version": "1.1.1", + "category": "main", + "optional": False, + "python-versions": "*", + "dependencies": {"click": ">=4.0"}, + }, + ], + "metadata": { + "lock-version": "1.1", + "python-versions": "^3.6", + "content-hash": ( + "832b13a88e5020c27cbcd95faa577bf0dbf054a65c023b45dc9442b640d414e6" + ), + "hashes": { + "celery": [], + "click-didyoumean": [], + "click-plugins": [], + "click": [], + }, + }, + } + ) + set_package_requires(poetry) + + exporter = Exporter(poetry) + + exporter.export("requirements.txt", Path(tmp_dir), sys.stdout) + + out, err = capsys.readouterr() + expected = ( + 'celery==5.1.2 ; python_version >= "3.6" and python_version < "3.7"\n' + 'celery==5.2.3 ; python_version >= "3.7" and python_version < "4.0"\n' + 'click-didyoumean==0.0.3 ; python_version >= "3.6" and python_version < "3.7"\n' # noqa: E501 + 'click-didyoumean==0.3.0 ; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.7" and python_version < "4.0"\n' # noqa: E501 + 'click-plugins==1.1.1 ; python_version >= "3.6" and python_version < "3.7" or python_version >= "3.7" and python_version < "4.0"\n' # noqa: E501 + 'click==7.1.2 ; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.7" or python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.5.0"\n' # noqa: E501 + 'click==8.0.3 ; python_version >= "3.7" and python_version < "4.0" or python_full_version >= "3.6.2" and python_full_version < "4.0.0" and python_version >= "3.7" and python_version < "4.0"\n' # noqa: E501 + ) + + assert out == expected