From 658f641037b5879a79d6b37515934054a790a9c1 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 24 Jul 2020 21:07:03 +0200 Subject: [PATCH 1/3] Account for pure/plat lib differences in env When detecting installed packages, ensure that we check both purelib and platlib site directories when determining if a package is a standard package. --- poetry/repositories/installed_repository.py | 6 +- poetry/utils/env.py | 31 ++++++++- .../lib64-2.3.4.dist-info/METADATA | 22 +++++++ .../repositories/test_installed_repository.py | 64 ++++++++++++++----- 4 files changed, 101 insertions(+), 22 deletions(-) create mode 100644 tests/repositories/fixtures/installed/lib64/python3.7/site-packages/lib64-2.3.4.dist-info/METADATA diff --git a/poetry/repositories/installed_repository.py b/poetry/repositories/installed_repository.py index 296b5c9ecec..be1ab7cdaa3 100644 --- a/poetry/repositories/installed_repository.py +++ b/poetry/repositories/installed_repository.py @@ -75,11 +75,7 @@ def load(cls, env): # type: (Env) -> InstalledRepository repo.add_package(package) - is_standard_package = True - try: - path.relative_to(env.site_packages) - except ValueError: - is_standard_package = False + is_standard_package = env.is_path_relative_to_lib(path) if is_standard_package: if path.name.endswith(".dist-info"): diff --git a/poetry/utils/env.py b/poetry/utils/env.py index a989438e025..2c07b0b764e 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -754,6 +754,8 @@ def __init__(self, path, base=None): # type: (Path, Optional[Path]) -> None self._site_packages = None self._paths = None self._supported_tags = None + self._purelib = None + self._platlib = None @property def path(self): # type: () -> Path @@ -810,10 +812,37 @@ def pip_version(self): @property def site_packages(self): # type: () -> Path if self._site_packages is None: - self._site_packages = Path(self.paths["purelib"]) + self._site_packages = Path(self.purelib) return self._site_packages + @property + def purelib(self): # type: () -> Path + if self._purelib is None: + self._purelib = Path(self.paths["purelib"]) + + return self._purelib + + @property + def platlib(self): # type: () -> Path + if self._platlib is None: + if "platlib" in self.paths: + self._platlib = Path(self.paths["platlib"]) + else: + self._platlib = self.purelib + + return self._platlib + + def is_path_relative_to_lib(self, path): # type: (Path) -> bool + for lib_path in [self.purelib, self.platlib]: + try: + path.relative_to(lib_path) + return True + except ValueError: + pass + + return False + @property def sys_path(self): # type: () -> List[str] raise NotImplementedError() diff --git a/tests/repositories/fixtures/installed/lib64/python3.7/site-packages/lib64-2.3.4.dist-info/METADATA b/tests/repositories/fixtures/installed/lib64/python3.7/site-packages/lib64-2.3.4.dist-info/METADATA new file mode 100644 index 00000000000..d9d9c5d0a94 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib64/python3.7/site-packages/lib64-2.3.4.dist-info/METADATA @@ -0,0 +1,22 @@ +Metadata-Version: 2.1 +Name: lib64 +Version: 2.3.4 +Summary: lib64 description. +License: MIT +Keywords: cli,commands +Author: Foo Bar +Author-email: foo@bar.com +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Description-Content-Type: text/x-rst + +lib64 +#### diff --git a/tests/repositories/test_installed_repository.py b/tests/repositories/test_installed_repository.py index 20443882b84..88a241a7df7 100644 --- a/tests/repositories/test_installed_repository.py +++ b/tests/repositories/test_installed_repository.py @@ -1,5 +1,8 @@ +from typing import Optional + import pytest +from poetry.core.packages import Package from poetry.repositories.installed_repository import InstalledRepository from poetry.utils._compat import PY36 from poetry.utils._compat import Path @@ -11,25 +14,34 @@ FIXTURES_DIR = Path(__file__).parent / "fixtures" ENV_DIR = (FIXTURES_DIR / "installed").resolve() -SITE_PACKAGES = ENV_DIR / "lib" / "python3.7" / "site-packages" +SITE_PURELIB = ENV_DIR / "lib" / "python3.7" / "site-packages" +SITE_PLATLIB = ENV_DIR / "lib64" / "python3.7" / "site-packages" SRC = ENV_DIR / "src" VENDOR_DIR = ENV_DIR / "vendor" / "py3.7" INSTALLED_RESULTS = [ - metadata.PathDistribution(SITE_PACKAGES / "cleo-0.7.6.dist-info"), + metadata.PathDistribution(SITE_PURELIB / "cleo-0.7.6.dist-info"), metadata.PathDistribution(SRC / "pendulum" / "pendulum.egg-info"), metadata.PathDistribution( - zipp.Path(str(SITE_PACKAGES / "foo-0.1.0-py3.8.egg"), "EGG-INFO") + zipp.Path(str(SITE_PURELIB / "foo-0.1.0-py3.8.egg"), "EGG-INFO") ), metadata.PathDistribution(VENDOR_DIR / "attrs-19.3.0.dist-info"), - metadata.PathDistribution(SITE_PACKAGES / "editable-2.3.4.dist-info"), - metadata.PathDistribution(SITE_PACKAGES / "editable-with-import-2.3.4.dist-info"), + metadata.PathDistribution(SITE_PURELIB / "editable-2.3.4.dist-info"), + metadata.PathDistribution(SITE_PURELIB / "editable-with-import-2.3.4.dist-info"), + metadata.PathDistribution(SITE_PLATLIB / "lib64-2.3.4.dist-info"), ] class MockEnv(BaseMockEnv): @property - def site_packages(self): # type: () -> Path - return SITE_PACKAGES + def paths(self): + return { + "purelib": SITE_PURELIB, + "platlib": SITE_PLATLIB, + } + + @property + def sys_path(self): + return [ENV_DIR, SITE_PLATLIB, SITE_PURELIB] @pytest.fixture @@ -58,17 +70,27 @@ def repository(mocker, env): # type: (MockFixture, MockEnv) -> InstalledReposit return InstalledRepository.load(env) +def get_package_from_repository( + name, repository +): # type: (str, InstalledRepository) -> Optional[Package] + for pkg in repository.packages: + if pkg.name == name: + return pkg + return None + + def test_load_successful(repository): - assert len(repository.packages) == 5 + assert len(repository.packages) == len(INSTALLED_RESULTS) - 1 def test_load_ensure_isolation(repository): - for pkg in repository.packages: - assert pkg.name != "attrs" + package = get_package_from_repository("attrs", repository) + assert package is None def test_load_standard_package(repository): - cleo = repository.packages[0] + cleo = get_package_from_repository("cleo", repository) + assert cleo is not None assert cleo.name == "cleo" assert cleo.version.text == "0.7.6" assert ( @@ -76,13 +98,14 @@ def test_load_standard_package(repository): == "Cleo allows you to create beautiful and testable command-line interfaces." ) - foo = repository.packages[3] - assert foo.name == "foo" + foo = get_package_from_repository("foo", repository) + assert foo is not None assert foo.version.text == "0.1.0" def test_load_git_package(repository): - pendulum = repository.packages[4] + pendulum = get_package_from_repository("pendulum", repository) + assert pendulum is not None assert pendulum.name == "pendulum" assert pendulum.version.text == "2.0.5" assert pendulum.description == "Python datetimes made easy" @@ -91,12 +114,20 @@ def test_load_git_package(repository): assert pendulum.source_reference == "bb058f6b78b2d28ef5d9a5e759cfa179a1a713d6" +def test_load_platlib_package(repository): + lib64 = get_package_from_repository("lib64", repository) + assert lib64 is not None + assert lib64.name == "lib64" + assert lib64.version.text == "2.3.4" + + @pytest.mark.skipif( not PY36, reason="pathlib.resolve() does not support strict argument" ) def test_load_editable_package(repository): # test editable package with text .pth file - editable = repository.packages[1] + editable = get_package_from_repository("editable", repository) + assert editable is not None assert editable.name == "editable" assert editable.version.text == "2.3.4" assert editable.source_type == "directory" @@ -108,7 +139,8 @@ def test_load_editable_package(repository): def test_load_editable_with_import_package(repository): # test editable package with executable .pth file - editable = repository.packages[2] + editable = get_package_from_repository("editable-with-import", repository) + assert editable is not None assert editable.name == "editable-with-import" assert editable.version.text == "2.3.4" assert editable.source_type == "" From b7aa9c2ab444bacd544b1872c3a0740dc064e8e9 Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 24 Jul 2020 21:10:23 +0200 Subject: [PATCH 2/3] Fix git installed package detection A package installed from git is sometimes mistaken as a standard package since it has a `.dist-info` directory. This change ensures that the pth file sources are also taken into consideration when determining if a package is git sourced. Additionally, this change ensures that a pth file is searched for in both platlib and purelib site directories. --- poetry/repositories/installed_repository.py | 108 +++++++++++++----- .../bender-2.0.5.dist-info/METADATA | 19 +++ .../lib64/python3.7/site-packages/bender.pth | 1 + .../src/bender/bender.egg-info/PKG-INFO | 19 +++ .../repositories/test_installed_repository.py | 14 ++- 5 files changed, 129 insertions(+), 32 deletions(-) create mode 100644 tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender-2.0.5.dist-info/METADATA create mode 100644 tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender.pth create mode 100644 tests/repositories/fixtures/installed/src/bender/bender.egg-info/PKG-INFO diff --git a/poetry/repositories/installed_repository.py b/poetry/repositories/installed_repository.py index be1ab7cdaa3..1b6fed488f7 100644 --- a/poetry/repositories/installed_repository.py +++ b/poetry/repositories/installed_repository.py @@ -1,6 +1,10 @@ +import itertools + from typing import Set +from typing import Union from poetry.core.packages import Package +from poetry.core.utils.helpers import module_name from poetry.utils._compat import Path from poetry.utils._compat import metadata from poetry.utils.env import Env @@ -11,11 +15,17 @@ _VENDORS = Path(__file__).parent.parent.joinpath("_vendor") +try: + FileNotFoundError +except NameError: + FileNotFoundError = OSError + + class InstalledRepository(Repository): @classmethod - def get_package_paths(cls, sitedir, name): # type: (Path, str) -> Set[Path] + def get_package_paths(cls, env, name): # type: (Env, str) -> Set[Path] """ - Process a .pth file within the site-packages directory, and return any valid + Process a .pth file within the site-packages directories, and return any valid paths. We skip executable .pth files as there is no reliable means to do this without side-effects to current run-time. Mo check is made that the item refers to a directory rather than a file, however, in order to maintain backwards @@ -24,25 +34,72 @@ def get_package_paths(cls, sitedir, name): # type: (Path, str) -> Set[Path] Reference: https://docs.python.org/3.8/library/site.html - :param sitedir: The site-packages directory to search for .pth file. + :param env: The environment to search for the .pth file in. :param name: The name of the package to search .pth file for. :return: A `Set` of valid `Path` objects. """ paths = set() - pth_file = sitedir.joinpath("{}.pth".format(name)) - if pth_file.exists(): + # we identify the candidate pth files to check, this is done so to handle cases + # where the pth file for foo-bar might have been installed as either foo-bar.pth or + # foo_bar.pth (expected) in either pure or platform lib directories. + candidates = itertools.product( + {env.purelib, env.platlib}, {name, module_name(name)}, + ) + + for lib, module in candidates: + pth_file = lib.joinpath(module).with_suffix(".pth") + if not pth_file.exists(): + continue + with pth_file.open() as f: for line in f: line = line.strip() if line and not line.startswith(("#", "import ", "import\t")): path = Path(line) if not path.is_absolute(): - path = sitedir.joinpath(path) + try: + path = lib.joinpath(path).resolve() + except FileNotFoundError: + # this is required to handle pathlib oddity on win32 python==3.5 + path = lib.joinpath(path) paths.add(path) - return paths + @classmethod + def set_package_vcs_properties_from_path( + cls, src, package + ): # type: (Path, Package) -> None + from poetry.core.vcs.git import Git + + git = Git() + revision = git.rev_parse("HEAD", src).strip() + url = git.remote_url(src) + + package.source_type = "git" + package.source_url = url + package.source_reference = revision + + @classmethod + def set_package_vcs_properties(cls, package, env): # type: (Package, Env) -> None + src = env.path / "src" / package.name + cls.set_package_vcs_properties_from_path(src, package) + + @classmethod + def is_vcs_package(cls, package, env): # type: (Union[Path, Package], Env) -> bool + # A VCS dependency should have been installed + # in the src directory. + src = env.path / "src" + if isinstance(package, Package): + return src.joinpath(package.name).is_dir() + + try: + package.relative_to(env.path / "src") + except ValueError: + return False + else: + return True + @classmethod def load(cls, env): # type: (Env) -> InstalledRepository """ @@ -79,33 +136,22 @@ def load(cls, env): # type: (Env) -> InstalledRepository if is_standard_package: if path.name.endswith(".dist-info"): - paths = cls.get_package_paths( - sitedir=env.site_packages, name=package.pretty_name - ) + paths = cls.get_package_paths(env=env, name=package.pretty_name) if paths: - # TODO: handle multiple source directories? - package.source_type = "directory" - package.source_url = paths.pop().as_posix() - + for src in paths: + if cls.is_vcs_package(src, env): + cls.set_package_vcs_properties(package, env) + break + else: + # TODO: handle multiple source directories? + package.source_type = "directory" + package.source_url = paths.pop().as_posix() continue - src_path = env.path / "src" - - # A VCS dependency should have been installed - # in the src directory. If not, it's a path dependency - try: - path.relative_to(src_path) - - from poetry.core.vcs.git import Git - - git = Git() - revision = git.rev_parse("HEAD", src_path / package.name).strip() - url = git.remote_url(src_path / package.name) - - package.source_type = "git" - package.source_url = url - package.source_reference = revision - except ValueError: + if cls.is_vcs_package(path, env): + cls.set_package_vcs_properties(package, env) + else: + # If not, it's a path dependency package.source_type = "directory" package.source_url = str(path.parent) diff --git a/tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender-2.0.5.dist-info/METADATA b/tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender-2.0.5.dist-info/METADATA new file mode 100644 index 00000000000..f7576123c28 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender-2.0.5.dist-info/METADATA @@ -0,0 +1,19 @@ +Metadata-Version: 2.1 +Name: bender +Version: 2.0.5 +Summary: Python datetimes made easy +License: MIT +Keywords: cli,commands +Author: Leela +Author-email: leela@planetexpress.com +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Description-Content-Type: text/x-rst diff --git a/tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender.pth b/tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender.pth new file mode 100644 index 00000000000..8450d61d59e --- /dev/null +++ b/tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender.pth @@ -0,0 +1 @@ +../../../src/bender \ No newline at end of file diff --git a/tests/repositories/fixtures/installed/src/bender/bender.egg-info/PKG-INFO b/tests/repositories/fixtures/installed/src/bender/bender.egg-info/PKG-INFO new file mode 100644 index 00000000000..f7576123c28 --- /dev/null +++ b/tests/repositories/fixtures/installed/src/bender/bender.egg-info/PKG-INFO @@ -0,0 +1,19 @@ +Metadata-Version: 2.1 +Name: bender +Version: 2.0.5 +Summary: Python datetimes made easy +License: MIT +Keywords: cli,commands +Author: Leela +Author-email: leela@planetexpress.com +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Description-Content-Type: text/x-rst diff --git a/tests/repositories/test_installed_repository.py b/tests/repositories/test_installed_repository.py index 88a241a7df7..b420a9726ed 100644 --- a/tests/repositories/test_installed_repository.py +++ b/tests/repositories/test_installed_repository.py @@ -28,6 +28,7 @@ metadata.PathDistribution(SITE_PURELIB / "editable-2.3.4.dist-info"), metadata.PathDistribution(SITE_PURELIB / "editable-with-import-2.3.4.dist-info"), metadata.PathDistribution(SITE_PLATLIB / "lib64-2.3.4.dist-info"), + metadata.PathDistribution(SITE_PLATLIB / "bender-2.0.5.dist-info"), ] @@ -110,10 +111,21 @@ def test_load_git_package(repository): assert pendulum.version.text == "2.0.5" assert pendulum.description == "Python datetimes made easy" assert pendulum.source_type == "git" - assert pendulum.source_url == "https://github.com/sdispater/pendulum.git" + assert pendulum.source_url in [ + "git@github.com:sdispater/pendulum.git", + "https://github.com/sdispater/pendulum.git", + ] assert pendulum.source_reference == "bb058f6b78b2d28ef5d9a5e759cfa179a1a713d6" +def test_load_git_package_pth(repository): + bender = get_package_from_repository("bender", repository) + assert bender is not None + assert bender.name == "bender" + assert bender.version.text == "2.0.5" + assert bender.source_type == "git" + + def test_load_platlib_package(repository): lib64 = get_package_from_repository("lib64", repository) assert lib64 is not None From e68797d754e0ca487d0162bd88dbf4408335898e Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 24 Jul 2020 22:09:30 +0200 Subject: [PATCH 3/3] tests: remove deprecated use of allows-prerelease --- tests/fixtures/complete.toml | 2 +- tests/installation/test_installer.py | 2 +- tests/installation/test_installer_old.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/complete.toml b/tests/fixtures/complete.toml index a894a89c689..ced35cb46e3 100644 --- a/tests/fixtures/complete.toml +++ b/tests/fixtures/complete.toml @@ -22,7 +22,7 @@ toml = "^0.9" # Dependencies with extras requests = { version = "^2.13", extras = [ "security" ] } # Python specific dependencies with prereleases allowed -pathlib2 = { version = "^2.2", python = "~2.7", allows-prereleases = true } +pathlib2 = { version = "^2.2", python = "~2.7", allow-prereleases = true } # Git dependencies cleo = { git = "https://github.com/sdispater/cleo.git", branch = "master" } diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index 1e76197ce7c..88103e7c323 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -890,7 +890,7 @@ def test_run_with_prereleases(installer, locker, repo, package): repo.add_package(package_a) repo.add_package(package_b) - package.add_dependency("A", {"version": "*", "allows-prereleases": True}) + package.add_dependency("A", {"version": "*", "allow-prereleases": True}) package.add_dependency("B", "^1.1") installer.update(True) diff --git a/tests/installation/test_installer_old.py b/tests/installation/test_installer_old.py index e50f8cc0202..3aab6831526 100644 --- a/tests/installation/test_installer_old.py +++ b/tests/installation/test_installer_old.py @@ -861,7 +861,7 @@ def test_run_with_prereleases(installer, locker, repo, package): repo.add_package(package_a) repo.add_package(package_b) - package.add_dependency("A", {"version": "*", "allows-prereleases": True}) + package.add_dependency("A", {"version": "*", "allow-prereleases": True}) package.add_dependency("B", "^1.1") installer.update(True)