From 0d203b5bd0b0fd296711ca77104e9bd39424afe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Eustace?= Date: Thu, 1 Apr 2021 18:39:28 +0200 Subject: [PATCH] Read PEP-610-compliant files when loading installed packages --- poetry/repositories/installed_repository.py | 194 +++++++++++++----- .../METADATA | 6 + .../direct_url.json | 4 + .../METADATA | 6 + .../direct_url.json | 6 + .../file_pep_610-1.2.3.dist-info/METADATA | 6 + .../direct_url.json | 6 + .../git_pep_610-1.2.3.dist-info/METADATA | 6 + .../direct_url.json | 8 + .../url_pep_610-1.2.3.dist-info/METADATA | 6 + .../direct_url.json | 4 + .../repositories/test_installed_repository.py | 64 ++++++ 12 files changed, 264 insertions(+), 52 deletions(-) create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/directory_pep_610-1.2.3.dist-info/METADATA create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/directory_pep_610-1.2.3.dist-info/direct_url.json create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/editable_directory_pep_610-1.2.3.dist-info/METADATA create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/editable_directory_pep_610-1.2.3.dist-info/direct_url.json create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/file_pep_610-1.2.3.dist-info/METADATA create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/file_pep_610-1.2.3.dist-info/direct_url.json create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/git_pep_610-1.2.3.dist-info/METADATA create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/git_pep_610-1.2.3.dist-info/direct_url.json create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610-1.2.3.dist-info/METADATA create mode 100644 tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610-1.2.3.dist-info/direct_url.json diff --git a/poetry/repositories/installed_repository.py b/poetry/repositories/installed_repository.py index f33bbd9b99f..b97921d7271 100644 --- a/poetry/repositories/installed_repository.py +++ b/poetry/repositories/installed_repository.py @@ -1,10 +1,17 @@ import itertools +import json from pathlib import Path +from typing import TYPE_CHECKING from typing import Set +from typing import Tuple from typing import Union +from urllib.parse import unquote +from urllib.parse import urlparse from poetry.core.packages.package import Package +from poetry.core.packages.utils.utils import url_to_path +from poetry.core.utils.helpers import canonicalize_name from poetry.core.utils.helpers import module_name from poetry.utils._compat import metadata from poetry.utils.env import Env @@ -14,6 +21,9 @@ _VENDORS = Path(__file__).parent.parent.joinpath("_vendor") +if TYPE_CHECKING: + from importlib.metadata import Distribution + try: FileNotFoundError @@ -68,21 +78,14 @@ def get_package_paths(cls, env: Env, name: str) -> Set[Path]: return paths @classmethod - def set_package_vcs_properties_from_path(cls, src: Path, package: Package) -> None: + def get_package_vcs_properties_from_path(cls, src: Path) -> Tuple[str, str, str]: from poetry.core.vcs.git import Git git = Git() revision = git.rev_parse("HEAD", src).strip() url = git.remote_url(src) - package._source_type = "git" - package._source_url = url - package._source_reference = revision - - @classmethod - def set_package_vcs_properties(cls, package: Package, env: Env) -> None: - src = env.path / "src" / package.name - cls.set_package_vcs_properties_from_path(src, package) + return "git", url, revision @classmethod def is_vcs_package(cls, package: Union[Path, Package], env: Env) -> bool: @@ -99,6 +102,125 @@ def is_vcs_package(cls, package: Union[Path, Package], env: Env) -> bool: else: return True + @classmethod + def create_package_from_distribution( + cls, distribution: "Distribution", env: "Env" + ) -> Package: + # We first check for a direct_url.json file to determine + # the type of package. + path = Path(str(distribution._path)) + + if ( + path.name.endswith(".dist-info") + and path.joinpath("direct_url.json").exists() + ): + return cls.create_package_from_pep610(distribution) + + is_standard_package = env.is_path_relative_to_lib(path) + + source_type = None + source_url = None + source_reference = None + source_resolved_reference = None + if is_standard_package: + if path.name.endswith(".dist-info"): + paths = cls.get_package_paths( + env=env, name=distribution.metadata["name"] + ) + if paths: + is_editable_package = False + for src in paths: + if cls.is_vcs_package(src, env): + ( + source_type, + source_url, + source_reference, + ) = cls.get_package_vcs_properties_from_path(src) + break + + if not ( + is_editable_package or env.is_path_relative_to_lib(src) + ): + is_editable_package = True + else: + # TODO: handle multiple source directories? + if is_editable_package: + source_type = "directory" + source_url = paths.pop().as_posix() + else: + if cls.is_vcs_package(path, env): + ( + source_type, + source_url, + source_reference, + ) = cls.get_package_vcs_properties_from_path( + env.path / "src" / canonicalize_name(distribution.metadata["name"]) + ) + else: + # If not, it's a path dependency + source_type = "directory" + source_url = str(path.parent) + + package = Package( + distribution.metadata["name"], + distribution.metadata["version"], + source_type=source_type, + source_url=source_url, + source_reference=source_reference, + source_resolved_reference=source_resolved_reference, + ) + package.description = distribution.metadata.get("summary", "") + + return package + + @classmethod + def create_package_from_pep610(cls, distribution: "Distribution") -> Package: + path = Path(str(distribution._path)) + source_type = None + source_url = None + source_reference = None + source_resolved_reference = None + develop = False + + url_reference = json.loads( + path.joinpath("direct_url.json").read_text(encoding="utf-8") + ) + if "archive_info" in url_reference: + # File or URL distribution + if url_reference["url"].startswith("file:"): + # File distribution + source_type = "file" + source_url = url_to_path(url_reference["url"]).as_posix() + else: + # URL distribution + source_type = "url" + source_url = url_reference["url"] + elif "dir_info" in url_reference: + # Directory distribution + source_type = "directory" + source_url = url_to_path(url_reference["url"]).as_posix() + develop = url_reference["dir_info"].get("editable", False) + elif "vcs_info" in url_reference: + # VCS distribution + source_type = url_reference["vcs_info"]["vcs"] + source_url = url_reference["url"] + source_reference = url_reference["vcs_info"]["requested_revision"] + source_resolved_reference = url_reference["vcs_info"]["commit_id"] + + package = Package( + distribution.metadata["name"], + distribution.metadata["version"], + source_type=source_type, + source_url=source_url, + source_reference=source_reference, + source_resolved_reference=source_resolved_reference, + develop=develop, + ) + + package.description = distribution.metadata.get("summary", "") + + return package + @classmethod def load(cls, env: Env, with_dependencies: bool = False) -> "InstalledRepository": """ @@ -114,20 +236,13 @@ def load(cls, env: Env, with_dependencies: bool = False) -> "InstalledRepository metadata.distributions(path=[entry]), key=lambda d: str(d._path), ): - name = distribution.metadata["name"] - path = Path(str(distribution._path)) - version = distribution.metadata["version"] - package = Package(name, version, version) - package.description = distribution.metadata.get("summary", "") + name = canonicalize_name(distribution.metadata["name"]) - if with_dependencies: - for require in distribution.metadata.get_all("requires-dist", []): - dep = Dependency.create_from_pep_508(require) - package.add_dependency(dep) - - if package.name in seen: + if name in seen: continue + path = Path(str(distribution._path)) + try: path.relative_to(_VENDORS) except ValueError: @@ -135,39 +250,14 @@ def load(cls, env: Env, with_dependencies: bool = False) -> "InstalledRepository else: continue - seen.add(package.name) - - repo.add_package(package) + package = cls.create_package_from_distribution(distribution, env) - is_standard_package = env.is_path_relative_to_lib(path) - - if is_standard_package: - if path.name.endswith(".dist-info"): - paths = cls.get_package_paths(env=env, name=package.pretty_name) - if paths: - is_editable_package = False - for src in paths: - if cls.is_vcs_package(src, env): - cls.set_package_vcs_properties(package, env) - break - - if not ( - is_editable_package - or env.is_path_relative_to_lib(src) - ): - is_editable_package = True - else: - # TODO: handle multiple source directories? - if is_editable_package: - package._source_type = "directory" - package._source_url = paths.pop().as_posix() - continue + if with_dependencies: + for require in distribution.metadata.get_all("requires-dist", []): + dep = Dependency.create_from_pep_508(require) + package.add_dependency(dep) - if cls.is_vcs_package(path, env): - cls.set_package_vcs_properties(package, env) - else: - # If not, it's a path dependency - package._source_type = "directory" - package._source_url = str(path.parent) + seen.add(package.name) + repo.add_package(package) return repo diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/directory_pep_610-1.2.3.dist-info/METADATA b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/directory_pep_610-1.2.3.dist-info/METADATA new file mode 100644 index 00000000000..30928a39f0c --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/directory_pep_610-1.2.3.dist-info/METADATA @@ -0,0 +1,6 @@ +Metadata-Version: 2.1 +Name: directory-pep-610 +Version: 1.2.3 +Summary: Foo +License: MIT +Requires-Python: >=3.6 diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/directory_pep_610-1.2.3.dist-info/direct_url.json b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/directory_pep_610-1.2.3.dist-info/direct_url.json new file mode 100644 index 00000000000..3385611ce74 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/directory_pep_610-1.2.3.dist-info/direct_url.json @@ -0,0 +1,4 @@ +{ + "url": "file:///path/to/distributions/directory-pep-610", + "dir_info": {} +} diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/editable_directory_pep_610-1.2.3.dist-info/METADATA b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/editable_directory_pep_610-1.2.3.dist-info/METADATA new file mode 100644 index 00000000000..337c6fc4301 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/editable_directory_pep_610-1.2.3.dist-info/METADATA @@ -0,0 +1,6 @@ +Metadata-Version: 2.1 +Name: editable-directory-pep-610 +Version: 1.2.3 +Summary: Foo +License: MIT +Requires-Python: >=3.6 diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/editable_directory_pep_610-1.2.3.dist-info/direct_url.json b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/editable_directory_pep_610-1.2.3.dist-info/direct_url.json new file mode 100644 index 00000000000..e45f7c31a35 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/editable_directory_pep_610-1.2.3.dist-info/direct_url.json @@ -0,0 +1,6 @@ +{ + "url": "file:///path/to/distributions/directory-pep-610", + "dir_info": { + "editable": true + } +} diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/file_pep_610-1.2.3.dist-info/METADATA b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/file_pep_610-1.2.3.dist-info/METADATA new file mode 100644 index 00000000000..9478ca1f064 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/file_pep_610-1.2.3.dist-info/METADATA @@ -0,0 +1,6 @@ +Metadata-Version: 2.1 +Name: file-pep-610 +Version: 1.2.3 +Summary: Foo +License: MIT +Requires-Python: >=3.6 diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/file_pep_610-1.2.3.dist-info/direct_url.json b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/file_pep_610-1.2.3.dist-info/direct_url.json new file mode 100644 index 00000000000..d481649fa32 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/file_pep_610-1.2.3.dist-info/direct_url.json @@ -0,0 +1,6 @@ +{ + "url": "file:///path/to/distributions/file-pep-610-1.2.3.tar.gz", + "archive_info": { + "hash": "sha256=2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae" + } +} diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/git_pep_610-1.2.3.dist-info/METADATA b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/git_pep_610-1.2.3.dist-info/METADATA new file mode 100644 index 00000000000..bfc73cf720c --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/git_pep_610-1.2.3.dist-info/METADATA @@ -0,0 +1,6 @@ +Metadata-Version: 2.1 +Name: git-pep-610 +Version: 1.2.3 +Summary: Foo +License: MIT +Requires-Python: >=3.6 diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/git_pep_610-1.2.3.dist-info/direct_url.json b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/git_pep_610-1.2.3.dist-info/direct_url.json new file mode 100644 index 00000000000..a3115254845 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/git_pep_610-1.2.3.dist-info/direct_url.json @@ -0,0 +1,8 @@ +{ + "url": "https://github.com/demo/git-pep-610.git", + "vcs_info": { + "vcs": "git", + "requested_revision": "my-branch", + "commit_id": "123456" + } +} diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610-1.2.3.dist-info/METADATA b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610-1.2.3.dist-info/METADATA new file mode 100644 index 00000000000..7b2afd3189b --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610-1.2.3.dist-info/METADATA @@ -0,0 +1,6 @@ +Metadata-Version: 2.1 +Name: url-pep-610 +Version: 1.2.3 +Summary: Foo +License: MIT +Requires-Python: >=3.6 diff --git a/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610-1.2.3.dist-info/direct_url.json b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610-1.2.3.dist-info/direct_url.json new file mode 100644 index 00000000000..b36e4055294 --- /dev/null +++ b/tests/repositories/fixtures/installed/lib/python3.7/site-packages/url_pep_610-1.2.3.dist-info/direct_url.json @@ -0,0 +1,4 @@ +{ + "url": "https://python-poetry.org/distributions/url-pep-610-1.2.3.tar.gz", + "archive_info": {} +} diff --git a/tests/repositories/test_installed_repository.py b/tests/repositories/test_installed_repository.py index d10ab37af1d..5342181588d 100644 --- a/tests/repositories/test_installed_repository.py +++ b/tests/repositories/test_installed_repository.py @@ -30,6 +30,13 @@ metadata.PathDistribution(SITE_PURELIB / "editable-with-import-2.3.4.dist-info"), metadata.PathDistribution(SITE_PLATLIB / "lib64-2.3.4.dist-info"), metadata.PathDistribution(SITE_PLATLIB / "bender-2.0.5.dist-info"), + metadata.PathDistribution(SITE_PURELIB / "git_pep_610-1.2.3.dist-info"), + metadata.PathDistribution(SITE_PURELIB / "url_pep_610-1.2.3.dist-info"), + metadata.PathDistribution(SITE_PURELIB / "file_pep_610-1.2.3.dist-info"), + metadata.PathDistribution(SITE_PURELIB / "directory_pep_610-1.2.3.dist-info"), + metadata.PathDistribution( + SITE_PURELIB / "editable_directory_pep_610-1.2.3.dist-info" + ), ] @@ -165,3 +172,60 @@ def test_load_standard_package_with_pth_file(repository): assert standard.version.text == "1.2.3" assert standard.source_type is None assert standard.source_url is None + + +def test_load_pep_610_compliant_git_packages(repository): + package = get_package_from_repository("git-pep-610", repository) + + assert package is not None + assert package.name == "git-pep-610" + assert package.version.text == "1.2.3" + assert package.source_type == "git" + assert package.source_url == "https://github.com/demo/git-pep-610.git" + assert package.source_reference == "my-branch" + assert package.source_resolved_reference == "123456" + + +def test_load_pep_610_compliant_url_packages(repository): + package = get_package_from_repository("url-pep-610", repository) + + assert package is not None + assert package.name == "url-pep-610" + assert package.version.text == "1.2.3" + assert package.source_type == "url" + assert ( + package.source_url + == "https://python-poetry.org/distributions/url-pep-610-1.2.3.tar.gz" + ) + + +def test_load_pep_610_compliant_file_packages(repository): + package = get_package_from_repository("file-pep-610", repository) + + assert package is not None + assert package.name == "file-pep-610" + assert package.version.text == "1.2.3" + assert package.source_type == "file" + assert package.source_url == "/path/to/distributions/file-pep-610-1.2.3.tar.gz" + + +def test_load_pep_610_compliant_directory_packages(repository): + package = get_package_from_repository("directory-pep-610", repository) + + assert package is not None + assert package.name == "directory-pep-610" + assert package.version.text == "1.2.3" + assert package.source_type == "directory" + assert package.source_url == "/path/to/distributions/directory-pep-610" + assert not package.develop + + +def test_load_pep_610_compliant_editable_directory_packages(repository): + package = get_package_from_repository("editable-directory-pep-610", repository) + + assert package is not None + assert package.name == "editable-directory-pep-610" + assert package.version.text == "1.2.3" + assert package.source_type == "directory" + assert package.source_url == "/path/to/distributions/directory-pep-610" + assert package.develop