From 9b8ab1ea442064ee5ae5cafc1b66137281813c81 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Tue, 24 May 2022 03:43:36 +1000 Subject: [PATCH] Init: pypa/installer (#700) * . * . * . * . * Fixes from review. Co-authored-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com> --- python/pip_install/extract_wheels/lib/BUILD | 2 +- .../pip_install/extract_wheels/lib/bazel.py | 19 +++--- .../pip_install/extract_wheels/lib/wheel.py | 63 ++++++++----------- python/pip_install/repositories.bzl | 10 +-- 4 files changed, 41 insertions(+), 53 deletions(-) diff --git a/python/pip_install/extract_wheels/lib/BUILD b/python/pip_install/extract_wheels/lib/BUILD index 3e2f307a73..48214126e4 100644 --- a/python/pip_install/extract_wheels/lib/BUILD +++ b/python/pip_install/extract_wheels/lib/BUILD @@ -18,7 +18,7 @@ py_library( "//python/pip_install/parse_requirements_to_bzl:__subpackages__", ], deps = [ - requirement("pkginfo"), + requirement("installer"), requirement("setuptools"), ], ) diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py index 9876e6cdf0..4b6a4ff0dd 100644 --- a/python/pip_install/extract_wheels/lib/bazel.py +++ b/python/pip_install/extract_wheels/lib/bazel.py @@ -21,29 +21,28 @@ def generate_entry_point_contents( - entry_point: str, shebang: str = "#!/usr/bin/env python3" + module: str, attribute: str, shebang: str = "#!/usr/bin/env python3" ) -> str: """Generate the contents of an entry point script. Args: - entry_point (str): The name of the entry point as show in the - `console_scripts` section of `entry_point.txt`. + module (str): The name of the module to use. + attribute (str): The name of the attribute to call. shebang (str, optional): The shebang to use for the entry point python file. Returns: str: A string of python code. """ - module, method = entry_point.split(":", 1) return textwrap.dedent( """\ {shebang} import sys - from {module} import {method} + from {module} import {attribute} if __name__ == "__main__": - sys.exit({method}()) + sys.exit({attribute}()) """.format( - shebang=shebang, module=module, method=method + shebang=shebang, module=module, attribute=attribute ) ) @@ -408,14 +407,14 @@ def extract_wheel( directory_path = Path(directory) entry_points = [] - for name, entry_point in sorted(whl.entry_points().items()): + for name, (module, attribute) in sorted(whl.entry_points().items()): # There is an extreme edge-case with entry_points that end with `.py` # See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174 entry_point_without_py = name[:-3] if name.endswith(".py") else name entry_point_target_name = f"{WHEEL_ENTRY_POINT_PREFIX}_{entry_point_without_py}" entry_point_script_name = f"{entry_point_target_name}.py" (directory_path / entry_point_script_name).write_text( - generate_entry_point_contents(entry_point) + generate_entry_point_contents(module, attribute) ) entry_points.append( generate_entry_point_rule( @@ -454,7 +453,7 @@ def extract_wheel( data_exclude=data_exclude, data=data, srcs_exclude=srcs_exclude, - tags=["pypi_name=" + whl.name, "pypi_version=" + whl.metadata.version], + tags=["pypi_name=" + whl.name, "pypi_version=" + whl.version], additional_content=additional_content, ) build_file.write(contents) diff --git a/python/pip_install/extract_wheels/lib/wheel.py b/python/pip_install/extract_wheels/lib/wheel.py index 85bc95830d..73d5eb53da 100644 --- a/python/pip_install/extract_wheels/lib/wheel.py +++ b/python/pip_install/extract_wheels/lib/wheel.py @@ -1,13 +1,13 @@ """Utility class to inspect an extracted wheel directory""" -import configparser +import email import glob import os import stat import zipfile from typing import Dict, Optional, Set +import installer import pkg_resources -import pkginfo def current_umask() -> int: @@ -37,57 +37,46 @@ def path(self) -> str: @property def name(self) -> str: - return str(self.metadata.name) + # TODO Also available as installer.sources.WheelSource.distribution + return str(self.metadata['Name']) @property - def metadata(self) -> pkginfo.Wheel: - return pkginfo.get_metadata(self.path) + def metadata(self) -> email.message.Message: + with installer.sources.WheelFile.open(self.path) as wheel_source: + metadata_contents = wheel_source.read_dist_info("METADATA") + metadata = installer.utils.parse_metadata_file(metadata_contents) + return metadata - def entry_points(self) -> Dict[str, str]: + @property + def version(self) -> str: + # TODO Also available as installer.sources.WheelSource.version + return str(self.metadata["Version"]) + + def entry_points(self) -> Dict[str, tuple[str, str]]: """Returns the entrypoints defined in the current wheel See https://packaging.python.org/specifications/entry-points/ for more info Returns: - Dict[str, str]: A mappying of the entry point's name to it's method + Dict[str, Tuple[str, str]]: A mapping of the entry point's name to it's module and attribute """ - with zipfile.ZipFile(self.path, "r") as whl: - # Calculate the location of the entry_points.txt file - metadata = self.metadata - name = "{}-{}".format(metadata.name.replace("-", "_"), metadata.version) - - # Note that the zipfile module always uses the forward slash as - # directory separator, even on Windows, so don't use os.path.join - # here. Reference for Python 3.10: - # https://github.com/python/cpython/blob/3.10/Lib/zipfile.py#L355. - # TODO: use zipfile.Path once 3.8 is our minimum supported version - entry_points_path = "{}.dist-info/entry_points.txt".format(name) - - # If this file does not exist in the wheel, there are no entry points - if entry_points_path not in whl.namelist(): + with installer.sources.WheelFile.open(self.path) as wheel_source: + if "entry_points.txt" not in wheel_source.dist_info_filenames: return dict() - # Parse the avaialble entry points - config = configparser.ConfigParser() - try: - config.read_string(whl.read(entry_points_path).decode("utf-8")) - if "console_scripts" in config.sections(): - return dict(config["console_scripts"]) - - # TODO: It's unclear what to do in a situation with duplicate sections or options. - # For now, we treat the config file as though it contains no scripts. For more - # details on the config parser, see: - # https://docs.python.org/3.7/library/configparser.html#configparser.ConfigParser - # https://docs.python.org/3.7/library/configparser.html#configparser.Error - except configparser.Error: - pass + entry_points_mapping = dict() + entry_points_contents = wheel_source.read_dist_info("entry_points.txt") + entry_points = installer.utils.parse_entrypoints(entry_points_contents) + for script, module, attribute, script_section in entry_points: + if script_section == "console": + entry_points_mapping[script] = (module, attribute) - return dict() + return entry_points_mapping def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: dependency_set = set() - for wheel_req in self.metadata.requires_dist: + for wheel_req in self.metadata.get_all('Requires-Dist', []): req = pkg_resources.Requirement(wheel_req) # type: ignore if req.marker is None or any( diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 352f66341c..a57ff9deaa 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -17,6 +17,11 @@ _RULE_DEPS = [ "https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl", "9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2", ), + ( + "pypi__installer", + "https://files.pythonhosted.org/packages/1b/21/3e6ebd12d8dccc55bcb7338db462c75ac86dbd0ac7439ac114616b21667b/installer-0.5.1-py3-none-any.whl", + "1d6c8d916ed82771945b9c813699e6f57424ded970c9d8bf16bbc23e1e826ed3", + ), ( "pypi__pip", "https://files.pythonhosted.org/packages/4d/16/0a14ca596f30316efd412a60bdfac02a7259bf8673d4d917dc60b9a21812/pip-22.0.4-py3-none-any.whl", @@ -27,11 +32,6 @@ _RULE_DEPS = [ "https://files.pythonhosted.org/packages/6d/16/75d65bdccd48bb59a08e2bf167b01d8532f65604270d0a292f0f16b7b022/pip_tools-5.5.0-py2.py3-none-any.whl", "10841c1e56c234d610d0466447685b9ea4ee4a2c274f858c0ef3c33d9bd0d985", ), - ( - "pypi__pkginfo", - "https://files.pythonhosted.org/packages/cd/00/49f59cdd2c6a52e6665fda4de671dac5614366dc827e050c55428241b929/pkginfo-1.8.2-py2.py3-none-any.whl", - "c24c487c6a7f72c66e816ab1796b96ac6c3d14d49338293d2141664330b55ffc", - ), ( "pypi__setuptools", "https://files.pythonhosted.org/packages/7c/5b/3d92b9f0f7ca1645cba48c080b54fe7d8b1033a4e5720091d1631c4266db/setuptools-60.10.0-py3-none-any.whl",