Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Init: pypa/installer #700

Merged
merged 7 commits into from
May 23, 2022
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion python/pip_install/extract_wheels/lib/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ py_library(
"//python/pip_install/parse_requirements_to_bzl:__subpackages__",
],
deps = [
requirement("pkginfo"),
requirement("installer"),
requirement("setuptools"),
],
)
Expand Down
19 changes: 9 additions & 10 deletions python/pip_install/extract_wheels/lib/bazel.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,29 +21,28 @@


def generate_entry_point_contents(
entry_point: str, shebang: str = "#!/usr/bin/env python3"
module: str, attribute: str, shebang: str = "#!/usr/bin/env python3"
) -> str:
"""Generate the contents of an entry point script.

Args:
entry_point (str): The name of the entry point as show in the
`console_scripts` section of `entry_point.txt`.
module (str): The name of the module to use.
attribute (str): The name of the attribute to call.
shebang (str, optional): The shebang to use for the entry point python
file.

Returns:
str: A string of python code.
"""
module, method = entry_point.split(":", 1)
return textwrap.dedent(
"""\
{shebang}
import sys
from {module} import {method}
from {module} import {attribute}
groodt marked this conversation as resolved.
Show resolved Hide resolved
if __name__ == "__main__":
sys.exit({method}())
sys.exit({attribute}())
""".format(
shebang=shebang, module=module, method=method
shebang=shebang, module=module, attribute=attribute
)
)

Expand Down Expand Up @@ -408,14 +407,14 @@ def extract_wheel(

directory_path = Path(directory)
entry_points = []
for name, entry_point in sorted(whl.entry_points().items()):
for name, (module, attribute) in sorted(whl.entry_points().items()):
# There is an extreme edge-case with entry_points that end with `.py`
# See: https://github.com/bazelbuild/bazel/blob/09c621e4cf5b968f4c6cdf905ab142d5961f9ddc/src/test/java/com/google/devtools/build/lib/rules/python/PyBinaryConfiguredTargetTest.java#L174
entry_point_without_py = name[:-3] if name.endswith(".py") else name
entry_point_target_name = f"{WHEEL_ENTRY_POINT_PREFIX}_{entry_point_without_py}"
entry_point_script_name = f"{entry_point_target_name}.py"
(directory_path / entry_point_script_name).write_text(
generate_entry_point_contents(entry_point)
generate_entry_point_contents(module, attribute)
)
entry_points.append(
generate_entry_point_rule(
Expand Down Expand Up @@ -454,7 +453,7 @@ def extract_wheel(
data_exclude=data_exclude,
data=data,
srcs_exclude=srcs_exclude,
tags=["pypi_name=" + whl.name, "pypi_version=" + whl.metadata.version],
tags=["pypi_name=" + whl.name, "pypi_version=" + whl.version],
additional_content=additional_content,
)
build_file.write(contents)
Expand Down
63 changes: 26 additions & 37 deletions python/pip_install/extract_wheels/lib/wheel.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
"""Utility class to inspect an extracted wheel directory"""
import configparser
import email
import glob
import os
import stat
import zipfile
from typing import Dict, Optional, Set

import installer
import pkg_resources
import pkginfo


def current_umask() -> int:
Expand Down Expand Up @@ -37,57 +37,46 @@ def path(self) -> str:

@property
def name(self) -> str:
return str(self.metadata.name)
# TODO Also available as installer.sources.WheelSource.distribution
return str(self.metadata['Name'])

@property
def metadata(self) -> pkginfo.Wheel:
return pkginfo.get_metadata(self.path)
def metadata(self) -> email.message.Message:
with installer.sources.WheelFile.open(self.path) as wheel_source:
metadata_contents = wheel_source.read_dist_info("METADATA")
metadata = installer.utils.parse_metadata_file(metadata_contents)
return metadata

def entry_points(self) -> Dict[str, str]:
@property
def version(self) -> str:
# TODO Also available as installer.sources.WheelSource.version
return str(self.metadata["Version"])

def entry_points(self) -> Dict[str, tuple[str, str]]:
"""Returns the entrypoints defined in the current wheel

See https://packaging.python.org/specifications/entry-points/ for more info

Returns:
Dict[str, str]: A mappying of the entry point's name to it's method
Dict[str, Tuple[str, str]]: A mapping of the entry point's name to it's module and attribute
"""
with zipfile.ZipFile(self.path, "r") as whl:
# Calculate the location of the entry_points.txt file
metadata = self.metadata
name = "{}-{}".format(metadata.name.replace("-", "_"), metadata.version)

# Note that the zipfile module always uses the forward slash as
# directory separator, even on Windows, so don't use os.path.join
# here. Reference for Python 3.10:
# https://github.com/python/cpython/blob/3.10/Lib/zipfile.py#L355.
# TODO: use zipfile.Path once 3.8 is our minimum supported version
entry_points_path = "{}.dist-info/entry_points.txt".format(name)

# If this file does not exist in the wheel, there are no entry points
if entry_points_path not in whl.namelist():
with installer.sources.WheelFile.open(self.path) as wheel_source:
if "entry_points.txt" not in wheel_source.dist_info_filenames:
return dict()

# Parse the avaialble entry points
config = configparser.ConfigParser()
try:
config.read_string(whl.read(entry_points_path).decode("utf-8"))
if "console_scripts" in config.sections():
return dict(config["console_scripts"])

# TODO: It's unclear what to do in a situation with duplicate sections or options.
# For now, we treat the config file as though it contains no scripts. For more
# details on the config parser, see:
# https://docs.python.org/3.7/library/configparser.html#configparser.ConfigParser
# https://docs.python.org/3.7/library/configparser.html#configparser.Error
except configparser.Error:
pass
entry_points_mapping = dict()
entry_points_contents = wheel_source.read_dist_info("entry_points.txt")
entry_points = installer.utils.parse_entrypoints(entry_points_contents)
for script, module, attribute, kind in entry_points:
if kind == "console":
groodt marked this conversation as resolved.
Show resolved Hide resolved
entry_points_mapping[script] = (module, attribute)

return dict()
return entry_points_mapping

def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]:
dependency_set = set()

for wheel_req in self.metadata.requires_dist:
for wheel_req in self.metadata.get_all('Requires-Dist', list()):
groodt marked this conversation as resolved.
Show resolved Hide resolved
req = pkg_resources.Requirement(wheel_req) # type: ignore

if req.marker is None or any(
Expand Down
10 changes: 5 additions & 5 deletions python/pip_install/repositories.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ _RULE_DEPS = [
"https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl",
"9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2",
),
(
"pypi__installer",
"https://files.pythonhosted.org/packages/1b/21/3e6ebd12d8dccc55bcb7338db462c75ac86dbd0ac7439ac114616b21667b/installer-0.5.1-py3-none-any.whl",
"1d6c8d916ed82771945b9c813699e6f57424ded970c9d8bf16bbc23e1e826ed3",
),
(
"pypi__pip",
"https://files.pythonhosted.org/packages/4d/16/0a14ca596f30316efd412a60bdfac02a7259bf8673d4d917dc60b9a21812/pip-22.0.4-py3-none-any.whl",
Expand All @@ -27,11 +32,6 @@ _RULE_DEPS = [
"https://files.pythonhosted.org/packages/6d/16/75d65bdccd48bb59a08e2bf167b01d8532f65604270d0a292f0f16b7b022/pip_tools-5.5.0-py2.py3-none-any.whl",
"10841c1e56c234d610d0466447685b9ea4ee4a2c274f858c0ef3c33d9bd0d985",
),
(
"pypi__pkginfo",
"https://files.pythonhosted.org/packages/cd/00/49f59cdd2c6a52e6665fda4de671dac5614366dc827e050c55428241b929/pkginfo-1.8.2-py2.py3-none-any.whl",
"c24c487c6a7f72c66e816ab1796b96ac6c3d14d49338293d2141664330b55ffc",
),
(
"pypi__setuptools",
"https://files.pythonhosted.org/packages/7c/5b/3d92b9f0f7ca1645cba48c080b54fe7d8b1033a4e5720091d1631c4266db/setuptools-60.10.0-py3-none-any.whl",
Expand Down