From 2cb0ca5b35e881a3703552ea9a897da39a2ac2fb Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Tue, 1 Oct 2024 18:36:07 +0200 Subject: [PATCH 1/5] Add initial support for plugins --- src/pip/_internal/models/plugin.py | 71 ++++++++++++++++++ src/pip/_internal/operations/install/wheel.py | 8 +++ src/pip/_internal/operations/prepare.py | 16 +++++ src/pip/_internal/utils/plugins.py | 72 +++++++++++++++++++ src/pip/_internal/utils/unpacking.py | 7 ++ 5 files changed, 174 insertions(+) create mode 100644 src/pip/_internal/models/plugin.py create mode 100644 src/pip/_internal/utils/plugins.py diff --git a/src/pip/_internal/models/plugin.py b/src/pip/_internal/models/plugin.py new file mode 100644 index 00000000000..1d33afa8b0e --- /dev/null +++ b/src/pip/_internal/models/plugin.py @@ -0,0 +1,71 @@ +import abc +import logging +from pathlib import Path +from types import ModuleType +from typing import Optional + +logger = logging.getLogger(__name__) + +PLUGIN_TYPE_DIST_INSPECTOR = "dist-inspector" +SUPPORTED_PLUGIN_TYPES = [PLUGIN_TYPE_DIST_INSPECTOR] + + +class Plugin(metaclass=abc.ABCMeta): + @abc.abstractmethod + def plugin_type(self) -> str: + raise NotImplementedError + + @property + @abc.abstractmethod + def name(self) -> str: + raise NotImplementedError + + +class DistInspectorPlugin(Plugin): + def __init__(self, name: str, loaded_module: ModuleType): + assert loaded_module.plugin_type() == PLUGIN_TYPE_DIST_INSPECTOR + if not hasattr(loaded_module, "pre_download") or not hasattr( + loaded_module, "pre_extract" + ): + raise ValueError( + f'Plugin "{name}" of type {PLUGIN_TYPE_DIST_INSPECTOR} is' + "missing pre_download and/or pre_extract definitions" + ) + + self._name = name + self._module = loaded_module + + def plugin_type(self) -> str: + return self._module.plugin_type() + + @property + def name(self) -> str: + return self._name + + def pre_download(self, url: str, filename: str, digest: str) -> None: + # contract: `pre_download` raises `ValueError` to terminate + # the operation that intends to download `filename` from `url` + # with hash `digest` + self._module.pre_download(url=url, filename=filename, digest=digest) + + def pre_extract(self, dist: Path) -> None: + # contract: `pre_extract` raises `ValueError` to terminate + # the operation that intends to unarchive `dist` + self._module.pre_extract(dist) + + +def plugin_from_module(name: str, loaded_module: ModuleType) -> Optional[Plugin]: + if not hasattr(loaded_module, "plugin_type"): + logger.warning("Ignoring plugin %s due to missing plugin_type definition", name) + plugin_type = loaded_module.plugin_type() + if plugin_type not in SUPPORTED_PLUGIN_TYPES: + logger.warning( + "Ignoring plugin %s due to unknown plugin type: %s", name, plugin_type + ) + + if plugin_type == PLUGIN_TYPE_DIST_INSPECTOR: + try: + return DistInspectorPlugin(name, loaded_module) + except ValueError as e: + logger.warning("Ignoring plugin %s due to error: %s", name, e) + return None diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index aef42aa9eef..821cb3fff29 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -15,6 +15,7 @@ from base64 import urlsafe_b64encode from email.message import Message from itertools import chain, filterfalse, starmap +from pathlib import Path from typing import ( IO, TYPE_CHECKING, @@ -52,6 +53,7 @@ from pip._internal.models.scheme import SCHEME_KEYS, Scheme from pip._internal.utils.filesystem import adjacent_tmp_file, replace from pip._internal.utils.misc import StreamWrapper, ensure_dir, hash_file, partition +from pip._internal.utils.plugins import plugin_pre_extract_hook from pip._internal.utils.unpacking import ( current_umask, is_within_directory, @@ -727,6 +729,12 @@ def install_wheel( direct_url: Optional[DirectUrl] = None, requested: bool = False, ) -> None: + try: + plugin_pre_extract_hook(Path(wheel_path)) + except ValueError as e: + raise InstallationError( + f"Could not unpack file {wheel_path} due to plugin:\n{e}" + ) with ZipFile(wheel_path, allowZip64=True) as z: with req_error_context(req_description): _install_wheel( diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index e6aa3447200..bdbed223a57 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -50,6 +50,7 @@ hide_url, redact_auth_from_requirement, ) +from pip._internal.utils.plugins import plugin_pre_download_hook from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.unpacking import unpack_file from pip._internal.vcs import vcs @@ -461,6 +462,14 @@ def _complete_partial_requirements( for req in partially_downloaded_reqs: assert req.link links_to_fully_download[req.link] = req + try: + plugin_pre_download_hook( + url=req.link.url, filename=req.link.filename, digest=req.link.hash + ) + except ValueError as e: + raise InstallationError( + f"Could not install requirement {req} due to plugin:\n{e}" + ) batch_download = self._batch_download( links_to_fully_download.keys(), @@ -595,6 +604,9 @@ def _prepare_linked_requirement( local_file = None elif link.url not in self._downloaded: try: + plugin_pre_download_hook( + url=req.link.url, filename=req.link.filename, digest=req.link.hash + ) local_file = unpack_url( link, req.source_dir, @@ -608,6 +620,10 @@ def _prepare_linked_requirement( f"Could not install requirement {req} because of HTTP " f"error {exc} for URL {link}" ) + except ValueError as e: + raise InstallationError( + f"Could not install requirement {req} due to plugin:\n{e}" + ) else: file_path = self._downloaded[link.url] if hashes: diff --git a/src/pip/_internal/utils/plugins.py b/src/pip/_internal/utils/plugins.py new file mode 100644 index 00000000000..d4acf1e5c69 --- /dev/null +++ b/src/pip/_internal/utils/plugins.py @@ -0,0 +1,72 @@ +import contextlib +import logging +from pathlib import Path +from typing import Iterator, List + +from pip._vendor.pygments.plugin import iter_entry_points + +from pip._internal.models.plugin import DistInspectorPlugin, Plugin, plugin_from_module + +logger = logging.getLogger(__name__) + +_loaded_plugins: List[Plugin] = [] +for entrypoint in iter_entry_points(group_name="pip.plugins"): + try: + module = entrypoint.load() + except ModuleNotFoundError: + logger.warning("Tried to load plugin %s but failed", entrypoint.name) + continue + plugin = plugin_from_module(entrypoint.name, module) + if plugin is not None: + _loaded_plugins.append(plugin) + + +@contextlib.contextmanager +def _only_raise_value_error(plugin_name: str) -> Iterator[None]: + try: + yield + except ValueError as e: + raise ValueError(f"Plugin {plugin_name}: {e}") from e + except Exception as e: + logger.warning( + "Plugin %s raised an unexpected exception type: %s", + plugin_name, + {e.__class__.__name__}, + ) + raise ValueError(f"Plugin {plugin_name}: {e}") from e + + +def plugin_pre_download_hook(url: str, filename: str, digest: str) -> None: + """Call the pre-download hook of all loaded plugins + + This function should be called right before a distribution is downloaded. + It will go through all the loaded plugins and call their `pre_download(url)` + function. + Only ValueError will be raised. If the plugin (incorrectly) raises another + exception type, this function will wrap it as a ValueError and log + a warning. + """ + + for p in _loaded_plugins: + if not isinstance(p, DistInspectorPlugin): + continue + with _only_raise_value_error(p.name): + p.pre_download(url=url, filename=filename, digest=digest) + + +def plugin_pre_extract_hook(dist: Path) -> None: + """Call the pre-extract hook of all loaded plugins + + This function should be called right before a distribution is extracted. + It will go through all the loaded plugins and call their `pre_extract(dist)` + function. + Only ValueError will be raised. If the plugin (incorrectly) raises another + exception type, this function will wrap it as a ValueError and log + a warning. + """ + + for p in _loaded_plugins: + if not isinstance(p, DistInspectorPlugin): + continue + with _only_raise_value_error(p.name): + p.pre_extract(dist) diff --git a/src/pip/_internal/utils/unpacking.py b/src/pip/_internal/utils/unpacking.py index 875e30e13ab..5ef409eef7b 100644 --- a/src/pip/_internal/utils/unpacking.py +++ b/src/pip/_internal/utils/unpacking.py @@ -8,6 +8,7 @@ import sys import tarfile import zipfile +from pathlib import Path from typing import Iterable, List, Optional from zipfile import ZipInfo @@ -19,6 +20,7 @@ ZIP_EXTENSIONS, ) from pip._internal.utils.misc import ensure_dir +from pip._internal.utils.plugins import plugin_pre_extract_hook logger = logging.getLogger(__name__) @@ -312,6 +314,11 @@ def unpack_file( content_type: Optional[str] = None, ) -> None: filename = os.path.realpath(filename) + try: + plugin_pre_extract_hook(Path(filename)) + except ValueError as e: + raise InstallationError(f"Could not unpack file {filename} due to plugin:\n{e}") + if ( content_type == "application/zip" or filename.lower().endswith(ZIP_EXTENSIONS) From c6e59353ec5faab6b37d4fd3849e0316c604fcc9 Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Tue, 1 Oct 2024 21:39:08 +0200 Subject: [PATCH 2/5] Add logic to iterate over entrypoints --- src/pip/_internal/utils/plugins.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/pip/_internal/utils/plugins.py b/src/pip/_internal/utils/plugins.py index d4acf1e5c69..697910324d9 100644 --- a/src/pip/_internal/utils/plugins.py +++ b/src/pip/_internal/utils/plugins.py @@ -1,15 +1,29 @@ import contextlib import logging +from importlib.metadata import EntryPoints, entry_points from pathlib import Path from typing import Iterator, List -from pip._vendor.pygments.plugin import iter_entry_points - from pip._internal.models.plugin import DistInspectorPlugin, Plugin, plugin_from_module logger = logging.getLogger(__name__) _loaded_plugins: List[Plugin] = [] + + +def iter_entry_points(group_name: str) -> EntryPoints: + groups = entry_points() + if hasattr(groups, "select"): + # New interface in Python 3.10 and newer versions of the + # importlib_metadata backport. + return groups.select(group=group_name) + else: + assert hasattr(groups, "get") + # Older interface, deprecated in Python 3.10 and recent + # importlib_metadata, but we need it in Python 3.8 and 3.9. + return groups.get(group_name, []) + + for entrypoint in iter_entry_points(group_name="pip.plugins"): try: module = entrypoint.load() From d5b3da5378996352a16613a770d3c2fe8b327ae6 Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Tue, 1 Oct 2024 21:41:30 +0200 Subject: [PATCH 3/5] Load plugins during init rather than on import --- src/pip/_internal/cli/main.py | 3 +++ src/pip/_internal/utils/plugins.py | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/pip/_internal/cli/main.py b/src/pip/_internal/cli/main.py index 563ac79c984..00637ef182c 100644 --- a/src/pip/_internal/cli/main.py +++ b/src/pip/_internal/cli/main.py @@ -13,6 +13,7 @@ from pip._internal.commands import create_command from pip._internal.exceptions import PipError from pip._internal.utils import deprecation +from pip._internal.utils.plugins import load_plugins logger = logging.getLogger(__name__) @@ -77,4 +78,6 @@ def main(args: Optional[List[str]] = None) -> int: logger.debug("Ignoring error %s when setting locale", e) command = create_command(cmd_name, isolated=("--isolated" in cmd_args)) + load_plugins() + return command.main(cmd_args) diff --git a/src/pip/_internal/utils/plugins.py b/src/pip/_internal/utils/plugins.py index 697910324d9..cb0408af935 100644 --- a/src/pip/_internal/utils/plugins.py +++ b/src/pip/_internal/utils/plugins.py @@ -7,7 +7,6 @@ from pip._internal.models.plugin import DistInspectorPlugin, Plugin, plugin_from_module logger = logging.getLogger(__name__) - _loaded_plugins: List[Plugin] = [] @@ -24,15 +23,16 @@ def iter_entry_points(group_name: str) -> EntryPoints: return groups.get(group_name, []) -for entrypoint in iter_entry_points(group_name="pip.plugins"): - try: - module = entrypoint.load() - except ModuleNotFoundError: - logger.warning("Tried to load plugin %s but failed", entrypoint.name) - continue - plugin = plugin_from_module(entrypoint.name, module) - if plugin is not None: - _loaded_plugins.append(plugin) +def load_plugins() -> None: + for entrypoint in iter_entry_points(group_name="pip.plugins"): + try: + module = entrypoint.load() + except ModuleNotFoundError: + logger.warning("Tried to load plugin %s but failed", entrypoint.name) + continue + plugin = plugin_from_module(entrypoint.name, module) + if plugin is not None: + _loaded_plugins.append(plugin) @contextlib.contextmanager From 9279db41ddeba474938e04d95038a43135a88b3d Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Tue, 29 Oct 2024 23:13:26 +0100 Subject: [PATCH 4/5] Fix entrypoints iteration in Python 3.9 Signed-off-by: Facundo Tuesca --- src/pip/_internal/utils/plugins.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pip/_internal/utils/plugins.py b/src/pip/_internal/utils/plugins.py index cb0408af935..132b049ed12 100644 --- a/src/pip/_internal/utils/plugins.py +++ b/src/pip/_internal/utils/plugins.py @@ -1,6 +1,6 @@ import contextlib import logging -from importlib.metadata import EntryPoints, entry_points +from importlib.metadata import EntryPoint, entry_points from pathlib import Path from typing import Iterator, List @@ -10,12 +10,14 @@ _loaded_plugins: List[Plugin] = [] -def iter_entry_points(group_name: str) -> EntryPoints: +def iter_entry_points(group_name: str) -> List[EntryPoint]: + # Only Python >= 3.10 supports the `EntryPoints` class, so we return + # a list of `EntryPoint` instead. groups = entry_points() if hasattr(groups, "select"): # New interface in Python 3.10 and newer versions of the # importlib_metadata backport. - return groups.select(group=group_name) + return list(groups.select(group=group_name)) else: assert hasattr(groups, "get") # Older interface, deprecated in Python 3.10 and recent From c8233a171377f277c1142d162a24812d72024301 Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Fri, 8 Nov 2024 18:05:24 +0100 Subject: [PATCH 5/5] Remove plugin types, register individual hooks instead Signed-off-by: Facundo Tuesca --- src/pip/_internal/models/plugin.py | 70 +++++++++++++++++------------- src/pip/_internal/utils/plugins.py | 8 +--- 2 files changed, 43 insertions(+), 35 deletions(-) diff --git a/src/pip/_internal/models/plugin.py b/src/pip/_internal/models/plugin.py index 1d33afa8b0e..9c87827c43a 100644 --- a/src/pip/_internal/models/plugin.py +++ b/src/pip/_internal/models/plugin.py @@ -2,17 +2,18 @@ import logging from pathlib import Path from types import ModuleType -from typing import Optional +from typing import List, Optional logger = logging.getLogger(__name__) -PLUGIN_TYPE_DIST_INSPECTOR = "dist-inspector" -SUPPORTED_PLUGIN_TYPES = [PLUGIN_TYPE_DIST_INSPECTOR] +PLUGIN_HOOK_PRE_DOWNLOAD = "pre_download" +PLUGIN_HOOK_PRE_EXTRACT = "pre_extract" +SUPPORTED_PLUGIN_HOOKS = [PLUGIN_HOOK_PRE_DOWNLOAD, PLUGIN_HOOK_PRE_EXTRACT] class Plugin(metaclass=abc.ABCMeta): @abc.abstractmethod - def plugin_type(self) -> str: + def provided_hooks(self) -> List[str]: raise NotImplementedError @property @@ -21,22 +22,40 @@ def name(self) -> str: raise NotImplementedError -class DistInspectorPlugin(Plugin): +class LoadedPlugin(Plugin): def __init__(self, name: str, loaded_module: ModuleType): - assert loaded_module.plugin_type() == PLUGIN_TYPE_DIST_INSPECTOR - if not hasattr(loaded_module, "pre_download") or not hasattr( - loaded_module, "pre_extract" - ): + self._pre_download = None + self._pre_extract = None + if not hasattr(loaded_module, "provided_hooks"): raise ValueError( - f'Plugin "{name}" of type {PLUGIN_TYPE_DIST_INSPECTOR} is' - "missing pre_download and/or pre_extract definitions" + f"Ignoring plugin {name} due to missing provided_hooks definition" ) + for hook in loaded_module.provided_hooks(): + if hook == PLUGIN_HOOK_PRE_DOWNLOAD: + if not hasattr(loaded_module, "pre_download"): + raise ValueError( + f'Plugin "{name}" wants to register a pre-download hook but ' + "does not define a pre_download method" + ) + self._pre_download = loaded_module.pre_download + elif hook == PLUGIN_HOOK_PRE_EXTRACT: + if not hasattr(loaded_module, "pre_extract"): + raise ValueError( + f'Plugin "{name}" wants to register a pre-extract hook but ' + "does not define a pre_extract method" + ) + self._pre_extract = loaded_module.pre_extract + else: + raise ValueError( + f'Plugin "{name}" wants to register a hook of unknown type:' + '"{hook}"' + ) self._name = name self._module = loaded_module - def plugin_type(self) -> str: - return self._module.plugin_type() + def provided_hooks(self) -> List[str]: + return self._module.provided_hooks() @property def name(self) -> str: @@ -46,26 +65,19 @@ def pre_download(self, url: str, filename: str, digest: str) -> None: # contract: `pre_download` raises `ValueError` to terminate # the operation that intends to download `filename` from `url` # with hash `digest` - self._module.pre_download(url=url, filename=filename, digest=digest) + if self._pre_download is not None: + self._pre_download(url=url, filename=filename, digest=digest) def pre_extract(self, dist: Path) -> None: # contract: `pre_extract` raises `ValueError` to terminate # the operation that intends to unarchive `dist` - self._module.pre_extract(dist) + if self._pre_extract is not None: + self._module.pre_extract(dist) -def plugin_from_module(name: str, loaded_module: ModuleType) -> Optional[Plugin]: - if not hasattr(loaded_module, "plugin_type"): - logger.warning("Ignoring plugin %s due to missing plugin_type definition", name) - plugin_type = loaded_module.plugin_type() - if plugin_type not in SUPPORTED_PLUGIN_TYPES: - logger.warning( - "Ignoring plugin %s due to unknown plugin type: %s", name, plugin_type - ) - - if plugin_type == PLUGIN_TYPE_DIST_INSPECTOR: - try: - return DistInspectorPlugin(name, loaded_module) - except ValueError as e: - logger.warning("Ignoring plugin %s due to error: %s", name, e) +def plugin_from_module(name: str, loaded_module: ModuleType) -> Optional[LoadedPlugin]: + try: + return LoadedPlugin(name, loaded_module) + except ValueError as e: + logger.warning("Ignoring plugin %s due to error: %s", name, e) return None diff --git a/src/pip/_internal/utils/plugins.py b/src/pip/_internal/utils/plugins.py index 132b049ed12..b7209350c6d 100644 --- a/src/pip/_internal/utils/plugins.py +++ b/src/pip/_internal/utils/plugins.py @@ -4,10 +4,10 @@ from pathlib import Path from typing import Iterator, List -from pip._internal.models.plugin import DistInspectorPlugin, Plugin, plugin_from_module +from pip._internal.models.plugin import LoadedPlugin, plugin_from_module logger = logging.getLogger(__name__) -_loaded_plugins: List[Plugin] = [] +_loaded_plugins: List[LoadedPlugin] = [] def iter_entry_points(group_name: str) -> List[EntryPoint]: @@ -64,8 +64,6 @@ def plugin_pre_download_hook(url: str, filename: str, digest: str) -> None: """ for p in _loaded_plugins: - if not isinstance(p, DistInspectorPlugin): - continue with _only_raise_value_error(p.name): p.pre_download(url=url, filename=filename, digest=digest) @@ -82,7 +80,5 @@ def plugin_pre_extract_hook(dist: Path) -> None: """ for p in _loaded_plugins: - if not isinstance(p, DistInspectorPlugin): - continue with _only_raise_value_error(p.name): p.pre_extract(dist)