From 3d8ec1fe6c178cf05c35f2116630ef2491070b33 Mon Sep 17 00:00:00 2001 From: Benjy Weinberger Date: Sat, 15 Feb 2020 11:21:35 -0500 Subject: [PATCH] Revendor pip (#890) New version is still a patched 20.0.dev0, with a new patch to fix a performance issue. Fixes #887. --- pex/vendor/README.md | 25 ++++--- pex/vendor/__init__.py | 2 +- .../pip/pip/_internal/index/collector.py | 74 ++++++++++++++++++- 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/pex/vendor/README.md b/pex/vendor/README.md index 987678ff1..e2e998e68 100644 --- a/pex/vendor/README.md +++ b/pex/vendor/README.md @@ -14,18 +14,25 @@ To update versions of vendored code or add new vendored code: 1. Modify `pex.vendor.iter_vendor_specs` with updated versions or new distributions. Today that function looks like: ```python - def iter_vendor_specs(include_wheel=True): + def iter_vendor_specs(): """Iterate specifications for code vendored by pex. - :param bool include_wheel: If ``True`` include the vendored wheel spec. - :return: An iterator over specs of all vendored code optionally including ``wheel``. + :return: An iterator over specs of all vendored code. :rtype: :class:`collection.Iterator` of :class:`VendorSpec` """ - yield VendorSpec.create('setuptools==40.6.2') - if include_wheel: - # We're currently stuck here due to removal of an API we depend on. - # See: https://github.com/pantsbuild/pex/issues/603 - yield VendorSpec.create('wheel==0.31.1') + # We use this via pex.third_party at runtime to check for compatible wheel tags. + yield VendorSpec.pinned('packaging', '19.2') + + # We shell out to pip at buildtime to resolve and install dependencies. + # N.B.: This is pip 20.0.dev0 with a patch to support foreign download targets more fully. + yield VendorSpec.vcs('git+https://github.com/pantsbuild/pip@5eb9470c0c59#egg=pip', rewrite=False) + + # We expose this to pip at buildtime for legacy builds, but we also use pkg_resources via + # pex.third_party at runtime in various ways. + yield VendorSpec.pinned('setuptools', '42.0.2') + + # We expose this to pip at buildtime for legacy builds. + yield VendorSpec.pinned('wheel', '0.33.6', rewrite=False) ``` Simply edit an existing `VendorSpec` or `yield` a new one. 2. Run `tox -e vendor`. @@ -37,4 +44,4 @@ To update versions of vendored code or add new vendored code: 3. Run tests and, once green, check in the newly vendored code. After this, and newly vendored distribution can be imported by pex code using the `pex.third_party` -import prefix. \ No newline at end of file +import prefix. diff --git a/pex/vendor/__init__.py b/pex/vendor/__init__.py index e120045e0..9538b9d8e 100644 --- a/pex/vendor/__init__.py +++ b/pex/vendor/__init__.py @@ -99,7 +99,7 @@ def iter_vendor_specs(): # We shell out to pip at buildtime to resolve and install dependencies. # N.B.: This is pip 20.0.dev0 with a patch to support foreign download targets more fully. - yield VendorSpec.vcs('git+https://github.com/pantsbuild/pip@e96681a0d5cf#egg=pip', rewrite=False) + yield VendorSpec.vcs('git+https://github.com/pantsbuild/pip@5eb9470c0c59#egg=pip', rewrite=False) # We expose this to pip at buildtime for legacy builds, but we also use pkg_resources via # pex.third_party at runtime in various ways. diff --git a/pex/vendor/_vendored/pip/pip/_internal/index/collector.py b/pex/vendor/_vendored/pip/pip/_internal/index/collector.py index 833079317..80ce2d98e 100644 --- a/pex/vendor/_vendored/pip/pip/_internal/index/collector.py +++ b/pex/vendor/_vendored/pip/pip/_internal/index/collector.py @@ -24,8 +24,8 @@ if MYPY_CHECK_RUNNING: from typing import ( - Callable, Iterable, List, MutableMapping, Optional, Sequence, Tuple, - Union, + Any, Callable, Dict, Iterable, List, MutableMapping, Optional, + Sequence, Tuple, Union, ) import xml.etree.ElementTree @@ -41,6 +41,32 @@ logger = logging.getLogger(__name__) +def lru_cache( + *args, # type: Any + **kwargs # type: Any +): + # type: (...) -> Any + cache = {} # type: Dict[Any, Any] + + def wrapper(fn): + # type: (Any) -> Any + + def wrapped( + *args, # type: Any + **kwargs # type: Any + ): + # type: (...) -> Any + cache_key = tuple(args) + tuple(kwargs.items()) + value = cache.get(cache_key, None) + if value is not None: + return value + value = fn(*args, **kwargs) + cache[cache_key] = value + return value + return wrapped + return wrapper + + def _match_vcs_scheme(url): # type: (str) -> Optional[str] """Look for VCS schemes in the URL. @@ -244,6 +270,38 @@ def _create_link_from_element( return link +class CacheablePageContent(object): + def __init__(self, page): + # type: (HTMLPage) -> None + self.page = page + + def __eq__(self, other): + # type: (object) -> bool + return (isinstance(other, type(self)) and + self.page.content == other.page.content and + self.page.encoding == other.page.encoding) + + def __hash__(self): + # type: () -> int + return hash((self.page.content, self.page.encoding)) + + +def with_cached_html_pages(fn): + # type: (Any) -> Any + + @lru_cache(maxsize=None) + def wrapper(cacheable_page): + # type: (CacheablePageContent) -> List[Any] + return list(fn(cacheable_page.page)) + + def wrapper_wrapper(page): + # type: (HTMLPage) -> List[Any] + return wrapper(CacheablePageContent(page)) + + return wrapper_wrapper + + +@with_cached_html_pages def parse_links(page): # type: (HTMLPage) -> Iterable[Link] """ @@ -308,6 +366,18 @@ def _make_html_page(response): return HTMLPage(response.content, encoding=encoding, url=response.url) +def with_cached_link_fetch(fn): + # type: (Any) -> Any + + @lru_cache(maxsize=None) + def wrapper(link, session=None): + # type: (Link, Optional[PipSession]) -> Optional[HTMLPage] + return fn(link, session=session) + + return wrapper + + +@with_cached_link_fetch def _get_html_page(link, session=None): # type: (Link, Optional[PipSession]) -> Optional[HTMLPage] if session is None: