Revendor pip (#890)

New version is still a patched 20.0.dev0, with a new patch
to fix a performance issue.

Fixes #887.

pex/vendor/README.md

@@ -14,18 +14,25 @@ To update versions of vendored code or add new vendored code:
 1. Modify `pex.vendor.iter_vendor_specs` with updated versions or new distributions.
    Today that function looks like:
    ```python
-   def iter_vendor_specs(include_wheel=True):
+   def iter_vendor_specs():
      """Iterate specifications for code vendored by pex.
 
-     :param bool include_wheel: If ``True`` include the vendored wheel spec.
-     :return: An iterator over specs of all vendored code optionally including ``wheel``.
+     :return: An iterator over specs of all vendored code.
      :rtype: :class:`collection.Iterator` of :class:`VendorSpec`
      """
-     yield VendorSpec.create('setuptools==40.6.2')
-     if include_wheel:
-       # We're currently stuck here due to removal of an API we depend on.
-       # See: https://github.com/pantsbuild/pex/issues/603
-       yield VendorSpec.create('wheel==0.31.1')
+     # We use this via pex.third_party at runtime to check for compatible wheel tags.
+     yield VendorSpec.pinned('packaging', '19.2')
+
+     # We shell out to pip at buildtime to resolve and install dependencies.
+     # N.B.: This is pip 20.0.dev0 with a patch to support foreign download targets more fully.
+     yield VendorSpec.vcs('git+https://github.com/pantsbuild/pip@5eb9470c0c59#egg=pip', rewrite=False)
+
+     # We expose this to pip at buildtime for legacy builds, but we also use pkg_resources via
+     # pex.third_party at runtime in various ways.
+     yield VendorSpec.pinned('setuptools', '42.0.2')
+
+     # We expose this to pip at buildtime for legacy builds.
+     yield VendorSpec.pinned('wheel', '0.33.6', rewrite=False)
    ```
    Simply edit an existing `VendorSpec` or `yield` a new one.
 2. Run `tox -e vendor`.
@@ -37,4 +44,4 @@ To update versions of vendored code or add new vendored code:
 3. Run tests and, once green, check in the newly vendored code.
 
 After this, and newly vendored distribution can be imported by pex code using the `pex.third_party`
-import prefix.
+import prefix.

pex/vendor/__init__.py

@@ -99,7 +99,7 @@ def iter_vendor_specs():
 
   # We shell out to pip at buildtime to resolve and install dependencies.
   # N.B.: This is pip 20.0.dev0 with a patch to support foreign download targets more fully.
-  yield VendorSpec.vcs('git+https://github.com/pantsbuild/pip@e96681a0d5cf#egg=pip', rewrite=False)
+  yield VendorSpec.vcs('git+https://github.com/pantsbuild/pip@5eb9470c0c59#egg=pip', rewrite=False)
 
   # We expose this to pip at buildtime for legacy builds, but we also use pkg_resources via
   # pex.third_party at runtime in various ways.

pex/vendor/_vendored/pip/pip/_internal/index/collector.py

@@ -24,8 +24,8 @@
 
 if MYPY_CHECK_RUNNING:
     from typing import (
-        Callable, Iterable, List, MutableMapping, Optional, Sequence, Tuple,
-        Union,
+        Any, Callable, Dict, Iterable, List, MutableMapping, Optional,
+        Sequence, Tuple, Union,
     )
     import xml.etree.ElementTree
 
@@ -41,6 +41,32 @@
 logger = logging.getLogger(__name__)
 
 
+def lru_cache(
+        *args,                  # type: Any
+        **kwargs                # type: Any
+):
+    # type: (...) -> Any
+    cache = {}                  # type: Dict[Any, Any]
+
+    def wrapper(fn):
+        # type: (Any) -> Any
+
+        def wrapped(
+                *args,          # type: Any
+                **kwargs        # type: Any
+        ):
+            # type: (...) -> Any
+            cache_key = tuple(args) + tuple(kwargs.items())
+            value = cache.get(cache_key, None)
+            if value is not None:
+                return value
+            value = fn(*args, **kwargs)
+            cache[cache_key] = value
+            return value
+        return wrapped
+    return wrapper
+
+
 def _match_vcs_scheme(url):
     # type: (str) -> Optional[str]
     """Look for VCS schemes in the URL.
@@ -244,6 +270,38 @@ def _create_link_from_element(
     return link
 
 
+class CacheablePageContent(object):
+    def __init__(self, page):
+        # type: (HTMLPage) -> None
+        self.page = page
+
+    def __eq__(self, other):
+        # type: (object) -> bool
+        return (isinstance(other, type(self)) and
+                self.page.content == other.page.content and
+                self.page.encoding == other.page.encoding)
+
+    def __hash__(self):
+        # type: () -> int
+        return hash((self.page.content, self.page.encoding))
+
+
+def with_cached_html_pages(fn):
+    # type: (Any) -> Any
+
+    @lru_cache(maxsize=None)
+    def wrapper(cacheable_page):
+        # type: (CacheablePageContent) -> List[Any]
+        return list(fn(cacheable_page.page))
+
+    def wrapper_wrapper(page):
+        # type: (HTMLPage) -> List[Any]
+        return wrapper(CacheablePageContent(page))
+
+    return wrapper_wrapper
+
+
+@with_cached_html_pages
 def parse_links(page):
     # type: (HTMLPage) -> Iterable[Link]
     """
@@ -308,6 +366,18 @@ def _make_html_page(response):
     return HTMLPage(response.content, encoding=encoding, url=response.url)
 
 
+def with_cached_link_fetch(fn):
+    # type: (Any) -> Any
+
+    @lru_cache(maxsize=None)
+    def wrapper(link, session=None):
+        # type: (Link, Optional[PipSession]) -> Optional[HTMLPage]
+        return fn(link, session=session)
+
+    return wrapper
+
+
+@with_cached_link_fetch
 def _get_html_page(link, session=None):
     # type: (Link, Optional[PipSession]) -> Optional[HTMLPage]
     if session is None: