avoid caching all transitively resolved requirements!

src/python/pants/backend/python/subsystems/ipex/ipex_launcher.py

@@ -16,7 +16,6 @@
 from pex.interpreter import PythonInterpreter
 from pex.pex_builder import PEXBuilder
 from pex.pex_info import PexInfo
-from pkg_resources import Requirement
 
 APP_CODE_PREFIX = "user_files/"
 

src/python/pants/python/pex_build_util.py

@@ -6,7 +6,7 @@
 import os
 from collections import defaultdict
 from pathlib import Path
-from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple
+from typing import Callable, Dict, List, Optional, Sequence, Set, Tuple, cast
 
 from pex.interpreter import PythonIdentity, PythonInterpreter
 from pex.pex_builder import PEXBuilder
@@ -214,18 +214,16 @@ def extract_single_dist_for_current_platform(self, reqs, dist_key) -> Distributi
         :raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the
                  given `dist_key`.
         """
-        distributions, _transitive_requirements = self.resolve_distributions(
-            reqs, platforms=["current"]
-        )
+        distributions, self.resolve_distributions(reqs, platforms=["current"])
         try:
-            matched_dist = assert_single_element(
+            matched_dist = cast(Distribution, assert_single_element(
                 list(
                     dist
-                    for _, dists in distributions.items()
+                    for dists in distributions.values()
                     for dist in dists
                     if dist.key == dist_key
                 )
-            )
+            ))
         except (StopIteration, ValueError) as e:
             raise self.SingleDistExtractionError(
                 f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}"
@@ -234,7 +232,7 @@ def extract_single_dist_for_current_platform(self, reqs, dist_key) -> Distributi
 
     def resolve_distributions(
         self, reqs: List[PythonRequirement], platforms: Optional[List[Platform]] = None,
-    ) -> Tuple[Dict[str, List[Distribution]], List[PythonRequirement]]:
+    ) -> Dict[str, List[Distribution]]:
         """Multi-platform dependency resolution.
 
         :param reqs: A list of :class:`PythonRequirement` to resolve.
@@ -254,10 +252,10 @@ def resolve_distributions(
                 find_links.add(req.repository)
 
         # Resolve the requirements into distributions.
-        distributions, transitive_requirements = self._resolve_multi(
+        distributions = self._resolve_multi(
             self._builder.interpreter, list(deduped_reqs), platforms, list(find_links),
         )
-        return (distributions, transitive_requirements)
+        return distributions
 
     def add_resolved_requirements(
         self,
@@ -279,9 +277,7 @@ def add_resolved_requirements(
                                                                         pex dependency to the output ipex file, and therefore needs to
                                                                         override the default behavior of this method.
         """
-        distributions, transitive_requirements = self.resolve_distributions(
-            reqs, platforms=platforms
-        )
+        distributions = self.resolve_distributions(reqs, platforms=platforms)
         locations: Set[str] = set()
         for platform, dists in distributions.items():
             for dist in dists:
@@ -297,19 +293,14 @@ def add_resolved_requirements(
                         )
                         self.add_distribution(dist)
                 locations.add(dist.location)
-        # In addition to the top-level requirements, we add all the requirements matching the resolved
-        # distributions to the resulting pex. If `generate_ipex=True` is set, we need to have all the
-        # transitive requirements resolved in order to hydrate the .ipex with an intransitive resolve.
-        if self._generate_ipex and not override_ipex_build_do_actually_add_distribution:
-            self.add_direct_requirements(transitive_requirements)
 
     def _resolve_multi(
         self,
         interpreter: PythonInterpreter,
         requirements: List[PythonRequirement],
         platforms: Optional[List[Platform]],
         find_links: Optional[List[str]],
-    ) -> Tuple[Dict[str, List[Distribution]], List[PythonRequirement]]:
+    ) -> Dict[str, List[Distribution]]:
         """Multi-platform dependency resolution for PEX files.
 
         Returns a tuple containing a list of distributions that must be included in order to satisfy a
@@ -337,9 +328,6 @@ def _resolve_multi(
         self._all_find_links.update(OrderedSet(find_links))
 
         distributions: Dict[str, List[Distribution]] = defaultdict(list)
-        transitive_requirements: List[PythonRequirement] = []
-
-        all_find_links = [*python_repos.repos, *find_links]
 
         for platform in platforms:
             requirements_cache_dir = os.path.join(
@@ -350,17 +338,15 @@ def _resolve_multi(
                 interpreter=interpreter,
                 platform=platform,
                 indexes=python_repos.indexes,
-                find_links=all_find_links,
+                find_links=find_links,
                 cache=requirements_cache_dir,
                 allow_prereleases=python_setup.resolver_allow_prereleases,
                 manylinux=python_setup.manylinux,
             )
             for resolved_dist in resolved_dists:
-                dist = resolved_dist.distribution
-                transitive_requirements.append(dist.as_requirement())
-                distributions[platform].append(dist)
+                distributions[platform].append(resolved_dist.distribution)
 
-        return (distributions, transitive_requirements)
+        return distributions
 
     def _create_source_dumper(self, tgt: Target) -> Callable[[str], None]:
         buildroot = get_buildroot()
@@ -447,6 +433,11 @@ def _shuffle_underlying_pex_builder(self) -> Tuple[PexInfo, Path]:
             self._builder.info, self._builder.interpreter.identity
         )
 
+        # Remove all the original top-level requirements in favor of the transitive == requirements.
+        self._builder.info = ipex_launcher.modify_pex_info(self._builder.info, requirements=[])
+        transitive_reqs = [dist.as_requirement() for dist in self._distributions.values()]
+        self.add_direct_requirements(transitive_reqs)
+
         orig_info = self._builder.info.copy()
 
         orig_chroot = self._builder.chroot()
@@ -457,6 +448,8 @@ def _shuffle_underlying_pex_builder(self) -> Tuple[PexInfo, Path]:
             self._builder.info, self._builder.interpreter.identity
         )
 
+        self._distributions = {}
+
         return (orig_info, Path(orig_chroot.path()))
 
     def _shuffle_original_build_info_into_ipex(self):
@@ -532,8 +525,7 @@ def _shuffle_original_build_info_into_ipex(self):
         # produced when the .ipex is first executed will read and resolve all those requirements from
         # the BOOTSTRAP-PEX-INFO.
         self.add_resolved_requirements(
-            [self._pex_requirement, self._setuptools_requirement,],
-            override_ipex_build_do_actually_add_distribution=True,
+            [self._pex_requirement], override_ipex_build_do_actually_add_distribution=True
         )
 
     def freeze(self) -> None: