Use the resolve cache to skip installs. (#815)

Previously the cache was only used by pip for http queries. Now we also
use the cache to skill wheel installs in their isolated chroots.

Fixes #809
Fixes #811

pex/bin/pex.py

@@ -538,7 +538,7 @@ def walk_and_do(fn, src_dir):
   if options.indexes != [_PYPI] and options.indexes is not None:
     indexes = [str(index) for index in options.indexes]
 
-  with TRACER.timed('Resolving distributions ({})'.format(reqs)):
+  with TRACER.timed('Resolving distributions ({})'.format(reqs + options.requirement_files)):
     try:
       resolveds = resolve_multi(requirements=reqs,
                                 requirement_files=options.requirement_files,

pex/interpreter.py

@@ -330,7 +330,7 @@ def _create_isolated_cmd(cls, binary, args=None, pythonpath=None, env=None):
     rendered_command = ' '.join(cmd)
     if pythonpath:
       rendered_command = 'PYTHONPATH={} {}'.format(env['PYTHONPATH'], rendered_command)
-    TRACER.log('Executing: {}'.format(rendered_command))
+    TRACER.log('Executing: {}'.format(rendered_command), V=3)
 
     return cmd, env
 

pex/resolver.py

@@ -6,13 +6,13 @@
 import functools
 import json
 import os
-import shutil
 import subprocess
 from collections import defaultdict, namedtuple
 from textwrap import dedent
+from uuid import uuid4
 
 from pex import third_party
-from pex.common import safe_mkdir, safe_mkdtemp
+from pex.common import safe_mkdtemp
 from pex.interpreter import PythonInterpreter
 from pex.orderedset import OrderedSet
 from pex.pip import PipError, build_wheels, download_distributions, install_wheel
@@ -172,27 +172,29 @@ def resolve(requirements=None,
     return []
 
   workspace = safe_mkdtemp()
+  cache = cache or workspace
   resolved_dists = os.path.join(workspace, 'resolved')
   built_wheels = os.path.join(workspace, 'wheels')
-  installed_chroots = os.path.join(workspace, 'chroots')
+  installed_chroots = os.path.join(cache, 'chroots')
 
   # 1. Resolve
-  try:
-    download_distributions(target=resolved_dists,
-                           requirements=requirements,
-                           requirement_files=requirement_files,
-                           constraint_files=constraint_files,
-                           allow_prereleases=allow_prereleases,
-                           transitive=transitive,
-                           interpreter=interpreter,
-                           platform=parsed_platform(platform),
-                           indexes=indexes,
-                           find_links=find_links,
-                           cache=cache,
-                           build=build,
-                           use_wheel=use_wheel)
-  except PipError as e:
-    raise Unsatisfiable(str(e))
+  with TRACER.timed('Resolving and downloading distributions'):
+    try:
+      download_distributions(target=resolved_dists,
+                             requirements=requirements,
+                             requirement_files=requirement_files,
+                             constraint_files=constraint_files,
+                             allow_prereleases=allow_prereleases,
+                             transitive=transitive,
+                             interpreter=interpreter,
+                             platform=parsed_platform(platform),
+                             indexes=indexes,
+                             find_links=find_links,
+                             cache=cache,
+                             build=build,
+                             use_wheel=use_wheel)
+    except PipError as e:
+      raise Unsatisfiable(str(e))
 
   # 2. Build
   to_build = []
@@ -205,57 +207,64 @@ def resolve(requirements=None,
     for requirement_file in requirement_files:
       to_build.extend(local_projects_from_requirement_file(requirement_file))
 
-  to_copy = []
+  to_install = []
   if os.path.exists(resolved_dists):
     for distribution in os.listdir(resolved_dists):
       path = os.path.join(resolved_dists, distribution)
       if os.path.isfile(path) and path.endswith('.whl'):
-        to_copy.append(path)
+        to_install.append(path)
       else:
         to_build.append(path)
 
-  if not any((to_build, to_copy)):
+  if not any((to_build, to_install)):
     # Nothing to build or install.
     return []
 
-  safe_mkdir(built_wheels)
-
   if to_build:
-    try:
-      build_wheels(distributions=to_build,
-                   target=built_wheels,
-                   cache=cache,
-                   interpreter=interpreter)
-    except PipError as e:
-      raise Untranslateable('Failed to build at least one of {}:\n\t{}'.format(to_build, str(e)))
-
-  if to_copy:
-    for wheel in to_copy:
-      dest = os.path.join(built_wheels, os.path.basename(wheel))
-      TRACER.log('Copying downloaded wheel from {} to {}'.format(wheel, dest))
-      shutil.copy(wheel, dest)
+    with TRACER.timed('Building distributions:\n  {}'.format('\n  '.join(to_build))):
+      try:
+        build_wheels(distributions=to_build,
+                     target=built_wheels,
+                     cache=cache,
+                     interpreter=interpreter)
+      except PipError as e:
+        raise Untranslateable('Failed to build at least one of {}:\n  {}'
+                              .format(', '.join(to_build), str(e)))
+      to_install.extend(os.path.join(built_wheels, wheel) for wheel in os.listdir(built_wheels))
 
   # 3. Chroot
   resolved_distributions = []
 
-  for wheel_file in os.listdir(built_wheels):
-    chroot = os.path.join(installed_chroots, wheel_file)
-    try:
-      install_wheel(wheel=os.path.join(built_wheels, wheel_file),
-                    target=chroot,
-                    compile=compile,
-                    overwrite=True,
-                    cache=cache,
-                    interpreter=interpreter)
-    except PipError as e:
-      raise Untranslateable('Failed to install {}:\n\t{}'.format(wheel_file, str(e)))
-
-    environment = Environment(search_path=[chroot])
-    for dist_project_name in environment:
-      resolved_distributions.extend(environment[dist_project_name])
-
-  markers_by_req_key = _calculate_dependency_markers(resolved_distributions,
-                                                     interpreter=interpreter)
+  with TRACER.timed('Installing distributions:\n  {}'.format('\n  '.join(to_install))):
+    for wheel_file_path in to_install:
+      wheel_file = os.path.basename(wheel_file_path)
+      chroot = os.path.join(installed_chroots, wheel_file)
+      if os.path.exists(chroot):
+        TRACER.log('Using cached installation of {} at {}'.format(wheel_file, chroot))
+      else:
+        TRACER.log('Installing {} in {}'.format(wheel_file_path, chroot))
+        tmp_chroot = '{}.{}'.format(chroot, uuid4().hex)
+        try:
+          install_wheel(wheel=wheel_file_path,
+                        target=tmp_chroot,
+                        compile=compile,
+                        overwrite=True,
+                        cache=cache,
+                        interpreter=interpreter)
+        except PipError as e:
+          raise Untranslateable('Failed to install {}:\n\t{}'.format(wheel_file_path, e))
+        os.rename(tmp_chroot, chroot)
+
+      environment = Environment(search_path=[chroot])
+      for dist_project_name in environment:
+        resolved_distributions.extend(environment[dist_project_name])
+
+  with TRACER.timed('Determining environment markers of installed distributions:\n  {}'
+                    .format('\n  '.join(map(str, resolved_distributions)))):
+    markers_by_req_key = _calculate_dependency_markers(
+      resolved_distributions,
+      interpreter=interpreter
+    )
 
   def to_requirement(dist):
     req = dist.as_requirement()

tests/test_resolver.py

@@ -6,6 +6,7 @@
 import pytest
 
 from pex.common import safe_copy, safe_mkdtemp, temporary_dir
+from pex.compatibility import nested
 from pex.interpreter import PythonInterpreter
 from pex.resolver import resolve_multi
 from pex.testing import (
@@ -50,6 +51,29 @@ def test_simple_local_resolve():
     assert len(resolved_dists) == 1
 
 
+def test_resolve_cache():
+  project_wheel = build_wheel(name='project')
+
+  with nested(temporary_dir(), temporary_dir()) as (td, cache):
+    safe_copy(project_wheel, os.path.join(td, os.path.basename(project_wheel)))
+
+    # Without a cache, each resolve should be isolated, but otherwise identical.
+    resolved_dists1 = local_resolve_multi(['project'], find_links=[td])
+    resolved_dists2 = local_resolve_multi(['project'], find_links=[td])
+    assert resolved_dists1 != resolved_dists2
+    assert len(resolved_dists1) == 1
+    assert len(resolved_dists2) == 1
+    assert resolved_dists1[0].requirement == resolved_dists2[0].requirement
+    assert resolved_dists1[0].distribution.location != resolved_dists2[0].distribution.location
+
+    # With a cache, each resolve should be identical.
+    resolved_dists3 = local_resolve_multi(['project'], find_links=[td], cache=cache)
+    resolved_dists4 = local_resolve_multi(['project'], find_links=[td], cache=cache)
+    assert resolved_dists1 != resolved_dists3
+    assert resolved_dists2 != resolved_dists3
+    assert resolved_dists3 == resolved_dists4
+
+
 def test_diamond_local_resolve_cached():
   # This exercises the issue described here: https://github.com/pantsbuild/pex/issues/120
   project1_wheel = build_wheel(name='project1', install_reqs=['project2<1.0.0'])