Further iteration on python distribution task, now functioning for py…

…thon binary create

src/python/pants/backend/python/register.py

@@ -66,11 +66,11 @@ def register_goals():
   task(name='interpreter', action=SelectInterpreter).install('pyprep')
   task(name='requirements', action=ResolveRequirements).install('pyprep')
   task(name='sources', action=GatherSources).install('pyprep')
+  task(name='python-dists', action=PythonCreateDistributions).install('pyprep')
   task(name='py', action=PythonRun).install('run')
   task(name='pytest-prep', action=PytestPrep).install('test')
   task(name='pytest', action=PytestRun).install('test')
   task(name='py', action=PythonRepl).install('repl')
-  task(name='python-dists', action=PythonCreateDistributions).install()
   task(name='setup-py', action=SetupPy).install()
   task(name='py', action=PythonBinaryCreate).install('binary')
   task(name='isort', action=IsortPythonTask).install('fmt')

src/python/pants/backend/python/tasks/pex_build_util.py

@@ -132,72 +132,46 @@ def dump_requirements(builder, interpreter, req_libs, log, platforms=None):
       locations.add(dist.location)
 
 
-def build_python_distribution_from_target(target, workdir):
-  # Create a temporary working directory, pydistworkdir
-  pydist_workdir = os.path.join(workdir, 'pydistworkdir')
-  safe_mkdir(pydist_workdir)
-  pex_name = "%s.pex" % target.name
-  path_to_target = os.path.dirname(target.address.rel_path)
-
-  # disable cache to ensure reproducible runs
-  args = ['--disable-cache', path_to_target, '-o', os.path.join(pydist_workdir, pex_name)]
-  try:
-    pex_main.main(args=args)
-  except SystemExit as e:
-    raise TaskError(e)
-  except Exception as e:
-    raise TaskError(e)
-
-  # clean up egg-info created by setup.py
-  for path in glob.glob(os.path.join(path_to_target, '*.egg-info')):
-    shutil.rmtree(path)
-
-  # unzip into a chroot within the python dist workdir
-  zip_ref = zipfile.ZipFile(os.path.join(pydist_workdir, pex_name), 'r')
-  fingerprint = target.payload.fingerprint()
-  safe_mkdir(os.path.join(pydist_workdir, fingerprint))
-  zip_ref.extractall(os.path.join(pydist_workdir, fingerprint))
-  zip_ref.close()
-
-  # read the contents from .deps of the chroot to obtain the whl location
-  chroot_deps_contents = os.listdir(os.path.join(pydist_workdir, fingerprint, '.deps'))
-  if chroot_deps_contents:
-    whl_dist = chroot_deps_contents[0]  # TODO: find better way to grab .whl from chroot 
-    whl_location = os.path.join(pydist_workdir, fingerprint, '.deps', whl_dist)
-    return whl_location
-  else:
-    raise TaskError('Failed to package python distribution for target: %s', target.name)
-
+def prepare_dist_workdir(dist_tgt, workdir, log):
+  """Prepare Python distribution directory for SetupPyRunner by copying the 
+  target sources into a working directory located in .pants.d. 
 
-def build_python_distribution(dist_tgt, interpreter, workdir, log):
-  """Dump python distribution targets into a given builder
+  :param dist_target: The :class:`PythonDistribution` to prepare a directory for. 
+  :param workdir: The working directory for this task.
+  :param log: Use this logger.
 
   """
-  # make directory based on fingerprint in workdir
+  # Make directory for local built distributions.
   local_dists_workdir = os.path.join(workdir, 'local_dists')
   if not os.path.exists(local_dists_workdir):
     safe_mkdir(local_dists_workdir)
 
+  # Fingerprint distribution target and create subdirectory for that target.
   fingerprint = dist_tgt.payload.fingerprint()
   dist_target_dir = os.path.join(local_dists_workdir, fingerprint)
   if not os.path.exists(dist_target_dir):
+    log.debug('Creating working directory for target %s with fingerprint %s', 
+      dist_tgt.name, fingerprint)
     safe_mkdir(dist_target_dir)
 
-  tmp_dir_for_dist = os.path.join(workdir, 'tmp')
+
+  # Create subdirectory based on target's name.
+  tmp_dir_for_dist = os.path.join(dist_target_dir, dist_tgt.name)
   if not tmp_dir_for_dist:
     safe_mkdir(tmp_dir_for_dist)
 
-  # copy sources and setup.py to this temp dir
+  # Copy sources and setup.py over for packaging.
   sources_rel_to_target_base = dist_tgt.sources_relative_to_target_base()
   sources_rel_to_buildroot = dist_tgt.sources_relative_to_buildroot()
+  # NB: We need target paths both relative to the target base and relative to 
+  # the build root for the shutil file copying below.
   sources = zip(sources_rel_to_buildroot, sources_rel_to_target_base)
   for source_relative_to_build_root, source_relative_to_target_base in sources:
     path_to_source = os.path.join(tmp_dir_for_dist, source_relative_to_target_base)
     if not os.path.exists(os.path.dirname(path_to_source)):
       os.makedirs(os.path.dirname(path_to_source))
     shutil.copyfile(os.path.join(get_buildroot(), source_relative_to_build_root), path_to_source)
 
-
   return tmp_dir_for_dist  
 
 

src/python/pants/backend/python/tasks/python_binary_create.py

@@ -132,9 +132,8 @@ def _create_binary(self, binary_tgt, results_dir):
         dump_sources(builder, tgt, self.context.log)
       dump_requirements(builder, interpreter, req_tgts, self.context.log, binary_tgt.platforms)
 
-      # Dump python_distributions, if any, into builder's chroot.
+      # Dump built python distributions, if any, into builder's chroot.
       built_dists = self.context.products.get_data('python-dists')
-      import pdb;pdb.set_trace()
       if built_dists:
         for dist in built_dists:
           builder.add_dist_location(dist)

src/python/pants/backend/python/tasks2/python_create_distributions.py

@@ -13,10 +13,10 @@
 
 from pants.backend.python.targets.python_binary import PythonBinary
 from pants.backend.python.targets.python_distribution import PythonDistribution
-from pants.backend.python.tasks2.pex_build_util import (build_python_distribution,
-                                                        dump_requirements, dump_sources,
+from pants.backend.python.tasks2.pex_build_util import (dump_requirements, dump_sources,
                                                         has_python_requirements, has_python_sources,
-                                                        has_resources, is_local_python_dist)
+                                                        has_resources, is_local_python_dist,
+                                                        prepare_dist_workdir)
 from pants.backend.python.tasks2.setup_py import SetupPyRunner
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
@@ -62,7 +62,6 @@ def execute(self):
         names[name] = dist_target
 
       with self.invalidated(dist_targets, invalidate_dependents=True) as invalidation_check:
-        import pdb;pdb.set_trace()
         for vt in invalidation_check.all_vts:
           built_dists.add(self._create_dist(vt.target)) # vt.results dir
 
@@ -71,19 +70,19 @@ def execute(self):
   def _create_dist(self, dist_tgt):
     """Create a .whl file for the specified python_distribution target."""
     interpreter = self.context.products.get_data(PythonInterpreter)
-
+
+    # Build whl from python_dist target.
     whl_location = ''
-    # build whl from python_dist target
-    dist_target_dir = build_python_distribution(dist_tgt, interpreter, self.workdir, self.context.log)
-
-    # build the whl from pex API using tempdir and get its location
+    dist_target_dir = prepare_dist_workdir(dist_tgt, self.workdir, self.context.log)
+
+    # Build the whl from pex API using tempdir and get its location.
     install_dir = os.path.join(dist_target_dir, 'dist')
     if not os.path.exists(install_dir):
       safe_mkdir(install_dir)
     setup_runner = SetupPyRunner(dist_target_dir, 'bdist_wheel', interpreter=interpreter, install_dir=install_dir)
     setup_runner.run()
 
-    # return the location of the whl on disk (somewhere in pantsd or dist)
+    # Return the location of the whl on disk.
     dists = os.listdir(install_dir)
     if len(dists) == 0:
       raise TaskError('No distributions were produced by python_create_distribution task.')