Create python distribution task and refactor example

examples/src/python/example/python_distribution/hello/superhello/BUILD

@@ -4,13 +4,19 @@
 # Like Hello world, but built with a python_distribution.
 # python_distribution allows you to use setup.py to depend on C/C++ extensions.
 
-# There must be a single BUILD file and a setup.py in a given python_distribution.
-
-# All you need to indicate is a name and any dependencies.
-# You are expected to define source locations in setup.py.
-# There should be a one-to-one mapping for the name of the python_distribution
-# and the name of the python_distribution's directory. In this case, it's superhello.
-
 python_distribution(
-  name='superhello'
+  name='superhello',
+  sources=[
+  	'super_greet.c',
+  	'hello.py',
+  	'setup.py'
+  ]
+)
+
+python_binary(
+  name='main',
+  source='main.py',
+  dependencies=[
+    'examples/src/python/example/python_distribution/hello/superhello:superhello',
+  ]
 )

examples/src/python/example/python_distribution/hello/superhello/hello_package/hello.py → examples/src/python/example/python_distribution/hello/superhello/hello.py

@@ -8,4 +8,4 @@
 import super_greet
 
 def hello():
-  print(super_greet.super_greet())
+  print(super_greet.super_greet())

examples/src/python/example/python_distribution/hello/superhello/setup.py

@@ -9,7 +9,7 @@
 from distutils.core import Extension
 
 
-c_module = Extension(str('super_greet'), sources=[str('c/super_greet.c')])
+c_module = Extension(str('super_greet'), sources=[str('super_greet.c')])
 
 setup(
     name='superhello',

src/python/pants/backend/python/targets/python_distribution.py

@@ -8,9 +8,9 @@
 
 from twitter.common.collections import maybe_list
 
-from pants.backend.python.targets.python_target import PythonTarget
 from pants.base.payload import Payload
 from pants.base.payload_field import PrimitiveField
+from pants.build_graph.target import Target
 
 class PythonDistribution(Target):
   """A Python distribution.
@@ -25,44 +25,49 @@ def __init__(self,
                sources=None,
                compatibility=None,
                **kwargs):
-  """
-  :param dependencies: The addresses of targets that this target depends on.
-    These dependencies may
-    be ``python_library``-like targets (``python_library``,
-    ``python_thrift_library``, ``python_antlr_library`` and so forth) or
-    ``python_requirement_library`` targets.
-  :type dependencies: list of strings
-  :param sources: Files to "include". Paths are relative to the
-    BUILD file's directory.
-  :type sources: ``Fileset`` or list of strings
-  :param resource_targets: The addresses of ``resources`` targets this target
-    depends on.
-  :param compatibility: either a string or list of strings that represents
-    interpreter compatibility for this target, using the Requirement-style
-    format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements
-    agnostic to interpreter class.
-  """
-  self.address = address
-  payload = payload or Payload()
-  payload.add_fields({
-    'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'),
-    'compatibility': PrimitiveField(maybe_list(compatibility or ())),
-  })
-  super(PythonDistribution, self).__init__(address=address, payload=payload, **kwargs)
-  self.add_labels('python')
+    """
+    :param dependencies: The addresses of targets that this target depends on.
+      These dependencies may
+      be ``python_library``-like targets (``python_library``,
+      ``python_thrift_library``, ``python_antlr_library`` and so forth) or
+      ``python_requirement_library`` targets.
+    :type dependencies: list of strings
+    :param sources: Files to "include". Paths are relative to the
+      BUILD file's directory.
+    :type sources: ``Fileset`` or list of strings
+    :param resource_targets: The addresses of ``resources`` targets this target
+      depends on.
+    :param compatibility: either a string or list of strings that represents
+      interpreter compatibility for this target, using the Requirement-style
+      format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements
+      agnostic to interpreter class.
+    """
+    self.address = address
+    payload = payload or Payload()
+    payload.add_fields({
+      'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'),
+      'compatibility': PrimitiveField(maybe_list(compatibility or ()))
+    })
+    super(PythonDistribution, self).__init__(address=address, payload=payload, **kwargs)
+    self.add_labels('python')
+
+    sources_basenames = [os.path.basename(source) for source in sources]
+    if not 'setup.py' in sources_basenames:
+      raise TargetDefinitionException(self,
+        'A setup.py is required to create a python_dist. You must include a setup.py file in your sources field.')
 
-  # Check that the compatibility requirements are well-formed.
-  for req in self.payload.compatibility:
-    try:
-      PythonIdentity.parse_requirement(req)
-    except ValueError as e:
-      raise TargetDefinitionException(self, str(e))
+    # Check that the compatibility requirements are well-formed.
+    for req in self.payload.compatibility:
+      try:
+        PythonIdentity.parse_requirement(req)
+      except ValueError as e:
+        raise TargetDefinitionException(self, str(e))
 
 
-  @classmethod
-  def alias(cls):
-    return 'python_dist'
+    @classmethod
+    def alias(cls):
+      return 'python_dist'
 
-  def __init__(self, **kwargs):
-    payload = Payload()
-    super(PythonDistribution, self).__init__(sources=[], payload=payload, **kwargs)
+    def __init__(self, **kwargs):
+      payload = Payload()
+      super(PythonDistribution, self).__init__(sources=[], payload=payload, **kwargs)

src/python/pants/backend/python/tasks/pex_build_util.py

@@ -22,6 +22,7 @@
 from pants.backend.python.targets.python_library import PythonLibrary
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_tests import PythonTests
+from pants.backend.python.tasks2.setup_py import SetupPyRunner
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
 from pants.build_graph.files import Files
@@ -169,32 +170,44 @@ def build_python_distribution_from_target(target, workdir):
     raise TaskError('Failed to package python distribution for target: %s', target.name)
 
 
-def dump_python_distributions(builder, dist_targets, workdir, log):
+def build_python_distribution(dist_tgt, interpreter, workdir, log):
   """Dump python distribution targets into a given builder
 
-  :param builder: Dump the python_distributions into this builder.
-  :param dist_targets: A list of `PythonDistribution` targets to build.
-  :param workdir: Working directory for python targets (./pantsd/python)
-  :param log: Use this logger.
   """
-
-  # build whl for target using pex wheel installer
-  locations = set()
-  for tgt in dist_targets:
-    whl_location = build_python_distribution_from_target(tgt, workdir)
-    if whl_location in locations:
-      raise TaskError('Two wheels of the same name have been created: %s.', whl_location)
-    if whl_location:
-      locations.add(whl_location)
-
-  # dump wheels into pex builder
-  # After this block, the whls built from python_distribution should be available
-  # for use in the produced binary
-  for location in locations:
-    log.debug('  Dumping distribution: .../{}'.format(os.path.basename(location)))
-    builder.add_dist_location(location)
-
-
+  # make directory based on fingerprint in workdir
+  local_dists_workdir = os.path.join(workdir, 'local_dists')
+  if not os.path.exists(local_dists_workdir):
+    safe_mkdir(local_dists_workdir)
+
+  fingerprint = dist_tgt.payload.fingerprint()
+  dist_target_dir = os.path.join(local_dists_workdir, fingerprint)
+  if not os.path.exists(dist_target_dir):
+    safe_mkdir(dist_target_dir)
+
+  tmp_dir_for_dist = os.path.join(workdir, 'tmp')
+  if not tmp_dir_for_dist:
+    safe_mkdir(tmp_dir_for_dist)
+
+  # copy sources and setup.py to this temp dir
+  sources = dist_tgt.sources_relative_to_buildroot()
+  for source in sources:
+    shutil.copyfile(os.path.join(buildroot, source), os.path.join(tmp_dir_for_dist, source))
+
+  # build the whl from pex API using tempdir and get its location
+  install_dir = os.path.join(dist_target_dir, 'dist')
+  if not install_dir:
+    safe_mkdir(install_dir)
+  setup_runner = SetupPyRunner(tmp_dir_for_dist, 'bdist_wheel', interpreter=interpreter, install_dir=install_dir)
+  setup_runner.run()
+
+  # return the location of the whl on disk (somewhere in pantsd or dist)
+  dists = os.listdir(self.install_tmp)
+  if len(dists) == 0:
+    raise TaskError('No distributions were produced by python_create_distribution task.')
+  elif len(dists) > 1:
+    raise TaskError('Ambiguous whls found: %s' % (' '.join(dists)))
+  else:
+    return os.path.join(self.install_tmp, dists[0])
 
 
 def _resolve_multi(interpreter, requirements, platforms, find_links):

src/python/pants/backend/python/tasks/python_binary_create.py

@@ -133,8 +133,10 @@ def _create_binary(self, binary_tgt, results_dir):
       dump_requirements(builder, interpreter, req_tgts, self.context.log, binary_tgt.platforms)
 
       # Dump python_distributions, if any, into builder's chroot.
-      if python_dist_targets:
-        dump_python_distributions(builder, python_dist_targets, self.workdir, self.context.log)
+      built_dists = self.context.products.get_data('python_dists')
+      if built_dists:
+        for dist in built_dists:
+          builder.add_location(dist)
 
       # Build the .pex file.
       pex_path = os.path.join(results_dir, '{}.pex'.format(binary_tgt.name))

src/python/pants/backend/python/tasks2/python_create_distributions.py

@@ -0,0 +1,84 @@
+# coding=utf-8
+# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pex.interpreter import PythonInterpreter
+from pex.pex_builder import PEXBuilder
+from pex.pex_info import PexInfo
+
+from pants.backend.python.targets.python_binary import PythonBinary
+from pants.backend.python.tasks2.pex_build_util import (dump_python_distributions,
+                                                        dump_requirements, dump_sources,
+                                                        has_python_requirements, has_python_sources,
+                                                        has_resources, is_local_python_dist)
+from pants.base.build_environment import get_buildroot
+from pants.base.exceptions import TaskError
+from pants.build_graph.target_scopes import Scopes
+from pants.task.task import Task
+from pants.util.contextutil import temporary_dir
+from pants.util.dirutil import safe_mkdir_for
+from pants.util.fileutil import atomic_copy
+
+
+class PythonCreateDistributions(Task):
+  """Create python distributions (.whl) from python_dist targets."""
+
+  @classmethod
+  def product_types(cls):
+    return ['python_dists']
+
+  @classmethod
+  def prepare(cls, options, round_manager):
+    round_manager.require_data(PythonInterpreter)
+    round_manager.require_data('python')  # For codegen.
+
+  @staticmethod
+  def is_distribution(target):
+    return isinstance(target, PythonDistribution)
+
+  def __init__(self, *args, **kwargs):
+    super(PythonCreateDistributions, self).__init__(*args, **kwargs)
+    self._distdir = self.get_options().pants_distdir
+
+  def execute(self):
+    dist_targets = self.context.targets(self.is_distribution)
+    built_dists = set()
+
+    if dist_targets:
+      # Check for duplicate distribution names, since we write the pexes to <dist>/<name>.pex.
+      names = {}
+      for dist_target in dist_targets:
+        name = dist_target.name
+        if name in names:
+          raise TaskError('Cannot build two dist_targets with the same name in a single invocation. '
+                          '{} and {} both have the name {}.'.format(dist_target, names[name], name))
+        names[name] = dist_target
+
+      with self.invalidated(dist_targets, invalidate_dependents=True) as invalidation_check:
+
+        for vt in invalidation_check.all_vts:
+          pex_path = os.path.join(vt.results_dir, '{}.pex'.format(vt.target.name))
+          if not vt.valid:
+            self.context.log.debug('cache for {} is invalid, rebuilding'.format(vt.target))
+            built_dists.add(self._create_dist(vt.target)). # vt.results dir
+          else:
+            self.context.log.debug('using cache for {}'.format(vt.target))
+
+    self.context.products.register_data('python_dists', built_dists)
+
+  def _create_dist(self, dist_tgt):
+    """Create a .whl file for the specified python_distribution target."""
+    interpreter = self.context.products.get_data(PythonInterpreter)
+
+    whl_location = ''
+    # build whl from python_dist target
+    whl = build_python_distribution(dist_tgt, interpreter, self.workdir, self.context.log)
+    if whl:
+      whl_location = whl
+
+    return whl_location