Python distribution task for user-defined setup.py + integration with ./pants {run/binary/test}

examples/src/python/example/python_distribution/hello/fasthello/BUILD

@@ -0,0 +1,23 @@
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# Like Hello world, but built with a python_dist.
+# python_dist allows you to use setup.py to depend on C/C++ extensions.
+
+python_dist(
+  name='fasthello',
+  sources=[
+    'super_greet.c',
+    'hello_package/hello.py',
+    'hello_package/__init__.py',
+    'setup.py'
+  ]
+)
+
+python_binary(
+  name='main',
+  source='main.py',
+  dependencies=[
+    ':fasthello',
+  ]
+)

examples/src/python/example/python_distribution/hello/fasthello/hello_package/hello.py

@@ -0,0 +1,12 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import super_greet
+
+def hello():
+  print(super_greet.super_greet())
+  return super_greet.super_greet()

examples/src/python/example/python_distribution/hello/fasthello/main.py

@@ -0,0 +1,13 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+# hello_package is a python module within the fasthello python_distribution
+from hello_package import hello
+
+
+if __name__ == '__main__':
+  hello.hello()

examples/src/python/example/python_distribution/hello/fasthello/setup.py

@@ -0,0 +1,19 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from setuptools import setup, find_packages
+from distutils.core import Extension
+
+
+c_module = Extension(str('super_greet'), sources=[str('super_greet.c')])
+
+setup(
+  name='fasthello',
+  version='1.0.0',
+  ext_modules=[c_module],
+  packages=find_packages(),
+)

examples/src/python/example/python_distribution/hello/fasthello/super_greet.c

@@ -0,0 +1,14 @@
+#include <Python.h>
+
+static PyObject * super_greet(PyObject *self, PyObject *args) {
+  return Py_BuildValue("s", "Super hello");
+}
+
+static PyMethodDef Methods[] = {
+  {"super_greet", super_greet, METH_VARARGS, "A super greeting"},
+  {NULL, NULL, 0, NULL}
+};
+
+PyMODINIT_FUNC initsuper_greet(void) {
+  (void) Py_InitModule("super_greet", Methods);
+}

examples/tests/python/example/python_distribution/hello/test_fasthello/BUILD

@@ -0,0 +1,14 @@
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+# Example of defining a test target that depends on a python_dist target.
+
+python_tests(
+  name='fasthello',
+  sources=[
+    'test_fasthello.py'
+  ],
+  dependencies=[
+    'examples/src/python/example/python_distribution/hello/fasthello:fasthello'
+  ]
+)

examples/tests/python/example/python_distribution/hello/test_fasthello/test_fasthello.py

@@ -0,0 +1,14 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+# hello_package is a python module within the fasthello python_distribution.
+from hello_package import hello
+
+
+# Example of writing a test that depends on a python_dist target.
+def test_fasthello():
+  assert hello.hello() == "Super hello"

src/python/pants/backend/python/register.py

@@ -10,9 +10,12 @@
 from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.python_requirements import PythonRequirements
 from pants.backend.python.targets.python_binary import PythonBinary
+from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.backend.python.targets.python_library import PythonLibrary
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_tests import PythonTests
+from pants.backend.python.tasks.build_local_python_distributions import \
+  BuildLocalPythonDistributions
 from pants.backend.python.tasks.gather_sources import GatherSources
 from pants.backend.python.tasks.pytest_prep import PytestPrep
 from pants.backend.python.tasks.pytest_run import PytestRun
@@ -34,6 +37,7 @@ def build_file_aliases():
       PythonBinary.alias(): PythonBinary,
       PythonLibrary.alias(): PythonLibrary,
       PythonTests.alias(): PythonTests,
+      PythonDistribution.alias(): PythonDistribution,
       'python_requirement_library': PythonRequirementLibrary,
       Resources.alias(): Resources,
     },
@@ -51,6 +55,7 @@ def build_file_aliases():
 
 def register_goals():
   task(name='interpreter', action=SelectInterpreter).install('pyprep')
+  task(name='build-local-dists', action=BuildLocalPythonDistributions).install('pyprep')
   task(name='requirements', action=ResolveRequirements).install('pyprep')
   task(name='sources', action=GatherSources).install('pyprep')
   task(name='py', action=PythonRun).install('run')

src/python/pants/backend/python/targets/BUILD

@@ -4,6 +4,7 @@
 python_library(
   sources = [
     'python_binary.py',
+    'python_distribution.py',
     'python_library.py',
     'python_requirement_library.py',
     'python_target.py',

src/python/pants/backend/python/targets/python_distribution.py

@@ -0,0 +1,63 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pex.interpreter import PythonIdentity
+from twitter.common.collections import maybe_list
+
+from pants.base.exceptions import TargetDefinitionException
+from pants.base.payload import Payload
+from pants.base.payload_field import PrimitiveField
+from pants.build_graph.target import Target
+
+
+class PythonDistribution(Target):
+  """A Python distribution target that accepts a user-defined setup.py."""
+
+  default_sources_globs = '*.py'
+
+  @classmethod
+  def alias(cls):
+    return 'python_dist'
+
+  def __init__(self,
+               address=None,
+               payload=None,
+               sources=None,
+               compatibility=None,
+               **kwargs):
+    """
+    :param address: The Address that maps to this Target in the BuildGraph.
+    :type address: :class:`pants.build_graph.address.Address`
+    :param payload: The configuration encapsulated by this target.  Also in charge of most
+                    fingerprinting details.
+    :type payload: :class:`pants.base.payload.Payload`
+    :param sources: Files to "include". Paths are relative to the
+      BUILD file's directory.
+    :type sources: ``Fileset`` or list of strings. Must include setup.py.
+    :param compatibility: either a string or list of strings that represents
+      interpreter compatibility for this target, using the Requirement-style
+      format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements
+      agnostic to interpreter class.
+    """
+    payload = payload or Payload()
+    payload.add_fields({
+      'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'),
+      'compatibility': PrimitiveField(maybe_list(compatibility or ()))
+    })
+    super(PythonDistribution, self).__init__(address=address, payload=payload, **kwargs)
+
+    if not 'setup.py' in sources:
+      raise TargetDefinitionException(
+        self, 'A setup.py in the top-level directory relative to the target definition is required.'
+      )
+
+    # Check that the compatibility requirements are well-formed.
+    for req in self.payload.compatibility:
+      try:
+        PythonIdentity.parse_requirement(req)
+      except ValueError as e:
+        raise TargetDefinitionException(self, str(e))

src/python/pants/backend/python/tasks/BUILD

@@ -6,6 +6,8 @@ python_library(
     '3rdparty/python:pex',
     '3rdparty/python/twitter/commons:twitter.common.collections',
     '3rdparty/python/twitter/commons:twitter.common.dirutil',
+    'src/python/pants/backend/python:python_requirement',
+    'src/python/pants/backend/python:python_requirements',
     'src/python/pants/backend/python:interpreter_cache',
     'src/python/pants/backend/python:python_chroot',
     'src/python/pants/backend/python/subsystems',

src/python/pants/backend/python/tasks/build_local_python_distributions.py

@@ -0,0 +1,88 @@
+# coding=utf-8
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import glob
+import os
+import shutil
+
+from pex.interpreter import PythonInterpreter
+
+from pants.backend.python.tasks.pex_build_util import is_local_python_dist
+from pants.backend.python.tasks.setup_py import SetupPyRunner
+from pants.base.build_environment import get_buildroot
+from pants.base.exceptions import TargetDefinitionException, TaskError
+from pants.base.fingerprint_strategy import DefaultFingerprintStrategy
+from pants.task.task import Task
+from pants.util.dirutil import safe_mkdir
+
+
+class BuildLocalPythonDistributions(Task):
+  """Create python distributions (.whl) from python_dist targets."""
+
+  options_scope = 'python-create-distributions'
+  PYTHON_DISTS = 'user_defined_python_dists'
+
+  @classmethod
+  def product_types(cls):
+    return [cls.PYTHON_DISTS]
+
+  @classmethod
+  def prepare(cls, options, round_manager):
+    round_manager.require_data(PythonInterpreter)
+
+  @property
+  def cache_target_dirs(self):
+    return True
+
+  def execute(self):
+    dist_targets = self.context.targets(is_local_python_dist)
+    built_dists = set()
+
+    if dist_targets:
+      with self.invalidated(dist_targets,
+                            fingerprint_strategy=DefaultFingerprintStrategy(),
+                            invalidate_dependents=True) as invalidation_check:
+        for vt in invalidation_check.all_vts:
+          if vt.valid:
+            built_dists.add(self._get_whl_from_dir(os.path.join(vt.results_dir, 'dist')))
+          else:
+            if vt.target.dependencies:
+              raise TargetDefinitionException(
+                vt.target, 'The `dependencies` field is disallowed on `python_dist` targets. List any 3rd '
+                           'party requirements in the install_requirements argument of your setup function.'
+              )
+            built_dists.add(self._create_dist(vt.target, vt.results_dir))
+
+    self.context.products.register_data(self.PYTHON_DISTS, built_dists)
+
+  def _create_dist(self, dist_tgt, dist_target_dir):
+    """Create a .whl file for the specified python_distribution target."""
+    interpreter = self.context.products.get_data(PythonInterpreter)
+
+    # Copy sources and setup.py over to vt results directory for packaging.
+    # NB: The directory structure of the destination directory needs to match 1:1
+    # with the directory structure that setup.py expects.
+    for src_relative_to_target_base in dist_tgt.sources_relative_to_target_base():
+      src_rel_to_results_dir = os.path.join(dist_target_dir, src_relative_to_target_base)
+      safe_mkdir(os.path.dirname(src_rel_to_results_dir))
+      abs_src_path = os.path.join(get_buildroot(),
+                                  dist_tgt.address.spec_path,
+                                  src_relative_to_target_base)
+      shutil.copyfile(abs_src_path, src_rel_to_results_dir)
+    # Build a whl using SetupPyRunner and return its absolute path.
+    setup_runner = SetupPyRunner(dist_target_dir, 'bdist_wheel', interpreter=interpreter)
+    setup_runner.run()
+    return self._get_whl_from_dir(os.path.join(dist_target_dir, 'dist'))
+
+  def _get_whl_from_dir(self, install_dir):
+    """Return the absolute path of the whl in a setup.py install directory."""
+    dists = glob.glob(os.path.join(install_dir, '*.whl'))
+    if len(dists) == 0:
+      raise TaskError('No distributions were produced by python_create_distribution task.')
+    if len(dists) > 1:
+      raise TaskError('Ambiguous local python distributions found: {}'.format(dists))
+    return dists[0]

src/python/pants/backend/python/tasks/pex_build_util.py

@@ -12,13 +12,16 @@
 from pex.resolver import resolve
 from twitter.common.collections import OrderedSet
 
+from pants.backend.python.python_requirement import PythonRequirement
 from pants.backend.python.subsystems.python_setup import PythonSetup
 from pants.backend.python.targets.python_binary import PythonBinary
+from pants.backend.python.targets.python_distribution import PythonDistribution
 from pants.backend.python.targets.python_library import PythonLibrary
 from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary
 from pants.backend.python.targets.python_tests import PythonTests
 from pants.base.build_environment import get_buildroot
 from pants.base.exceptions import TaskError
+from pants.build_graph.address import Address
 from pants.build_graph.files import Files
 from pants.python.python_repos import PythonRepos
 
@@ -34,6 +37,10 @@ def has_python_sources(tgt):
   return is_python_target(tgt) and tgt.has_sources()
 
 
+def is_local_python_dist(tgt):
+  return isinstance(tgt, PythonDistribution)
+
+
 def has_resources(tgt):
   return isinstance(tgt, Files) and tgt.has_sources()
 
@@ -113,7 +120,6 @@ def dump_requirements(builder, interpreter, req_libs, log, platforms=None):
 
   # Resolve the requirements into distributions.
   distributions = _resolve_multi(interpreter, reqs_to_build, platforms, find_links)
-
   locations = set()
   for platform, dists in distributions.items():
     for dist in dists:
@@ -158,3 +164,43 @@ def _resolve_multi(interpreter, requirements, platforms, find_links):
       allow_prereleases=python_setup.resolver_allow_prereleases)
 
   return distributions
+
+
+def inject_synthetic_dist_requirements(build_graph, local_built_dists, synthetic_address, binary_tgt=None):
+  """Inject a synthetic requirements library from a local wheel.
+
+  :param build_graph: The build graph needed for injecting synthetic targets.
+  :param local_built_dists: A list of paths to locally built wheels to package into
+  requirements libraries.
+  :param synthetic_address: A generative address for addressing synthetic targets.
+  :param binary_tgt: An optional parameter to be passed only when called by the `python_binary_create`
+  task. This is needed to ensure that only python_dist targets in a binary target's closure are included
+  in the binary for the case where a user specifies mulitple binary targets in a single invocation of
+  `./pants binary`.
+  :return: a :class: `PythonRequirementLibrary` containing a requirements that maps to a locally-built wheels.
+  """
+  def should_create_req(bin_tgt, loc):
+    if not bin_tgt:
+      return True
+    # Ensure that a target is in a binary target's closure. See docstring for more detail.
+    return any([tgt.id in loc for tgt in bin_tgt.closure()])
+
+  def python_requirement_from_wheel(path):
+    base = os.path.basename(path)
+    whl_dir = os.path.dirname(path)
+    whl_metadata = base.split('-')
+    req_name = '=='.join([whl_metadata[0], whl_metadata[1]])
+    return PythonRequirement(req_name, repository=whl_dir)
+
+  local_whl_reqs = [
+    python_requirement_from_wheel(whl_location)
+    for whl_location in local_built_dists
+    if should_create_req(binary_tgt, whl_location)
+  ]
+
+  if not local_whl_reqs:
+    return []
+
+  addr = Address.parse(synthetic_address)
+  build_graph.inject_synthetic_target(addr, PythonRequirementLibrary, requirements=local_whl_reqs)
+  return [build_graph.get_target(addr)]