Clean up + add some docs

3rdparty/python/requirements.txt

@@ -30,5 +30,4 @@ setuptools==30.0.0
 subprocess32==3.2.7
 six>=1.9.0,<2
 thrift>=0.9.1
-tensorflow==1.4.0
 wheel==0.29.0

examples/src/python/example/python_distribution/hello/main/BUILD

@@ -1,4 +1,4 @@
-# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 # Like Hello world, but built with Pants.

examples/src/python/example/python_distribution/hello/main/main.py

@@ -1,12 +1,13 @@
 # coding=utf-8
-# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
 import sys
 
+# hello_package is a python module within the superhello python_distribution
 from hello_package import hello
 
 if __name__ == '__main__':

examples/src/python/example/python_distribution/hello/superhello/BUILD

@@ -1,14 +1,16 @@
-# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-# Like Hello world, but built with Pants. This time, with both C++ and Py sources
+# Like Hello world, but built with a python_distribution.
+# python_distribution allows you to use setup.py to depend on C/C++ extensions.
 
-# The BUILD file must be in the directory of the Python Distribution, and there must
-# only be a single BUILD file in a given directory.
+# There must be a single BUILD file and a setup.py in a given python_distribution.
+
+# All you need to indicate is a name and any dependencies.
+# You are expected to define source locations in setup.py.
+# There should be a one-to-one mapping for the name of the python_distribution
+# and the name of the python_distribution's directory. In this case, it's superhello.
 
 python_distribution(
-  name='superhello',
-  dependencies=[
-	'3rdparty/python:tensorflow'
-  ]
+  name='superhello'
 )

examples/src/python/example/python_distribution/hello/superhello/c/super_greet.c

@@ -1,4 +1,3 @@
-
 #include <Python.h>
 
 static PyObject * super_greet(PyObject *self, PyObject *args) {

examples/src/python/example/python_distribution/hello/superhello/hello_package/hello.py

@@ -6,8 +6,6 @@
                         unicode_literals, with_statement)
 
 import super_greet
-import tensorflow
 
 def hello():
-  print(tensorflow)
   print(super_greet.super_greet())

examples/src/python/example/python_distribution/hello/superhello/setup.py

@@ -3,13 +3,13 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
-                        with_statement)
+                        unicode_literals, with_statement)
 
 from setuptools import setup, find_packages
 from distutils.core import Extension
 
 
-c_module = Extension('super_greet', sources=['c/super_greet.c'])
+c_module = Extension(str('super_greet'), sources=[str('c/super_greet.c')])
 
 setup(
     name='superhello',

src/python/pants/backend/python/targets/python_distribution.py

@@ -1,17 +1,14 @@
 # coding=utf-8
-# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
+# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
 
-from pex.pex_info import PexInfo
-from six import string_types
 from twitter.common.collections import maybe_list
 
 from pants.backend.python.targets.python_target import PythonTarget
-from pants.backend.python.targets.python_tests import PythonTests
 from pants.base.payload import Payload
 from pants.base.payload_field import PrimitiveField
 

src/python/pants/backend/python/tasks/pex_build_util.py

@@ -133,12 +133,13 @@ def dump_requirements(builder, interpreter, req_libs, log, platforms=None):
 
 
 def build_python_distribution_from_target(target, workdir):
-
-  pydist_workdir = os.path.join(workdir, '.pydistworkdir')
+  # Create a temporary working directory, pydistworkdir
+  pydist_workdir = os.path.join(workdir, 'pydistworkdir')
   safe_mkdir(pydist_workdir)
   pex_name = "%s.pex" % target.name
   path_to_target = os.path.dirname(target.address.rel_path)
 
+  # disable cache to ensure reproducible runs
   args = ['--disable-cache', path_to_target, '-o', os.path.join(pydist_workdir, pex_name)]
   try:
     pex_main.main(args=args)
@@ -167,26 +168,30 @@ def build_python_distribution_from_target(target, workdir):
   return None
 
 
-def dump_python_distibutions(builder, dist_targets, workdir, log):
-  # de-dup all dist targets.
-  dist_targets_set = OrderedSet()
-  for tgt in dist_targets:
-    dist_targets_set.add(tgt)
+def dump_python_distributions(builder, dist_targets, workdir, log):
+  """Dump python distribution targets into a given builder
+
+  :param builder: Dump the python_distributions into this builder.
+  :param dist_targets: PythonDistribution targets to dump
+  :param workdir: Working directory for python targets (./pantsd/python)
+  :param log: Use this logger.
+  """
 
   # build whl for target using pex wheel installer
   locations = set()
-  for tgt in dist_targets_set:
+  for tgt in dist_targets:
     whl_location = build_python_distribution_from_target(tgt, workdir)
     if whl_location:
       locations.add(whl_location)
 
-  # dump prebuilt wheels into pex builder
+  # dump wheels into pex builder
+  # After this block, the whls built from python_distribution should be available
+  # for use in the produced binary
   for location in locations:
     log.debug('  Dumping distribution: .../{}'.format(os.path.basename(location)))
     builder.add_dist_location(location)
 
-  # by this point, the whl built from python_distribution should be available for use in the
-  # produced binary
+
 
 
 def _resolve_multi(interpreter, requirements, platforms, find_links):

src/python/pants/backend/python/tasks/python_binary_create.py

@@ -12,7 +12,7 @@
 from pex.pex_info import PexInfo
 
 from pants.backend.python.targets.python_binary import PythonBinary
-from pants.backend.python.tasks2.pex_build_util import (dump_python_distibutions, dump_requirements,
+from pants.backend.python.tasks2.pex_build_util import (dump_python_distributions, dump_requirements,
                                                         dump_sources, has_python_and_c_sources,
                                                         has_python_requirements, has_python_sources,
                                                         has_resources)
@@ -131,8 +131,10 @@ def _create_binary(self, binary_tgt, results_dir):
       for tgt in source_tgts:
         dump_sources(builder, tgt, self.context.log)
       dump_requirements(builder, interpreter, req_tgts, self.context.log, binary_tgt.platforms)
+
+      # Dump python_distributions, if any, into builder's chroot.
       if python_dist_targets:
-        dump_python_distibutions(builder, python_dist_targets, self.workdir, self.context.log)
+        dump_python_distributions(builder, python_dist_targets, self.workdir, self.context.log)
 
       # Build the .pex file.
       pex_path = os.path.join(results_dir, '{}.pex'.format(binary_tgt.name))