diff --git a/examples/src/python/example/python_distribution/hello/main/BUILD b/examples/src/python/example/python_distribution/hello/main/BUILD deleted file mode 100644 index ac0850abdee..00000000000 --- a/examples/src/python/example/python_distribution/hello/main/BUILD +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -# Like Hello world, but built with Pants. - -python_binary( - dependencies=[ - 'examples/src/python/example/python_distribution/hello/superhello:superhello', - ], - source='main.py', -) diff --git a/examples/src/python/example/python_distribution/hello/main/__init__.py b/examples/src/python/example/python_distribution/hello/main/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/examples/src/python/example/python_distribution/hello/superhello/BUILD b/examples/src/python/example/python_distribution/hello/superhello/BUILD index 5637032a5b0..65f1fa4ecfe 100644 --- a/examples/src/python/example/python_distribution/hello/superhello/BUILD +++ b/examples/src/python/example/python_distribution/hello/superhello/BUILD @@ -4,13 +4,19 @@ # Like Hello world, but built with a python_distribution. # python_distribution allows you to use setup.py to depend on C/C++ extensions. -# There must be a single BUILD file and a setup.py in a given python_distribution. - -# All you need to indicate is a name and any dependencies. -# You are expected to define source locations in setup.py. -# There should be a one-to-one mapping for the name of the python_distribution -# and the name of the python_distribution's directory. In this case, it's superhello. - python_distribution( - name='superhello' + name='superhello', + sources=[ + 'super_greet.c', + 'hello.py', + 'setup.py' + ] +) + +python_binary( + name='main', + source='main.py', + dependencies=[ + 'examples/src/python/example/python_distribution/hello/superhello:superhello', + ] ) diff --git a/examples/src/python/example/python_distribution/hello/superhello/hello_package/hello.py b/examples/src/python/example/python_distribution/hello/superhello/hello.py similarity index 90% rename from examples/src/python/example/python_distribution/hello/superhello/hello_package/hello.py rename to examples/src/python/example/python_distribution/hello/superhello/hello.py index fa29f9a2bbd..5581f9cac75 100644 --- a/examples/src/python/example/python_distribution/hello/superhello/hello_package/hello.py +++ b/examples/src/python/example/python_distribution/hello/superhello/hello.py @@ -8,4 +8,4 @@ import super_greet def hello(): - print(super_greet.super_greet()) + print(super_greet.super_greet()) \ No newline at end of file diff --git a/examples/src/python/example/python_distribution/hello/superhello/hello_package/__init__.py b/examples/src/python/example/python_distribution/hello/superhello/hello_package/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/examples/src/python/example/python_distribution/hello/main/main.py b/examples/src/python/example/python_distribution/hello/superhello/main.py similarity index 100% rename from examples/src/python/example/python_distribution/hello/main/main.py rename to examples/src/python/example/python_distribution/hello/superhello/main.py diff --git a/examples/src/python/example/python_distribution/hello/superhello/setup.py b/examples/src/python/example/python_distribution/hello/superhello/setup.py index 501e2e6988e..0a43a8c9a0a 100644 --- a/examples/src/python/example/python_distribution/hello/superhello/setup.py +++ b/examples/src/python/example/python_distribution/hello/superhello/setup.py @@ -9,7 +9,7 @@ from distutils.core import Extension -c_module = Extension(str('super_greet'), sources=[str('c/super_greet.c')]) +c_module = Extension(str('super_greet'), sources=[str('super_greet.c')]) setup( name='superhello', diff --git a/examples/src/python/example/python_distribution/hello/superhello/c/super_greet.c b/examples/src/python/example/python_distribution/hello/superhello/super_greet.c similarity index 100% rename from examples/src/python/example/python_distribution/hello/superhello/c/super_greet.c rename to examples/src/python/example/python_distribution/hello/superhello/super_greet.c diff --git a/src/python/pants/backend/python/targets/python_distribution.py b/src/python/pants/backend/python/targets/python_distribution.py index ffb427e1d7c..8c4ad9879e8 100644 --- a/src/python/pants/backend/python/targets/python_distribution.py +++ b/src/python/pants/backend/python/targets/python_distribution.py @@ -8,9 +8,9 @@ from twitter.common.collections import maybe_list -from pants.backend.python.targets.python_target import PythonTarget from pants.base.payload import Payload from pants.base.payload_field import PrimitiveField +from pants.build_graph.target import Target class PythonDistribution(Target): """A Python distribution. @@ -25,44 +25,49 @@ def __init__(self, sources=None, compatibility=None, **kwargs): - """ - :param dependencies: The addresses of targets that this target depends on. - These dependencies may - be ``python_library``-like targets (``python_library``, - ``python_thrift_library``, ``python_antlr_library`` and so forth) or - ``python_requirement_library`` targets. - :type dependencies: list of strings - :param sources: Files to "include". Paths are relative to the - BUILD file's directory. - :type sources: ``Fileset`` or list of strings - :param resource_targets: The addresses of ``resources`` targets this target - depends on. - :param compatibility: either a string or list of strings that represents - interpreter compatibility for this target, using the Requirement-style - format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements - agnostic to interpreter class. - """ - self.address = address - payload = payload or Payload() - payload.add_fields({ - 'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'), - 'compatibility': PrimitiveField(maybe_list(compatibility or ())), - }) - super(PythonDistribution, self).__init__(address=address, payload=payload, **kwargs) - self.add_labels('python') + """ + :param dependencies: The addresses of targets that this target depends on. + These dependencies may + be ``python_library``-like targets (``python_library``, + ``python_thrift_library``, ``python_antlr_library`` and so forth) or + ``python_requirement_library`` targets. + :type dependencies: list of strings + :param sources: Files to "include". Paths are relative to the + BUILD file's directory. + :type sources: ``Fileset`` or list of strings + :param resource_targets: The addresses of ``resources`` targets this target + depends on. + :param compatibility: either a string or list of strings that represents + interpreter compatibility for this target, using the Requirement-style + format, e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements + agnostic to interpreter class. + """ + self.address = address + payload = payload or Payload() + payload.add_fields({ + 'sources': self.create_sources_field(sources, address.spec_path, key_arg='sources'), + 'compatibility': PrimitiveField(maybe_list(compatibility or ())) + }) + super(PythonDistribution, self).__init__(address=address, payload=payload, **kwargs) + self.add_labels('python') + + sources_basenames = [os.path.basename(source) for source in sources] + if not 'setup.py' in sources_basenames: + raise TargetDefinitionException(self, + 'A setup.py is required to create a python_dist. You must include a setup.py file in your sources field.') - # Check that the compatibility requirements are well-formed. - for req in self.payload.compatibility: - try: - PythonIdentity.parse_requirement(req) - except ValueError as e: - raise TargetDefinitionException(self, str(e)) + # Check that the compatibility requirements are well-formed. + for req in self.payload.compatibility: + try: + PythonIdentity.parse_requirement(req) + except ValueError as e: + raise TargetDefinitionException(self, str(e)) - @classmethod - def alias(cls): - return 'python_dist' + @classmethod + def alias(cls): + return 'python_dist' - def __init__(self, **kwargs): - payload = Payload() - super(PythonDistribution, self).__init__(sources=[], payload=payload, **kwargs) + def __init__(self, **kwargs): + payload = Payload() + super(PythonDistribution, self).__init__(sources=[], payload=payload, **kwargs) diff --git a/src/python/pants/backend/python/tasks/pex_build_util.py b/src/python/pants/backend/python/tasks/pex_build_util.py index e0f556f570a..32c4c50b7dd 100644 --- a/src/python/pants/backend/python/tasks/pex_build_util.py +++ b/src/python/pants/backend/python/tasks/pex_build_util.py @@ -22,6 +22,7 @@ from pants.backend.python.targets.python_library import PythonLibrary from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.targets.python_tests import PythonTests +from pants.backend.python.tasks2.setup_py import SetupPyRunner from pants.base.build_environment import get_buildroot from pants.base.exceptions import TaskError from pants.build_graph.files import Files @@ -169,32 +170,44 @@ def build_python_distribution_from_target(target, workdir): raise TaskError('Failed to package python distribution for target: %s', target.name) -def dump_python_distributions(builder, dist_targets, workdir, log): +def build_python_distribution(dist_tgt, interpreter, workdir, log): """Dump python distribution targets into a given builder - :param builder: Dump the python_distributions into this builder. - :param dist_targets: A list of `PythonDistribution` targets to build. - :param workdir: Working directory for python targets (./pantsd/python) - :param log: Use this logger. """ - - # build whl for target using pex wheel installer - locations = set() - for tgt in dist_targets: - whl_location = build_python_distribution_from_target(tgt, workdir) - if whl_location in locations: - raise TaskError('Two wheels of the same name have been created: %s.', whl_location) - if whl_location: - locations.add(whl_location) - - # dump wheels into pex builder - # After this block, the whls built from python_distribution should be available - # for use in the produced binary - for location in locations: - log.debug(' Dumping distribution: .../{}'.format(os.path.basename(location))) - builder.add_dist_location(location) - - + # make directory based on fingerprint in workdir + local_dists_workdir = os.path.join(workdir, 'local_dists') + if not os.path.exists(local_dists_workdir): + safe_mkdir(local_dists_workdir) + + fingerprint = dist_tgt.payload.fingerprint() + dist_target_dir = os.path.join(local_dists_workdir, fingerprint) + if not os.path.exists(dist_target_dir): + safe_mkdir(dist_target_dir) + + tmp_dir_for_dist = os.path.join(workdir, 'tmp') + if not tmp_dir_for_dist: + safe_mkdir(tmp_dir_for_dist) + + # copy sources and setup.py to this temp dir + sources = dist_tgt.sources_relative_to_buildroot() + for source in sources: + shutil.copyfile(os.path.join(buildroot, source), os.path.join(tmp_dir_for_dist, source)) + + # build the whl from pex API using tempdir and get its location + install_dir = os.path.join(dist_target_dir, 'dist') + if not install_dir: + safe_mkdir(install_dir) + setup_runner = SetupPyRunner(tmp_dir_for_dist, 'bdist_wheel', interpreter=interpreter, install_dir=install_dir) + setup_runner.run() + + # return the location of the whl on disk (somewhere in pantsd or dist) + dists = os.listdir(self.install_tmp) + if len(dists) == 0: + raise TaskError('No distributions were produced by python_create_distribution task.') + elif len(dists) > 1: + raise TaskError('Ambiguous whls found: %s' % (' '.join(dists))) + else: + return os.path.join(self.install_tmp, dists[0]) def _resolve_multi(interpreter, requirements, platforms, find_links): diff --git a/src/python/pants/backend/python/tasks/python_binary_create.py b/src/python/pants/backend/python/tasks/python_binary_create.py index 9fe36255b15..6d01d8fd13f 100644 --- a/src/python/pants/backend/python/tasks/python_binary_create.py +++ b/src/python/pants/backend/python/tasks/python_binary_create.py @@ -133,8 +133,10 @@ def _create_binary(self, binary_tgt, results_dir): dump_requirements(builder, interpreter, req_tgts, self.context.log, binary_tgt.platforms) # Dump python_distributions, if any, into builder's chroot. - if python_dist_targets: - dump_python_distributions(builder, python_dist_targets, self.workdir, self.context.log) + built_dists = self.context.products.get_data('python_dists') + if built_dists: + for dist in built_dists: + builder.add_location(dist) # Build the .pex file. pex_path = os.path.join(results_dir, '{}.pex'.format(binary_tgt.name)) diff --git a/src/python/pants/backend/python/tasks2/python_create_distributions.py b/src/python/pants/backend/python/tasks2/python_create_distributions.py new file mode 100644 index 00000000000..1fa8b9089bb --- /dev/null +++ b/src/python/pants/backend/python/tasks2/python_create_distributions.py @@ -0,0 +1,84 @@ +# coding=utf-8 +# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os + +from pex.interpreter import PythonInterpreter +from pex.pex_builder import PEXBuilder +from pex.pex_info import PexInfo + +from pants.backend.python.targets.python_binary import PythonBinary +from pants.backend.python.tasks2.pex_build_util import (dump_python_distributions, + dump_requirements, dump_sources, + has_python_requirements, has_python_sources, + has_resources, is_local_python_dist) +from pants.base.build_environment import get_buildroot +from pants.base.exceptions import TaskError +from pants.build_graph.target_scopes import Scopes +from pants.task.task import Task +from pants.util.contextutil import temporary_dir +from pants.util.dirutil import safe_mkdir_for +from pants.util.fileutil import atomic_copy + + +class PythonCreateDistributions(Task): + """Create python distributions (.whl) from python_dist targets.""" + + @classmethod + def product_types(cls): + return ['python_dists'] + + @classmethod + def prepare(cls, options, round_manager): + round_manager.require_data(PythonInterpreter) + round_manager.require_data('python') # For codegen. + + @staticmethod + def is_distribution(target): + return isinstance(target, PythonDistribution) + + def __init__(self, *args, **kwargs): + super(PythonCreateDistributions, self).__init__(*args, **kwargs) + self._distdir = self.get_options().pants_distdir + + def execute(self): + dist_targets = self.context.targets(self.is_distribution) + built_dists = set() + + if dist_targets: + # Check for duplicate distribution names, since we write the pexes to /.pex. + names = {} + for dist_target in dist_targets: + name = dist_target.name + if name in names: + raise TaskError('Cannot build two dist_targets with the same name in a single invocation. ' + '{} and {} both have the name {}.'.format(dist_target, names[name], name)) + names[name] = dist_target + + with self.invalidated(dist_targets, invalidate_dependents=True) as invalidation_check: + + for vt in invalidation_check.all_vts: + pex_path = os.path.join(vt.results_dir, '{}.pex'.format(vt.target.name)) + if not vt.valid: + self.context.log.debug('cache for {} is invalid, rebuilding'.format(vt.target)) + built_dists.add(self._create_dist(vt.target)). # vt.results dir + else: + self.context.log.debug('using cache for {}'.format(vt.target)) + + self.context.products.register_data('python_dists', built_dists) + + def _create_dist(self, dist_tgt): + """Create a .whl file for the specified python_distribution target.""" + interpreter = self.context.products.get_data(PythonInterpreter) + + whl_location = '' + # build whl from python_dist target + whl = build_python_distribution(dist_tgt, interpreter, self.workdir, self.context.log) + if whl: + whl_location = whl + + return whl_location