From 74bfb8ce01008f39d2bdc29fe834ae8dbc72a7b4 Mon Sep 17 00:00:00 2001 From: Chris Livingston Date: Tue, 21 Nov 2017 16:45:55 -0800 Subject: [PATCH] Finalized initial approach at packaging wheels w/ c sources into a pex --- .../python_distribution/hello/main/main.py | 1 - .../hello/superhello/BUILD | 3 ++ .../hello/superhello/hello_package/hello.py | 2 +- .../python/targets/python_distribution.py | 29 ++----------------- .../backend/python/tasks/pex_build_util.py | 27 ++++++++++------- 5 files changed, 23 insertions(+), 39 deletions(-) diff --git a/examples/src/python/example/python_distribution/hello/main/main.py b/examples/src/python/example/python_distribution/hello/main/main.py index 5251b3c5674..8fee83c518e 100644 --- a/examples/src/python/example/python_distribution/hello/main/main.py +++ b/examples/src/python/example/python_distribution/hello/main/main.py @@ -9,6 +9,5 @@ from hello_package import hello - if __name__ == '__main__': hello.hello() diff --git a/examples/src/python/example/python_distribution/hello/superhello/BUILD b/examples/src/python/example/python_distribution/hello/superhello/BUILD index 8d8a441a6b9..9eb88ddb6d8 100644 --- a/examples/src/python/example/python_distribution/hello/superhello/BUILD +++ b/examples/src/python/example/python_distribution/hello/superhello/BUILD @@ -3,6 +3,9 @@ # Like Hello world, but built with Pants. This time, with both C++ and Py sources +# The BUILD file must be in the directory of the Python Distribution, and there must +# only be a single BUILD file in a given directory. + python_distribution( name='superhello' ) diff --git a/examples/src/python/example/python_distribution/hello/superhello/hello_package/hello.py b/examples/src/python/example/python_distribution/hello/superhello/hello_package/hello.py index 7bb4dce3c78..fa29f9a2bbd 100644 --- a/examples/src/python/example/python_distribution/hello/superhello/hello_package/hello.py +++ b/examples/src/python/example/python_distribution/hello/superhello/hello_package/hello.py @@ -8,4 +8,4 @@ import super_greet def hello(): - print(super_greet.super_greet()) + print(super_greet.super_greet()) diff --git a/src/python/pants/backend/python/targets/python_distribution.py b/src/python/pants/backend/python/targets/python_distribution.py index 6e4d65b153e..ce07dc2ef26 100644 --- a/src/python/pants/backend/python/targets/python_distribution.py +++ b/src/python/pants/backend/python/targets/python_distribution.py @@ -25,41 +25,16 @@ class PythonDistribution(PythonTarget): def alias(cls): return 'python_distribution' - default_sources_globs = '*.(c|cpp|h|py)' - def __init__(self, - source=None, - setup_file=None, - repositories=None, - package_dir=None, platforms=(), **kwargs): payload = Payload() payload.add_fields({ - 'setup_file': PrimitiveField(setup_file), - 'repositories': PrimitiveField(maybe_list(repositories or [])), - 'platforms': PrimitiveField(tuple(maybe_list(platforms or []))), - 'package_dir': PrimitiveField(package_dir), + 'platforms': PrimitiveField(tuple(maybe_list(platforms or []))) }) - - sources = [] if source is None else [source] - super(PythonDistribution, self).__init__(sources=sources, payload=payload, **kwargs) - - @property - def setup_file(self): - return self.payload.setup_file + super(PythonDistribution, self).__init__(sources=[], payload=payload, **kwargs) @property def platforms(self): return self.payload.platforms - - @property - def repositories(self): - return self.payload.repositories - - def package_dir(self): - return self.payload.package_dir - - - diff --git a/src/python/pants/backend/python/tasks/pex_build_util.py b/src/python/pants/backend/python/tasks/pex_build_util.py index 923df56b9fd..17e89b687c5 100644 --- a/src/python/pants/backend/python/tasks/pex_build_util.py +++ b/src/python/pants/backend/python/tasks/pex_build_util.py @@ -135,8 +135,9 @@ def build_python_distribution_from_target(target, workdir): pydist_workdir = os.path.join(workdir, '.pydistworkdir') safe_mkdir(pydist_workdir) pex_name = "%s.pex" % target.name - - args = ['--disable-cache', '/Users/clivingston/workspace/pants/examples/src/python/example/python_distribution/hello/superhello', '-o', pex_name] + path_to_target = os.path.dirname(target.address.rel_path) + + args = ['--disable-cache', path_to_target, '-o', os.path.join(pydist_workdir, pex_name)] try: pex_main.main(args=args) except SystemExit as e: @@ -144,16 +145,20 @@ def build_python_distribution_from_target(target, workdir): except Exception as e: raise TaskError(e) + # unzip into a chroot within the python dist workdir zip_ref = zipfile.ZipFile(os.path.join(pydist_workdir, pex_name), 'r') - safe_mkdir(os.path.join(pydist_workdir, pex_name + '_chroot')) - zip_ref.extractall(os.path.join(pydist_workdir, pex_name + '_chroot')) + fingerprint = target.payload.fingerprint() + safe_mkdir(os.path.join(pydist_workdir, fingerprint)) + zip_ref.extractall(os.path.join(pydist_workdir, fingerprint)) zip_ref.close() - contents = os.listdir(os.path.join(pydist_workdir, pex_name + '_chroot', '.deps')) - whl_file = [wheel for wheel in contents if target.name in wheel] - whl_location = os.path.join(pydist_workdir, pex_name + '_chroot', '.deps', whl_file[0]) - - return whl_location + # read the contents from .deps of the chroot to obtain the whl location + chroot_deps_contents = os.listdir(os.path.join(pydist_workdir, fingerprint, '.deps')) + if chroot_deps_contents: + whl_dist = chroot_deps_contents[0] # TODO: find better way to grab .whl from chroot + whl_location = os.path.join(pydist_workdir, fingerprint, '.deps', whl_dist) + return whl_location + return None def dump_python_distibutions(builder, dist_targets, workdir, log): @@ -165,7 +170,9 @@ def dump_python_distibutions(builder, dist_targets, workdir, log): # build whl for target using pex wheel installer locations = set() for tgt in dist_targets_set: - locations.add(build_python_distribution_from_target(tgt, workdir)) + whl_location = build_python_distribution_from_target(tgt, workdir) + if whl_location: + locations.add(whl_location) # dump prebuilt wheels into pex builder for location in locations: