Skip to content

Commit

Permalink
Clean up + add some docs
Browse files Browse the repository at this point in the history
  • Loading branch information
Lionel Vital authored and Chris Livingston committed Jan 30, 2018
1 parent c8a954e commit 9cd32a4
Show file tree
Hide file tree
Showing 10 changed files with 36 additions and 33 deletions.
1 change: 0 additions & 1 deletion 3rdparty/python/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,4 @@ setuptools==30.0.0
subprocess32==3.2.7
six>=1.9.0,<2
thrift>=0.9.1
tensorflow==1.4.0
wheel==0.29.0
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

# Like Hello world, but built with Pants.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
# coding=utf-8
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
unicode_literals, with_statement)

import sys

# hello_package is a python module within the superhello python_distribution
from hello_package import hello

if __name__ == '__main__':
Expand Down
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

# Like Hello world, but built with Pants. This time, with both C++ and Py sources
# Like Hello world, but built with a python_distribution.
# python_distribution allows you to use setup.py to depend on C/C++ extensions.

# The BUILD file must be in the directory of the Python Distribution, and there must
# only be a single BUILD file in a given directory.
# There must be a single BUILD file and a setup.py in a given python_distribution.

# All you need to indicate is a name and any dependencies.
# You are expected to define source locations in setup.py.
# There should be a one-to-one mapping for the name of the python_distribution
# and the name of the python_distribution's directory. In this case, it's superhello.

python_distribution(
name='superhello',
dependencies=[
'3rdparty/python:tensorflow'
]
name='superhello'
)
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

#include <Python.h>

static PyObject * super_greet(PyObject *self, PyObject *args) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
unicode_literals, with_statement)

import super_greet
import tensorflow

def hello():
print(tensorflow)
print(super_greet.super_greet())
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
with_statement)
unicode_literals, with_statement)

from setuptools import setup, find_packages
from distutils.core import Extension


c_module = Extension('super_greet', sources=['c/super_greet.c'])
c_module = Extension(str('super_greet'), sources=[str('c/super_greet.c')])

setup(
name='superhello',
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
# coding=utf-8
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
unicode_literals, with_statement)


from pex.pex_info import PexInfo
from six import string_types
from twitter.common.collections import maybe_list

from pants.backend.python.targets.python_target import PythonTarget
from pants.backend.python.targets.python_tests import PythonTests
from pants.base.payload import Payload
from pants.base.payload_field import PrimitiveField

Expand Down
27 changes: 16 additions & 11 deletions src/python/pants/backend/python/tasks/pex_build_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,12 +133,13 @@ def dump_requirements(builder, interpreter, req_libs, log, platforms=None):


def build_python_distribution_from_target(target, workdir):

pydist_workdir = os.path.join(workdir, '.pydistworkdir')
# Create a temporary working directory, pydistworkdir
pydist_workdir = os.path.join(workdir, 'pydistworkdir')
safe_mkdir(pydist_workdir)
pex_name = "%s.pex" % target.name
path_to_target = os.path.dirname(target.address.rel_path)

# disable cache to ensure reproducible runs
args = ['--disable-cache', path_to_target, '-o', os.path.join(pydist_workdir, pex_name)]
try:
pex_main.main(args=args)
Expand Down Expand Up @@ -167,26 +168,30 @@ def build_python_distribution_from_target(target, workdir):
return None


def dump_python_distibutions(builder, dist_targets, workdir, log):
# de-dup all dist targets.
dist_targets_set = OrderedSet()
for tgt in dist_targets:
dist_targets_set.add(tgt)
def dump_python_distributions(builder, dist_targets, workdir, log):
"""Dump python distribution targets into a given builder
:param builder: Dump the python_distributions into this builder.
:param dist_targets: PythonDistribution targets to dump
:param workdir: Working directory for python targets (./pantsd/python)
:param log: Use this logger.
"""

# build whl for target using pex wheel installer
locations = set()
for tgt in dist_targets_set:
for tgt in dist_targets:
whl_location = build_python_distribution_from_target(tgt, workdir)
if whl_location:
locations.add(whl_location)

# dump prebuilt wheels into pex builder
# dump wheels into pex builder
# After this block, the whls built from python_distribution should be available
# for use in the produced binary
for location in locations:
log.debug(' Dumping distribution: .../{}'.format(os.path.basename(location)))
builder.add_dist_location(location)

# by this point, the whl built from python_distribution should be available for use in the
# produced binary



def _resolve_multi(interpreter, requirements, platforms, find_links):
Expand Down
6 changes: 4 additions & 2 deletions src/python/pants/backend/python/tasks/python_binary_create.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from pex.pex_info import PexInfo

from pants.backend.python.targets.python_binary import PythonBinary
from pants.backend.python.tasks2.pex_build_util import (dump_python_distibutions, dump_requirements,
from pants.backend.python.tasks2.pex_build_util import (dump_python_distributions, dump_requirements,
dump_sources, has_python_and_c_sources,
has_python_requirements, has_python_sources,
has_resources)
Expand Down Expand Up @@ -131,8 +131,10 @@ def _create_binary(self, binary_tgt, results_dir):
for tgt in source_tgts:
dump_sources(builder, tgt, self.context.log)
dump_requirements(builder, interpreter, req_tgts, self.context.log, binary_tgt.platforms)

# Dump python_distributions, if any, into builder's chroot.
if python_dist_targets:
dump_python_distibutions(builder, python_dist_targets, self.workdir, self.context.log)
dump_python_distributions(builder, python_dist_targets, self.workdir, self.context.log)

# Build the .pex file.
pex_path = os.path.join(results_dir, '{}.pex'.format(binary_tgt.name))
Expand Down

0 comments on commit 9cd32a4

Please sign in to comment.