diff --git a/.travis.yml b/.travis.yml index c45ed5e3ed3..e4b42cf1f23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -397,9 +397,8 @@ base_build_wheels: &base_build_wheels env: - &base_build_wheels_env PREPARE_DEPLOY=1 -py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs - # Similar to the bootstrap shard, we build Linux wheels in a docker image to maximize - # compatibility. This is a Py2.7 shard, so it is not subject to #6985. +base_linux_build_wheels: &base_linux_build_wheels + # Similar to the bootstrap shard, we build Linux wheels in a docker image to maximize compatibility. <<: *travis_docker_image <<: *base_build_wheels # Callers of this anchor are expected to provide values in their `env` for @@ -423,7 +422,7 @@ py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2 <<: *py27_linux_config - <<: *py27_linux_build_wheels_no_ucs + <<: *base_linux_build_wheels <<: *native_engine_cache_config name: "Build wheels - Linux and cp27m (UCS2)" env: @@ -435,7 +434,7 @@ py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2 - CACHE_NAME=linuxwheelsbuild.ucs2 py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4 - <<: *py27_linux_build_wheels_no_ucs + <<: *base_linux_build_wheels <<: *py27_linux_test_config # `py27_linux_test_config` overrides the stage set by `base_build_wheels`, so we re-override it. stage: *test @@ -448,13 +447,25 @@ py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4 && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n" - CACHE_NAME=linuxwheelsbuild.ucs4 -py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs +py36_linux_build_wheels: &py36_linux_build_wheels + <<: *base_linux_build_wheels + <<: *py36_linux_test_config + name: "Build wheels - Linux and abi3 (Py3.6+)" + env: + - *py36_linux_test_config_env + - *base_build_wheels_env + - docker_image_name=travis_ci_py36 + - docker_run_command="./build-support/bin/check_pants_pex_abi.py cp36m + && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -3n" + - CACHE_NAME=linuxwheelsbuild.abi3 + +base_osx_build_wheels: &base_osx_build_wheels <<: *base_build_wheels osx_image: xcode8 py27_osx_build_wheels_ucs2: &py27_osx_build_wheels_ucs2 <<: *py27_osx_test_config - <<: *py27_osx_build_wheels_no_ucs + <<: *base_osx_build_wheels name: "Build wheels - OSX and cp27m (UCS2)" env: - *py27_osx_test_config_env @@ -466,7 +477,7 @@ py27_osx_build_wheels_ucs2: &py27_osx_build_wheels_ucs2 py27_osx_build_wheels_ucs4: &py27_osx_build_wheels_ucs4 <<: *py27_osx_config - <<: *py27_osx_build_wheels_no_ucs + <<: *base_osx_build_wheels <<: *native_engine_cache_config name: "Build wheels - OSX and cp27mu (UCS4)" addons: @@ -497,6 +508,18 @@ py27_osx_build_wheels_ucs4: &py27_osx_build_wheels_ucs4 - ./build-support/bin/check_pants_pex_abi.py cp27mu - RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n +py36_osx_build_wheels: &py36_osx_build_wheels + <<: *py36_osx_test_config + <<: *base_osx_build_wheels + name: "Build wheels - OSX and abi3 (Py3.6+)" + env: + - *py36_osx_test_config_env + - *base_build_wheels_env + - CACHE_NAME=osxwheelsbuild.abi3 + script: + - ./build-support/bin/check_pants_pex_abi.py cp36m + - RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -3n + # ------------------------------------------------------------------------- # Rust tests # ------------------------------------------------------------------------- @@ -720,8 +743,11 @@ matrix: - <<: *py27_linux_build_wheels_ucs2 - <<: *py27_linux_build_wheels_ucs4 + - <<: *py36_linux_build_wheels + - <<: *py27_osx_build_wheels_ucs2 - <<: *py27_osx_build_wheels_ucs4 + - <<: *py36_osx_build_wheels - <<: *py36_linux_test_config name: "Integration tests for pants - shard 0 (Py3.6 PEX)" diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh index ec50f988b19..5f85366c441 100755 --- a/build-support/bin/release.sh +++ b/build-support/bin/release.sh @@ -7,20 +7,40 @@ set -e ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd "$(git rev-parse --show-toplevel)" && pwd) source ${ROOT}/build-support/common.sh +# Note we parse some options here, but parse most at the bottom. This is due to execution order. +# If the option must be used right away, we parse at the top of the script, whereas if it +# depends on functions defined later in the script, we parse at the end. +_OPTS="hdnftcloepqw3" + +while getopts "${_OPTS}" opt; do + case ${opt} in + 3) python_three="true" ;; + *) ;; # skip over other args to be parsed later + esac +done + +# Reset opt parsing's position to start +OPTIND=0 + # Set the Python interpreter to be used for the virtualenv. Note we allow the user to # predefine this value so that they may point to a specific interpreter, e.g. 2.7.13 vs. 2.7.15. -export PY="${PY:-python2.7}" +default_interpreter="python2.7"; +if [[ "${python_three:-false}" == "true" ]]; then + default_interpreter="python3.6" +fi +export PY="${PY:-${default_interpreter}}" if ! which "${PY}" >/dev/null; then die "Python interpreter ${PY} not discoverable on your PATH." fi py_major_minor=$(${PY} -c 'import sys; print(".".join(map(str, sys.version_info[0:2])))') -if [[ "${py_major_minor}" != "2.7" ]]; then - die "Invalid interpreter. The release script requires python2.7, and you are using python${py_major_minor}." +if [[ "${py_major_minor}" != "2.7" ]] && [[ "${py_major_minor}" != "3.6" ]]; then + die "Invalid interpreter. The release script requires Python 2.7 or 3.6 (you are using ${py_major_minor})." fi # Also set PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS. We set this to the exact Python version # to resolve any potential ambiguity when multiple Python interpreters are discoverable, such as -# Python 2.7.13 vs. 2.7.15. +# Python 2.7.13 vs. 2.7.15. We must also set this when running with Python 3 to ensure +# that spawned subprocesses use Python 3. py_major_minor_patch=$(${PY} -c 'import sys; print(".".join(map(str, sys.version_info[0:3])))') export PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS="${PANTS_PYTHON_SETUP_INTERPRETER_CONSTRAINTS:-['CPython==${py_major_minor_patch}']}" @@ -55,11 +75,14 @@ function requirement() { grep "^${package}[^A-Za-z0-9]" "${ROOT}/3rdparty/python/requirements.txt" || die "Could not find requirement for ${package}" } -function run_pex27() { +function run_pex() { # TODO: Cache this in case we run pex multiple times ( PEX_VERSION="$(requirement pex | sed -e "s|pex==||")" PEX_PEX=pex27 + if [[ "${python_three:-false}" == "true" ]]; then + PEX_PEX=pex36 + fi pexdir="$(mktemp -d -t build_pex.XXXXX)" trap "rm -rf ${pexdir}" EXIT @@ -75,7 +98,12 @@ function run_pex27() { function run_packages_script() { ( cd "${ROOT}" - run_pex27 "$(requirement future)" "$(requirement beautifulsoup4)" "$(requirement configparser)" "$(requirement subprocess32)" -- "${ROOT}/src/python/pants/releases/packages.py" "$@" + args=("$@") + if [[ "${python_three:-false}" == "true" ]]; then + args=("--py3" ${args[@]}) + fi + requirements=("$(requirement future)" "$(requirement beautifulsoup4)" "$(requirement configparser)" "$(requirement subprocess32)") + run_pex "${requirements[@]}" -- "${ROOT}/src/python/pants/releases/packages.py" "${args[@]}" ) } @@ -420,12 +448,16 @@ from __future__ import print_function import sys import urllib import xml.etree.ElementTree as ET +try: + from urllib.parse import quote_plus +except ImportError: + from urllib import quote_plus root = ET.parse("${wheel_listing}") ns = {'s3': 'http://s3.amazonaws.com/doc/2006-03-01/'} for key in root.findall('s3:Contents/s3:Key', ns): # Because filenames may contain characters that have different meanings # in URLs (namely '+'), # print the key both as url-encoded and as a file path. - print('{}\t{}'.format(key.text, urllib.quote_plus(key.text))) + print('{}\t{}'.format(key.text, quote_plus(key.text))) EOF done } @@ -483,9 +515,8 @@ function fetch_and_check_prebuilt_wheels() { fi done - # N.B. For platform-specific wheels, we expect 4 wheels: {linux,osx} * {cp27m,cp27mu}. - # Once we release Python 3 wheels, we will expect 6 wheels: {linux,osx} * {cp27m,cp27mu,abi3}. - if [ "${cross_platform}" != "true" ] && [ ${#packages[@]} -ne 4 ]; then + # N.B. For platform-specific wheels, we expect 6 wheels: {linux,osx} * {cp27m,cp27mu,abi3}. + if [ "${cross_platform}" != "true" ] && [ ${#packages[@]} -ne 6 ]; then missing+=("${NAME} (expected whls for each platform: had only ${packages[@]})") continue fi @@ -523,7 +554,7 @@ function activate_twine() { } function execute_pex() { - run_pex27 \ + run_pex \ --no-build \ --no-pypi \ --disable-cache \ @@ -634,9 +665,10 @@ function usage() { echo "PyPi. Credentials are needed for this as described in the" echo "release docs: http://pantsbuild.org/release.html" echo - echo "Usage: $0 [-d] [-c] (-h|-n|-f|-t|-l|-o|-e|-p)" + echo "Usage: $0 [-d] [-c] [-3] (-h|-n|-f|-t|-l|-o|-e|-p)" echo " -d Enables debug mode (verbose output, script pauses after venv creation)" echo " -h Prints out this help message." + echo " -3 Release any non-universal wheels (i.e. pantsbuild.pants) as Python 3. Defaults to Python 2." echo " -n Performs a release dry run." echo " All package distributions will be built, installed locally in" echo " an ephemeral virtualenv and exercised to validate basic" @@ -651,7 +683,7 @@ function usage() { echo " -p Build a pex from prebuilt wheels for this release." echo " -q Build a pex which only works on the host platform, using the code as exists on disk." echo - echo "All options (except for '-d') are mutually exclusive." + echo "All options (except for '-d' and '-3') are mutually exclusive." if (( $# > 0 )); then die "$@" @@ -660,7 +692,7 @@ function usage() { fi } -while getopts "hdnftcloepqw" opt; do +while getopts "${_OPTS}" opt; do case ${opt} in h) usage ;; d) debug="true" ;; @@ -673,6 +705,7 @@ while getopts "hdnftcloepqw" opt; do p) build_pex fetch ; exit $? ;; q) build_pex build ; exit $? ;; w) list_prebuilt_wheels ; exit $? ;; + 3) ;; # already parsed at top of file *) usage "Invalid option: -${OPTARG}" ;; esac done diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 09550a74b32..3e23aac934e 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -361,9 +361,8 @@ base_build_wheels: &base_build_wheels env: - &base_build_wheels_env PREPARE_DEPLOY=1 -py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs - # Similar to the bootstrap shard, we build Linux wheels in a docker image to maximize - # compatibility. This is a Py2.7 shard, so it is not subject to #6985. +base_linux_build_wheels: &base_linux_build_wheels + # Similar to the bootstrap shard, we build Linux wheels in a docker image to maximize compatibility. <<: *travis_docker_image <<: *base_build_wheels # Callers of this anchor are expected to provide values in their `env` for @@ -376,7 +375,7 @@ py27_linux_build_wheels_no_ucs: &py27_linux_build_wheels_no_ucs py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2 <<: *py27_linux_config - <<: *py27_linux_build_wheels_no_ucs + <<: *base_linux_build_wheels <<: *native_engine_cache_config name: "Build wheels - Linux and cp27m (UCS2)" env: @@ -388,7 +387,7 @@ py27_linux_build_wheels_ucs2: &py27_linux_build_wheels_ucs2 - CACHE_NAME=linuxwheelsbuild.ucs2 py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4 - <<: *py27_linux_build_wheels_no_ucs + <<: *base_linux_build_wheels <<: *py27_linux_test_config # `py27_linux_test_config` overrides the stage set by `base_build_wheels`, so we re-override it. stage: *test @@ -401,13 +400,25 @@ py27_linux_build_wheels_ucs4: &py27_linux_build_wheels_ucs4 && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n" - CACHE_NAME=linuxwheelsbuild.ucs4 -py27_osx_build_wheels_no_ucs: &py27_osx_build_wheels_no_ucs +py36_linux_build_wheels: &py36_linux_build_wheels + <<: *base_linux_build_wheels + <<: *py36_linux_test_config + name: "Build wheels - Linux and abi3 (Py3.6+)" + env: + - *py36_linux_test_config_env + - *base_build_wheels_env + - docker_image_name=travis_ci_py36 + - docker_run_command="./build-support/bin/check_pants_pex_abi.py cp36m + && RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -3n" + - CACHE_NAME=linuxwheelsbuild.abi3 + +base_osx_build_wheels: &base_osx_build_wheels <<: *base_build_wheels osx_image: xcode8 py27_osx_build_wheels_ucs2: &py27_osx_build_wheels_ucs2 <<: *py27_osx_test_config - <<: *py27_osx_build_wheels_no_ucs + <<: *base_osx_build_wheels name: "Build wheels - OSX and cp27m (UCS2)" env: - *py27_osx_test_config_env @@ -419,7 +430,7 @@ py27_osx_build_wheels_ucs2: &py27_osx_build_wheels_ucs2 py27_osx_build_wheels_ucs4: &py27_osx_build_wheels_ucs4 <<: *py27_osx_config - <<: *py27_osx_build_wheels_no_ucs + <<: *base_osx_build_wheels <<: *native_engine_cache_config name: "Build wheels - OSX and cp27mu (UCS4)" addons: @@ -444,6 +455,18 @@ py27_osx_build_wheels_ucs4: &py27_osx_build_wheels_ucs4 - ./build-support/bin/check_pants_pex_abi.py cp27mu - RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -n +py36_osx_build_wheels: &py36_osx_build_wheels + <<: *py36_osx_test_config + <<: *base_osx_build_wheels + name: "Build wheels - OSX and abi3 (Py3.6+)" + env: + - *py36_osx_test_config_env + - *base_build_wheels_env + - CACHE_NAME=osxwheelsbuild.abi3 + script: + - ./build-support/bin/check_pants_pex_abi.py cp36m + - RUN_PANTS_FROM_PEX=1 ./build-support/bin/release.sh -3n + # ------------------------------------------------------------------------- # Rust tests # ------------------------------------------------------------------------- @@ -667,8 +690,11 @@ matrix: - <<: *py27_linux_build_wheels_ucs2 - <<: *py27_linux_build_wheels_ucs4 + - <<: *py36_linux_build_wheels + - <<: *py27_osx_build_wheels_ucs2 - <<: *py27_osx_build_wheels_ucs4 + - <<: *py36_osx_build_wheels {{#py3_integration_shards}} - <<: *py36_linux_test_config diff --git a/src/docs/release.md b/src/docs/release.md index ab2518b7c85..cf21c217d50 100644 --- a/src/docs/release.md +++ b/src/docs/release.md @@ -156,7 +156,7 @@ is not required. 3. Publish to PyPi ------------------ -Once the four "Build wheels" Travis shards have completed for your release +Once the six "Build wheels" Travis shards have completed for your release commit, you can publish to PyPi. First, ensure that you are on your release branch at your version bump commit. Then, publish the release: diff --git a/src/python/pants/backend/python/pants_requirement.py b/src/python/pants/backend/python/pants_requirement.py index f71d8ef2d5e..dcb9ebabde2 100644 --- a/src/python/pants/backend/python/pants_requirement.py +++ b/src/python/pants/backend/python/pants_requirement.py @@ -50,14 +50,7 @@ def __call__(self, name=None, dist=None): msg='The {} target only works for pantsbuild.pants ' 'distributions, given {}'.format(self.alias, dist)) - # Update the environment marker in lockstep with other changes as described in - # https://github.com/pantsbuild/pants/issues/6450 - env_marker = "python_version>='2.7' and python_version<'3'" - - requirement = PythonRequirement(requirement="{key}=={version} ; {env_marker}" - .format(key=dist, - version=pants_version(), - env_marker=env_marker)) + requirement = PythonRequirement(requirement="{key}=={version}".format(key=dist, version=pants_version())) self._parse_context.create_object('python_requirement_library', name=name, diff --git a/src/python/pants/releases/packages.py b/src/python/pants/releases/packages.py index 710324e0f55..9a47ef7e540 100644 --- a/src/python/pants/releases/packages.py +++ b/src/python/pants/releases/packages.py @@ -14,11 +14,15 @@ from distutils.util import get_platform from functools import total_ordering -import subprocess32 as subprocess from bs4 import BeautifulSoup from future.moves.urllib.error import HTTPError from future.moves.urllib.request import Request, urlopen +from future.utils import PY2 +if PY2: + import subprocess32 as subprocess +else: + import subprocess COLOR_BLUE = "\x1b[34m" COLOR_RESET = "\x1b[0m" @@ -34,9 +38,7 @@ class Package(object): def __init__(self, name, target, bdist_wheel_flags=None): self.name = name self.target = target - # Update the --python-tag default in lockstep with other changes as described in - # https://github.com/pantsbuild/pants/issues/6450 - self.bdist_wheel_flags = bdist_wheel_flags or ("--python-tag", "py27") + self.bdist_wheel_flags = bdist_wheel_flags or ("--python-tag", "py27.py36.py37") def __lt__(self, other): return self.name < other.name @@ -89,14 +91,19 @@ def find_platform_name(): return get_platform().replace("-", "_").replace(".", "_") -core_packages = { - Package( - "pantsbuild.pants", - "//src/python/pants:pants-packaged", - bdist_wheel_flags=("--python-tag", "cp27", "--plat-name", find_platform_name()), - ), - Package("pantsbuild.pants.testinfra", "//tests/python/pants_test:test_infra"), -} +def core_packages(py3): + # N.B. When releasing with Python 3, we constrain the ABI (Application Binary Interface) to cp36 to allow + # pantsbuild.pants to work with any Python 3 version>= 3.6. We are able to get this future compatibility by + # specifing `abi3`, which signifies any version >= 3.6 must work. This is possible to set because in + # `src/rust/engine/src/cffi/native_engine.c` we set up `Py_LIMITED_API` and in `src/python/pants/BUILD` we + # set ext_modules, which together allows us to mark the abi tag. See https://docs.python.org/3/c-api/stable.html + # for documentation and https://bitbucket.org/pypa/wheel/commits/1f63b534d74b00e8c2e8809f07914f6da4502490?at=default#Ldocs/index.rstT121 + # for how to mark the ABI through bdist_wheel. + bdist_wheel_flags = ("--py-limited-api", "cp36") if py3 else ("--python-tag", "cp27", "--plat-name", find_platform_name()) + return { + Package("pantsbuild.pants", "//src/python/pants:pants-packaged", bdist_wheel_flags=bdist_wheel_flags), + Package("pantsbuild.pants.testinfra", "//tests/python/pants_test:test_infra"), + } def contrib_packages(): @@ -177,13 +184,13 @@ def contrib_packages(): } -def all_packages(): - return core_packages.union(contrib_packages()) +def all_packages(py3): + return core_packages(py3).union(contrib_packages()) -def build_and_print_packages(version): +def build_and_print_packages(version, py3=False): packages_by_flags = defaultdict(list) - for package in sorted(all_packages()): + for package in sorted(all_packages(py3)): packages_by_flags[package.bdist_wheel_flags].append(package) for (flags, packages) in packages_by_flags.items(): @@ -210,9 +217,9 @@ def get_pypi_config(section, option): return config.get(section, option) -def check_ownership(users, minimum_owner_count=3): +def check_ownership(users, minimum_owner_count=3, py3=False): minimum_owner_count = max(len(users), minimum_owner_count) - packages = sorted(all_packages()) + packages = sorted(all_packages(py3)) banner("Checking package ownership for {} packages".format(len(packages))) users = {user.lower() for user in users} insufficient = set() @@ -248,6 +255,9 @@ def check_ownership(i, package): def _create_parser(): parser = argparse.ArgumentParser() + # Note because of how argparse handles subparsers, the --py3 flag must be passed before any of the subparser + # flags to resolve properly. + parser.add_argument("-3", "--py3", action="store_true", default=False, help="Release any non-universal packages as Python 3.") subparsers = parser.add_subparsers(dest="command") # list parser_list = subparsers.add_parser('list') @@ -268,11 +278,11 @@ def _create_parser(): if args.with_packages: print('\n'.join( '{} {} {}'.format(package.name, package.target, " ".join(package.bdist_wheel_flags)) - for package in sorted(all_packages()))) + for package in sorted(all_packages(args.py3)))) else: - print('\n'.join(package.name for package in sorted(all_packages()))) + print('\n'.join(package.name for package in sorted(all_packages(args.py3)))) elif args.command == "list-owners": - for package in sorted(all_packages()): + for package in sorted(all_packages(args.py3)): if not package.exists(): print("The {} package is new! There are no owners yet.".format(package.name), file=sys.stderr) continue @@ -281,8 +291,8 @@ def _create_parser(): print("{}".format(owner)) elif args.command == "check-my-ownership": me = get_pypi_config('server-login', 'username') - check_ownership({me}) + check_ownership({me}, py3=args.py3) elif args.command == "build_and_print": - build_and_print_packages(args.version) + build_and_print_packages(args.version, py3=args.py3) else: raise argparse.ArgumentError("Didn't recognise arguments {}".format(args)) diff --git a/tests/python/pants_test/backend/python/test_pants_requirement.py b/tests/python/pants_test/backend/python/test_pants_requirement.py index d81e4c0fe59..e21d09a96ae 100644 --- a/tests/python/pants_test/backend/python/test_pants_requirement.py +++ b/tests/python/pants_test/backend/python/test_pants_requirement.py @@ -31,15 +31,6 @@ def key(python_requirement): self.assertEqual([key(expected)], [key(pr) for pr in python_requirement_library.payload.requirements]) - req = list(python_requirement_library.payload.requirements)[0] - self.assertIsNotNone(req.requirement.marker) - - def evaluate_version(version): - return req.requirement.marker.evaluate({'python_version': version}) - - self.assertTrue(evaluate_version('2.7')) - self.assertFalse(all(evaluate_version(v) for v in ('2.6', '3.4', '3.5', '3.6', '3.7'))) - def test_default_name(self): self.add_to_build_file('3rdparty/python/pants', 'pants_requirement()')