From f1e3dd75edd3f62c9164240141710bdd75f20ab3 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Wed, 21 Nov 2018 15:57:00 -0800 Subject: [PATCH 1/4] Python 37 support --- .appveyor.yml | 2 +- .travis.yml | 10 +++++----- aws_lambda_builders/validate.py | 3 ++- aws_lambda_builders/workflows/python_pip/compat.py | 4 +++- aws_lambda_builders/workflows/python_pip/packager.py | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 6bdd2ed63..1e458eb55 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,7 +7,7 @@ environment: - PYTHON: "C:\\Python27-x64" - PYTHON: "C:\\Python36-x64" -# - PYTHON: "C:\\Python37-x64" + - PYTHON: "C:\\Python37-x64" build: off diff --git a/.travis.yml b/.travis.yml index 27091558d..62494ef7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,11 @@ python: - "2.7" - "3.6" # Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs -#matrix: -# include: -# - python: 3.7 -# dist: xenial -# sudo: true +matrix: + include: + - python: 3.7 + dist: xenial + sudo: true install: # Install the code requirements - make init diff --git a/aws_lambda_builders/validate.py b/aws_lambda_builders/validate.py index 7f4cbeb9b..025a570f2 100644 --- a/aws_lambda_builders/validate.py +++ b/aws_lambda_builders/validate.py @@ -32,7 +32,8 @@ def validate_python_cmd(required_language, required_runtime_version): class RuntimeValidator(object): SUPPORTED_RUNTIMES = [ "python2.7", - "python3.6" + "python3.6", + "python3.7", ] @classmethod diff --git a/aws_lambda_builders/workflows/python_pip/compat.py b/aws_lambda_builders/workflows/python_pip/compat.py index 1a08b645b..c25f32deb 100644 --- a/aws_lambda_builders/workflows/python_pip/compat.py +++ b/aws_lambda_builders/workflows/python_pip/compat.py @@ -1,5 +1,6 @@ import os import six +import sys def pip_import_string(): @@ -104,6 +105,7 @@ def raise_compile_error(*args, **kwargs): if six.PY3: - lambda_abi = 'cp36m' + # cp37m or cp36m + lambda_abi = 'cp{}{}m'.format(sys.version_info.major, sys.version_info.minor) else: lambda_abi = 'cp27mu' diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index d9c78cd74..e6cc7041f 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -326,7 +326,7 @@ def _is_compatible_wheel_filename(self, filename): # Deploying python 3 function which means we need cp36m abi # We can also accept abi3 which is the CPython 3 Stable ABI and # will work on any version of python 3. - return abi == 'cp36m' or abi == 'abi3' + return abi == lambda_abi or abi == 'abi3' elif prefix_version == 'cp2': # Deploying to python 2 function which means we need cp27mu abi return abi == 'cp27mu' From f1d6d065f22c004e2329d08797c212d66a3ff32a Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Wed, 21 Nov 2018 17:13:42 -0800 Subject: [PATCH 2/4] Making the ABI selection explicit --- .../workflows/python_pip/compat.py | 9 ------- .../workflows/python_pip/packager.py | 26 ++++++++++++++----- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/aws_lambda_builders/workflows/python_pip/compat.py b/aws_lambda_builders/workflows/python_pip/compat.py index c25f32deb..64aba2f06 100644 --- a/aws_lambda_builders/workflows/python_pip/compat.py +++ b/aws_lambda_builders/workflows/python_pip/compat.py @@ -1,6 +1,4 @@ import os -import six -import sys def pip_import_string(): @@ -102,10 +100,3 @@ def raise_compile_error(*args, **kwargs): pip_no_compile_c_env_vars = { 'CC': '/var/false' } - - -if six.PY3: - # cp37m or cp36m - lambda_abi = 'cp{}{}m'.format(sys.version_info.major, sys.version_info.minor) -else: - lambda_abi = 'cp27mu' diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index e6cc7041f..5ecbd12db 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -8,7 +8,6 @@ from email.parser import FeedParser -from .compat import lambda_abi from .compat import pip_import_string from .compat import pip_no_compile_c_env_vars from .compat import pip_no_compile_c_shim @@ -56,6 +55,14 @@ class PackageDownloadError(PackagerError): pass +def get_lambda_abi(runtime): + return { + "python2.7": "cp27mu", + "python3.6": "cp36m", + "python3.7": "cp37m" + }.get(runtime) + + class PythonPipDependencyBuilder(object): def __init__(self, osutils=None, dependency_builder=None): """Initialize a PythonPipDependencyBuilder. @@ -92,8 +99,8 @@ def build_dependencies(self, artifacts_dir_path, scratch_dir_path, :type runtime: str :param runtime: Python version to build dependencies for. This can - either be python2.7 or python3.6. These are currently the only - supported values. + either be python2.7, python3.6 or python3.7. These are currently the + only supported values. :type ui: :class:`lambda_builders.utils.UI` or None :param ui: A class that traps all progress information such as status @@ -138,7 +145,7 @@ class DependencyBuilder(object): 'sqlalchemy' } - def __init__(self, osutils, pip_runner=None): + def __init__(self, osutils, pip_runner=None, runtime="python2.7"): """Initialize a DependencyBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` @@ -148,11 +155,16 @@ def __init__(self, osutils, pip_runner=None): :type pip_runner: :class:`PipRunner` :param pip_runner: This class is responsible for executing our pip on our behalf. + + :type runtime: str + :param runtime: AWS Lambda Python runtime to build for. Defaults to Python2.7 + """ self._osutils = osutils if pip_runner is None: pip_runner = PipRunner(SubprocessPip(osutils)) self._pip = pip_runner + self.lambda_abi = get_lambda_abi(runtime) def build_site_packages(self, requirements_filepath, target_directory, @@ -290,7 +302,7 @@ def _download_all_dependencies(self, requirements_filename, directory): def _download_binary_wheels(self, packages, directory): # Try to get binary wheels for each package that isn't compatible. self._pip.download_manylinux_wheels( - [pkg.identifier for pkg in packages], directory) + [pkg.identifier for pkg in packages], directory, self.lambda_abi) def _build_sdists(self, sdists, directory, compile_c=True): for sdist in sdists: @@ -326,7 +338,7 @@ def _is_compatible_wheel_filename(self, filename): # Deploying python 3 function which means we need cp36m abi # We can also accept abi3 which is the CPython 3 Stable ABI and # will work on any version of python 3. - return abi == lambda_abi or abi == 'abi3' + return abi == self.lambda_abi or abi == 'abi3' elif prefix_version == 'cp2': # Deploying to python 2 function which means we need cp27mu abi return abi == 'cp27mu' @@ -589,7 +601,7 @@ def download_all_dependencies(self, requirements_filename, directory): # complain at deployment time. self.build_wheel(wheel_package_path, directory) - def download_manylinux_wheels(self, packages, directory): + def download_manylinux_wheels(self, packages, directory, lambda_abi): """Download wheel files for manylinux for all the given packages.""" # If any one of these dependencies fails pip will bail out. Since we # are only interested in all the ones we can download, we need to feed From 0ea62bc8e91ddb7c19bc38b0be3631ea9637e872 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Wed, 21 Nov 2018 17:51:24 -0800 Subject: [PATCH 3/4] Fixing the lambda_abi to a supported list --- .../workflows/python_pip/actions.py | 5 +- .../workflows/python_pip/packager.py | 52 ++++++++++++------- .../workflows/python_pip/test_packager.py | 12 +++-- .../workflows/python_pip/test_python_pip.py | 8 ++- .../unit/workflows/python_pip/test_actions.py | 3 +- .../workflows/python_pip/test_packager.py | 25 ++++++--- 6 files changed, 64 insertions(+), 41 deletions(-) diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index cd8a6590e..81d5fe981 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -17,15 +17,14 @@ def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime): self.manifest_path = manifest_path self.scratch_dir = scratch_dir self.runtime = runtime - self.package_builder = PythonPipDependencyBuilder() + self.package_builder = PythonPipDependencyBuilder(runtime=runtime) def execute(self): try: self.package_builder.build_dependencies( self.artifacts_dir, self.manifest_path, - self.scratch_dir, - self.runtime, + self.scratch_dir ) except PackagerError as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 5ecbd12db..caa541471 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -55,18 +55,36 @@ class PackageDownloadError(PackagerError): pass +class UnsupportedPythonVersion(PackagerError): + """Generic networking error during a package download.""" + def __init__(self, version): + super(UnsupportedPythonVersion, self).__init__( + "'%s' version of python is not supported" % version + ) + + def get_lambda_abi(runtime): - return { + supported = { "python2.7": "cp27mu", "python3.6": "cp36m", "python3.7": "cp37m" - }.get(runtime) + } + + if runtime not in supported: + raise UnsupportedPythonVersion(runtime) + + return supported[runtime] class PythonPipDependencyBuilder(object): - def __init__(self, osutils=None, dependency_builder=None): + def __init__(self, runtime, osutils=None, dependency_builder=None): """Initialize a PythonPipDependencyBuilder. + :type runtime: str + :param runtime: Python version to build dependencies for. This can + either be python2.7, python3.6 or python3.7. These are currently the + only supported values. + :type osutils: :class:`lambda_builders.utils.OSUtils` :param osutils: A class used for all interactions with the outside OS. @@ -80,11 +98,11 @@ def __init__(self, osutils=None, dependency_builder=None): self.osutils = OSUtils() if dependency_builder is None: - dependency_builder = DependencyBuilder(self.osutils) + dependency_builder = DependencyBuilder(self.osutils, runtime) self._dependency_builder = dependency_builder def build_dependencies(self, artifacts_dir_path, scratch_dir_path, - requirements_path, runtime, ui=None, config=None): + requirements_path, ui=None, config=None): """Builds a python project's dependencies into an artifact directory. :type artifacts_dir_path: str @@ -97,11 +115,6 @@ def build_dependencies(self, artifacts_dir_path, scratch_dir_path, :param requirements_path: Path to a requirements.txt file to inspect for a list of dependencies. - :type runtime: str - :param runtime: Python version to build dependencies for. This can - either be python2.7, python3.6 or python3.7. These are currently the - only supported values. - :type ui: :class:`lambda_builders.utils.UI` or None :param ui: A class that traps all progress information such as status and errors. If injected by the caller, it can be used to monitor @@ -145,26 +158,25 @@ class DependencyBuilder(object): 'sqlalchemy' } - def __init__(self, osutils, pip_runner=None, runtime="python2.7"): + def __init__(self, osutils, runtime, pip_runner=None): """Initialize a DependencyBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` :param osutils: A class used for all interactions with the outside OS. + :type runtime: str + :param runtime: AWS Lambda Python runtime to build for + :type pip_runner: :class:`PipRunner` :param pip_runner: This class is responsible for executing our pip on our behalf. - - :type runtime: str - :param runtime: AWS Lambda Python runtime to build for. Defaults to Python2.7 - """ self._osutils = osutils if pip_runner is None: pip_runner = PipRunner(SubprocessPip(osutils)) self._pip = pip_runner - self.lambda_abi = get_lambda_abi(runtime) + self.runtime = runtime def build_site_packages(self, requirements_filepath, target_directory, @@ -301,8 +313,9 @@ def _download_all_dependencies(self, requirements_filename, directory): def _download_binary_wheels(self, packages, directory): # Try to get binary wheels for each package that isn't compatible. + lambda_abi = get_lambda_abi(self.runtime) self._pip.download_manylinux_wheels( - [pkg.identifier for pkg in packages], directory, self.lambda_abi) + [pkg.identifier for pkg in packages], directory, lambda_abi) def _build_sdists(self, sdists, directory, compile_c=True): for sdist in sdists: @@ -328,6 +341,9 @@ def _is_compatible_wheel_filename(self, filename): # Verify platform is compatible if platform not in self._MANYLINUX_COMPATIBLE_PLATFORM: return False + + lambda_runtime_abi = get_lambda_abi(self.runtime) + # Verify that the ABI is compatible with lambda. Either none or the # correct type for the python version cp27mu for py27 and cp36m for # py36. @@ -338,7 +354,7 @@ def _is_compatible_wheel_filename(self, filename): # Deploying python 3 function which means we need cp36m abi # We can also accept abi3 which is the CPython 3 Stable ABI and # will work on any version of python 3. - return abi == self.lambda_abi or abi == 'abi3' + return abi == lambda_runtime_abi or abi == 'abi3' elif prefix_version == 'cp2': # Deploying to python 2 function which means we need cp27mu abi return abi == 'cp27mu' diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index f51c1b4b9..e86ca8979 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -1,3 +1,4 @@ +import sys import os import zipfile import tarfile @@ -16,7 +17,7 @@ from aws_lambda_builders.workflows.python_pip.packager import SDistMetadataFetcher from aws_lambda_builders.workflows.python_pip.packager import \ InvalidSourceDistributionNameError -from aws_lambda_builders.workflows.python_pip.compat import lambda_abi +from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_env_vars from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_shim from aws_lambda_builders.workflows.python_pip.utils import OSUtils @@ -214,7 +215,8 @@ def _write_requirements_txt(self, packages, directory): def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner): appdir = str(_create_app_structure(tmpdir)) self._write_requirements_txt(reqs, appdir) - builder = DependencyBuilder(OSUtils(), runner) + runtime = "python{}.{}".format(sys.version_info.major, sys.version_info.minor) + builder = DependencyBuilder(OSUtils(), runtime, runner) return appdir, builder def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils): @@ -644,7 +646,7 @@ def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner): expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, 'bar==1.2' ], packages=[ @@ -677,7 +679,7 @@ def test_whitelist_sqlalchemy(self, tmpdir, osutils, pip_runner): expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, 'sqlalchemy==1.1.18' ], packages=[ @@ -839,7 +841,7 @@ def test_build_into_existing_dir_with_preinstalled_packages( expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, 'foo==1.2' ], packages=[ diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index de3b67a6d..68838b5cc 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -47,11 +47,9 @@ def test_must_build_python_project(self): self.assertEquals(expected_files, output_files) def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): - self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, - runtime="python2.8") - expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"}) - output_files = set(os.listdir(self.artifacts_dir)) - self.assertEquals(expected_files, output_files) + with self.assertRaises(WorkflowFailedError) as ctx: + self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, + runtime="python2.8") def test_must_fail_to_resolve_dependencies(self): diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index 208681a52..1f691efff 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -20,8 +20,7 @@ def test_action_must_call_builder(self, PythonPipDependencyBuilderMock): builder_instance.build_dependencies.assert_called_with("artifacts", "scratch_dir", - "manifest", - "runtime") + "manifest") @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder") def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock): diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 01f81a735..df8a760f9 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -12,6 +12,7 @@ from aws_lambda_builders.workflows.python_pip.packager import Package from aws_lambda_builders.workflows.python_pip.packager import PipRunner from aws_lambda_builders.workflows.python_pip.packager import SubprocessPip +from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi from aws_lambda_builders.workflows.python_pip.packager \ import InvalidSourceDistributionNameError from aws_lambda_builders.workflows.python_pip.packager import NoSuchPackageError @@ -85,6 +86,17 @@ def popen(self, *args, **kwargs): return self._processes.pop() +class TestGetLambdaAbi(object): + def test_get_lambda_abi_python27(self): + assert "cp27mu" == get_lambda_abi("python2.7") + + def test_get_lambda_abi_python36(self): + assert "cp36m" == get_lambda_abi("python3.6") + + def test_get_lambda_abi_python37(self): + assert "cp37m" == get_lambda_abi("python3.7") + + class TestPythonPipDependencyBuilder(object): def test_can_call_dependency_builder(self, osutils): mock_dep_builder = mock.Mock(spec=DependencyBuilder) @@ -92,10 +104,11 @@ def test_can_call_dependency_builder(self, osutils): builder = PythonPipDependencyBuilder( osutils=osutils_mock, dependency_builder=mock_dep_builder, + runtime="runtime" ) builder.build_dependencies( 'artifacts/path/', 'scratch_dir/path/', - 'path/to/requirements.txt', 'python3.6' + 'path/to/requirements.txt' ) mock_dep_builder.build_site_packages.assert_called_once_with( 'path/to/requirements.txt', 'artifacts/path/', 'scratch_dir/path/') @@ -218,14 +231,10 @@ def test_download_wheels(self, pip_factory): # for getting lambda compatible wheels. pip, runner = pip_factory() packages = ['foo', 'bar', 'baz'] - runner.download_manylinux_wheels(packages, 'directory') - if sys.version_info[0] == 2: - abi = 'cp27mu' - else: - abi = 'cp36m' + runner.download_manylinux_wheels(packages, 'directory', "abi") expected_prefix = ['download', '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', - '--implementation', 'cp', '--abi', abi, + '--implementation', 'cp', '--abi', "abi", '--dest', 'directory'] for i, package in enumerate(packages): assert pip.calls[i].args == expected_prefix + [package] @@ -234,7 +243,7 @@ def test_download_wheels(self, pip_factory): def test_download_wheels_no_wheels(self, pip_factory): pip, runner = pip_factory() - runner.download_manylinux_wheels([], 'directory') + runner.download_manylinux_wheels([], 'directory', "abi") assert len(pip.calls) == 0 def test_raise_no_such_package_error(self, pip_factory): From a74033214a0945f3be4297b20827258707500a5d Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Wed, 21 Nov 2018 18:07:01 -0800 Subject: [PATCH 4/4] Making tests work --- tests/functional/workflows/python_pip/test_packager.py | 3 +-- tests/integration/workflows/python_pip/test_python_pip.py | 2 +- tests/unit/workflows/python_pip/test_packager.py | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index e86ca8979..8214ff1be 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -215,8 +215,7 @@ def _write_requirements_txt(self, packages, directory): def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner): appdir = str(_create_app_structure(tmpdir)) self._write_requirements_txt(reqs, appdir) - runtime = "python{}.{}".format(sys.version_info.major, sys.version_info.minor) - builder = DependencyBuilder(OSUtils(), runtime, runner) + builder = DependencyBuilder(OSUtils(), "python3.6", runner) return appdir, builder def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils): diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 68838b5cc..51fe6927b 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -47,7 +47,7 @@ def test_must_build_python_project(self): self.assertEquals(expected_files, output_files) def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): - with self.assertRaises(WorkflowFailedError) as ctx: + with self.assertRaises(WorkflowFailedError): self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime="python2.8") diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index df8a760f9..228735e33 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -1,4 +1,3 @@ -import sys from collections import namedtuple import mock