diff --git a/.appveyor.yml b/.appveyor.yml index 6bdd2ed63..1e458eb55 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,7 +7,7 @@ environment: - PYTHON: "C:\\Python27-x64" - PYTHON: "C:\\Python36-x64" -# - PYTHON: "C:\\Python37-x64" + - PYTHON: "C:\\Python37-x64" build: off diff --git a/.travis.yml b/.travis.yml index 27091558d..62494ef7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,11 @@ python: - "2.7" - "3.6" # Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs -#matrix: -# include: -# - python: 3.7 -# dist: xenial -# sudo: true +matrix: + include: + - python: 3.7 + dist: xenial + sudo: true install: # Install the code requirements - make init diff --git a/aws_lambda_builders/validate.py b/aws_lambda_builders/validate.py index 7f4cbeb9b..025a570f2 100644 --- a/aws_lambda_builders/validate.py +++ b/aws_lambda_builders/validate.py @@ -32,7 +32,8 @@ def validate_python_cmd(required_language, required_runtime_version): class RuntimeValidator(object): SUPPORTED_RUNTIMES = [ "python2.7", - "python3.6" + "python3.6", + "python3.7", ] @classmethod diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index cd8a6590e..81d5fe981 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -17,15 +17,14 @@ def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime): self.manifest_path = manifest_path self.scratch_dir = scratch_dir self.runtime = runtime - self.package_builder = PythonPipDependencyBuilder() + self.package_builder = PythonPipDependencyBuilder(runtime=runtime) def execute(self): try: self.package_builder.build_dependencies( self.artifacts_dir, self.manifest_path, - self.scratch_dir, - self.runtime, + self.scratch_dir ) except PackagerError as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/python_pip/compat.py b/aws_lambda_builders/workflows/python_pip/compat.py index 1a08b645b..64aba2f06 100644 --- a/aws_lambda_builders/workflows/python_pip/compat.py +++ b/aws_lambda_builders/workflows/python_pip/compat.py @@ -1,5 +1,4 @@ import os -import six def pip_import_string(): @@ -101,9 +100,3 @@ def raise_compile_error(*args, **kwargs): pip_no_compile_c_env_vars = { 'CC': '/var/false' } - - -if six.PY3: - lambda_abi = 'cp36m' -else: - lambda_abi = 'cp27mu' diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 9cda81084..a3a2f0130 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -9,7 +9,6 @@ from email.parser import FeedParser -from .compat import lambda_abi from .compat import pip_import_string from .compat import pip_no_compile_c_env_vars from .compat import pip_no_compile_c_shim @@ -59,10 +58,36 @@ class PackageDownloadError(PackagerError): pass +class UnsupportedPythonVersion(PackagerError): + """Generic networking error during a package download.""" + def __init__(self, version): + super(UnsupportedPythonVersion, self).__init__( + "'%s' version of python is not supported" % version + ) + + +def get_lambda_abi(runtime): + supported = { + "python2.7": "cp27mu", + "python3.6": "cp36m", + "python3.7": "cp37m" + } + + if runtime not in supported: + raise UnsupportedPythonVersion(runtime) + + return supported[runtime] + + class PythonPipDependencyBuilder(object): - def __init__(self, osutils=None, dependency_builder=None): + def __init__(self, runtime, osutils=None, dependency_builder=None): """Initialize a PythonPipDependencyBuilder. + :type runtime: str + :param runtime: Python version to build dependencies for. This can + either be python2.7, python3.6 or python3.7. These are currently the + only supported values. + :type osutils: :class:`lambda_builders.utils.OSUtils` :param osutils: A class used for all interactions with the outside OS. @@ -76,11 +101,11 @@ def __init__(self, osutils=None, dependency_builder=None): self.osutils = OSUtils() if dependency_builder is None: - dependency_builder = DependencyBuilder(self.osutils) + dependency_builder = DependencyBuilder(self.osutils, runtime) self._dependency_builder = dependency_builder def build_dependencies(self, artifacts_dir_path, scratch_dir_path, - requirements_path, runtime, ui=None, config=None): + requirements_path, ui=None, config=None): """Builds a python project's dependencies into an artifact directory. :type artifacts_dir_path: str @@ -93,11 +118,6 @@ def build_dependencies(self, artifacts_dir_path, scratch_dir_path, :param requirements_path: Path to a requirements.txt file to inspect for a list of dependencies. - :type runtime: str - :param runtime: Python version to build dependencies for. This can - either be python2.7 or python3.6. These are currently the only - supported values. - :type ui: :class:`lambda_builders.utils.UI` or None :param ui: A class that traps all progress information such as status and errors. If injected by the caller, it can be used to monitor @@ -141,13 +161,16 @@ class DependencyBuilder(object): 'sqlalchemy' } - def __init__(self, osutils, pip_runner=None): + def __init__(self, osutils, runtime, pip_runner=None): """Initialize a DependencyBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` :param osutils: A class used for all interactions with the outside OS. + :type runtime: str + :param runtime: AWS Lambda Python runtime to build for + :type pip_runner: :class:`PipRunner` :param pip_runner: This class is responsible for executing our pip on our behalf. @@ -156,6 +179,7 @@ def __init__(self, osutils, pip_runner=None): if pip_runner is None: pip_runner = PipRunner(SubprocessPip(osutils)) self._pip = pip_runner + self.runtime = runtime def build_site_packages(self, requirements_filepath, target_directory, @@ -312,8 +336,9 @@ def _download_all_dependencies(self, requirements_filename, directory): def _download_binary_wheels(self, packages, directory): # Try to get binary wheels for each package that isn't compatible. LOG.debug("Downloading missing wheels: %s", packages) + lambda_abi = get_lambda_abi(self.runtime) self._pip.download_manylinux_wheels( - [pkg.identifier for pkg in packages], directory) + [pkg.identifier for pkg in packages], directory, lambda_abi) def _build_sdists(self, sdists, directory, compile_c=True): LOG.debug("Build missing wheels from sdists " @@ -341,6 +366,9 @@ def _is_compatible_wheel_filename(self, filename): # Verify platform is compatible if platform not in self._MANYLINUX_COMPATIBLE_PLATFORM: return False + + lambda_runtime_abi = get_lambda_abi(self.runtime) + # Verify that the ABI is compatible with lambda. Either none or the # correct type for the python version cp27mu for py27 and cp36m for # py36. @@ -351,7 +379,7 @@ def _is_compatible_wheel_filename(self, filename): # Deploying python 3 function which means we need cp36m abi # We can also accept abi3 which is the CPython 3 Stable ABI and # will work on any version of python 3. - return abi == 'cp36m' or abi == 'abi3' + return abi == lambda_runtime_abi or abi == 'abi3' elif prefix_version == 'cp2': # Deploying to python 2 function which means we need cp27mu abi return abi == 'cp27mu' @@ -615,7 +643,7 @@ def download_all_dependencies(self, requirements_filename, directory): # complain at deployment time. self.build_wheel(wheel_package_path, directory) - def download_manylinux_wheels(self, packages, directory): + def download_manylinux_wheels(self, packages, directory, lambda_abi): """Download wheel files for manylinux for all the given packages.""" # If any one of these dependencies fails pip will bail out. Since we # are only interested in all the ones we can download, we need to feed diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index f51c1b4b9..8214ff1be 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -1,3 +1,4 @@ +import sys import os import zipfile import tarfile @@ -16,7 +17,7 @@ from aws_lambda_builders.workflows.python_pip.packager import SDistMetadataFetcher from aws_lambda_builders.workflows.python_pip.packager import \ InvalidSourceDistributionNameError -from aws_lambda_builders.workflows.python_pip.compat import lambda_abi +from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_env_vars from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_shim from aws_lambda_builders.workflows.python_pip.utils import OSUtils @@ -214,7 +215,7 @@ def _write_requirements_txt(self, packages, directory): def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner): appdir = str(_create_app_structure(tmpdir)) self._write_requirements_txt(reqs, appdir) - builder = DependencyBuilder(OSUtils(), runner) + builder = DependencyBuilder(OSUtils(), "python3.6", runner) return appdir, builder def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils): @@ -644,7 +645,7 @@ def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner): expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, 'bar==1.2' ], packages=[ @@ -677,7 +678,7 @@ def test_whitelist_sqlalchemy(self, tmpdir, osutils, pip_runner): expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, 'sqlalchemy==1.1.18' ], packages=[ @@ -839,7 +840,7 @@ def test_build_into_existing_dir_with_preinstalled_packages( expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, 'foo==1.2' ], packages=[ diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index de3b67a6d..51fe6927b 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -47,11 +47,9 @@ def test_must_build_python_project(self): self.assertEquals(expected_files, output_files) def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): - self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, - runtime="python2.8") - expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"}) - output_files = set(os.listdir(self.artifacts_dir)) - self.assertEquals(expected_files, output_files) + with self.assertRaises(WorkflowFailedError): + self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, + runtime="python2.8") def test_must_fail_to_resolve_dependencies(self): diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index 208681a52..1f691efff 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -20,8 +20,7 @@ def test_action_must_call_builder(self, PythonPipDependencyBuilderMock): builder_instance.build_dependencies.assert_called_with("artifacts", "scratch_dir", - "manifest", - "runtime") + "manifest") @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder") def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock): diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 01f81a735..228735e33 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -1,4 +1,3 @@ -import sys from collections import namedtuple import mock @@ -12,6 +11,7 @@ from aws_lambda_builders.workflows.python_pip.packager import Package from aws_lambda_builders.workflows.python_pip.packager import PipRunner from aws_lambda_builders.workflows.python_pip.packager import SubprocessPip +from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi from aws_lambda_builders.workflows.python_pip.packager \ import InvalidSourceDistributionNameError from aws_lambda_builders.workflows.python_pip.packager import NoSuchPackageError @@ -85,6 +85,17 @@ def popen(self, *args, **kwargs): return self._processes.pop() +class TestGetLambdaAbi(object): + def test_get_lambda_abi_python27(self): + assert "cp27mu" == get_lambda_abi("python2.7") + + def test_get_lambda_abi_python36(self): + assert "cp36m" == get_lambda_abi("python3.6") + + def test_get_lambda_abi_python37(self): + assert "cp37m" == get_lambda_abi("python3.7") + + class TestPythonPipDependencyBuilder(object): def test_can_call_dependency_builder(self, osutils): mock_dep_builder = mock.Mock(spec=DependencyBuilder) @@ -92,10 +103,11 @@ def test_can_call_dependency_builder(self, osutils): builder = PythonPipDependencyBuilder( osutils=osutils_mock, dependency_builder=mock_dep_builder, + runtime="runtime" ) builder.build_dependencies( 'artifacts/path/', 'scratch_dir/path/', - 'path/to/requirements.txt', 'python3.6' + 'path/to/requirements.txt' ) mock_dep_builder.build_site_packages.assert_called_once_with( 'path/to/requirements.txt', 'artifacts/path/', 'scratch_dir/path/') @@ -218,14 +230,10 @@ def test_download_wheels(self, pip_factory): # for getting lambda compatible wheels. pip, runner = pip_factory() packages = ['foo', 'bar', 'baz'] - runner.download_manylinux_wheels(packages, 'directory') - if sys.version_info[0] == 2: - abi = 'cp27mu' - else: - abi = 'cp36m' + runner.download_manylinux_wheels(packages, 'directory', "abi") expected_prefix = ['download', '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', - '--implementation', 'cp', '--abi', abi, + '--implementation', 'cp', '--abi', "abi", '--dest', 'directory'] for i, package in enumerate(packages): assert pip.calls[i].args == expected_prefix + [package] @@ -234,7 +242,7 @@ def test_download_wheels(self, pip_factory): def test_download_wheels_no_wheels(self, pip_factory): pip, runner = pip_factory() - runner.download_manylinux_wheels([], 'directory') + runner.download_manylinux_wheels([], 'directory', "abi") assert len(pip.calls) == 0 def test_raise_no_such_package_error(self, pip_factory):