From 180063a230b28cec68b67cd104c5a30403f74173 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Wed, 21 Nov 2018 18:11:25 -0800 Subject: [PATCH 1/7] fix: Adding wheel dependency for python_pip packager (#34) --- MANIFEST.in | 3 +-- requirements/python_pip.txt | 5 +++++ setup.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 requirements/python_pip.txt diff --git a/MANIFEST.in b/MANIFEST.in index e5b81496c..34dc6eea7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,5 @@ include LICENSE -include requirements/base.txt -include requirements/dev.txt +include requirements/* recursive-include aws_lambda_builders/workflows * prune tests diff --git a/requirements/python_pip.txt b/requirements/python_pip.txt new file mode 100644 index 000000000..b0958467c --- /dev/null +++ b/requirements/python_pip.txt @@ -0,0 +1,5 @@ + +# Following packages are required by `python_pip` workflow to run. +# TODO: Consider moving this dependency directly into the `python_pip` workflow module +setuptools +wheel \ No newline at end of file diff --git a/setup.py b/setup.py index 5824183b4..002325d8a 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ def read_version(): '{}=aws_lambda_builders.__main__:main'.format(cmd_name) ] }, - install_requires=read_requirements('base.txt'), + install_requires=read_requirements('base.txt') + read_requirements("python_pip.txt"), extras_require={ 'dev': read_requirements('dev.txt') }, From 3fcb6733149cf199252165950322adeb5ec58626 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Wed, 21 Nov 2018 18:12:16 -0800 Subject: [PATCH 2/7] fix: Adding debug logging to python_pip builder (#33) --- aws_lambda_builders/__init__.py | 2 +- .../workflows/python_pip/packager.py | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index f38463afc..f9b942ea8 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = '0.0.2' +__version__ = '0.0.3-dev' RPC_PROTOCOL_VERSION = "0.1" diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index d9c78cd74..9cda81084 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -5,6 +5,7 @@ import sys import re import subprocess +import logging from email.parser import FeedParser @@ -14,6 +15,8 @@ from .compat import pip_no_compile_c_shim from .utils import OSUtils +LOG = logging.getLogger(__name__) + # TODO update the wording here MISSING_DEPENDENCIES_TEMPLATE = r""" @@ -229,6 +232,9 @@ def _download_dependencies(self, directory, requirements_filename): else: incompatible_wheels.add(package) + LOG.debug("initial compatible: %s", compatible_wheels) + LOG.debug("initial incompatible: %s", incompatible_wheels | sdists) + # Next we need to go through the downloaded packages and pick out any # dependencies that do not have a compatible wheel file downloaded. # For these packages we need to explicitly try to download a @@ -242,6 +248,10 @@ def _download_dependencies(self, directory, requirements_filename): # file ourselves. compatible_wheels, incompatible_wheels = self._categorize_wheel_files( directory) + LOG.debug( + "compatible wheels after second download pass: %s", + compatible_wheels + ) missing_wheels = sdists - compatible_wheels self._build_sdists(missing_wheels, directory, compile_c=True) @@ -255,6 +265,10 @@ def _download_dependencies(self, directory, requirements_filename): # compiler. compatible_wheels, incompatible_wheels = self._categorize_wheel_files( directory) + LOG.debug( + "compatible after building wheels (no C compiling): %s", + compatible_wheels + ) missing_wheels = sdists - compatible_wheels self._build_sdists(missing_wheels, directory, compile_c=False) @@ -264,6 +278,10 @@ def _download_dependencies(self, directory, requirements_filename): # compatible version directly and building from source. compatible_wheels, incompatible_wheels = self._categorize_wheel_files( directory) + LOG.debug( + "compatible after building wheels (C compiling): %s", + compatible_wheels + ) # Now there is still the case left over where the setup.py has been # made in such a way to be incompatible with python's setup tools, @@ -273,6 +291,9 @@ def _download_dependencies(self, directory, requirements_filename): compatible_wheels, incompatible_wheels = self._apply_wheel_whitelist( compatible_wheels, incompatible_wheels) missing_wheels = deps - compatible_wheels + LOG.debug("Final compatible: %s", compatible_wheels) + LOG.debug("Final incompatible: %s", incompatible_wheels) + LOG.debug("Final missing wheels: %s", missing_wheels) return compatible_wheels, missing_wheels @@ -285,14 +306,18 @@ def _download_all_dependencies(self, requirements_filename, directory): self._pip.download_all_dependencies(requirements_filename, directory) deps = {Package(directory, filename) for filename in self._osutils.get_directory_contents(directory)} + LOG.debug("Full dependency closure: %s", deps) return deps def _download_binary_wheels(self, packages, directory): # Try to get binary wheels for each package that isn't compatible. + LOG.debug("Downloading missing wheels: %s", packages) self._pip.download_manylinux_wheels( [pkg.identifier for pkg in packages], directory) def _build_sdists(self, sdists, directory, compile_c=True): + LOG.debug("Build missing wheels from sdists " + "(C compiling %s): %s", compile_c, sdists) for sdist in sdists: path_to_sdist = self._osutils.joinpath(directory, sdist.filename) self._pip.build_wheel(path_to_sdist, directory, compile_c) @@ -537,6 +562,7 @@ def __init__(self, pip, osutils=None): def _execute(self, command, args, env_vars=None, shim=None): """Execute a pip command with the given arguments.""" main_args = [command] + args + LOG.debug("calling pip %s", ' '.join(main_args)) rc, out, err = self._wrapped_pip.main(main_args, env_vars=env_vars, shim=shim) return rc, out, err From 1c59f4c74154a4129248395ab8b6834b82691780 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Wed, 21 Nov 2018 18:37:17 -0800 Subject: [PATCH 3/7] feat: Python 37 support (#32) * Python 37 support * Making the ABI selection explicit * Fixing the lambda_abi to a supported list * Making tests work --- .appveyor.yml | 2 +- .travis.yml | 10 ++-- aws_lambda_builders/validate.py | 3 +- .../workflows/python_pip/actions.py | 5 +- .../workflows/python_pip/compat.py | 7 --- .../workflows/python_pip/packager.py | 54 ++++++++++++++----- .../workflows/python_pip/test_packager.py | 11 ++-- .../workflows/python_pip/test_python_pip.py | 8 ++- .../unit/workflows/python_pip/test_actions.py | 3 +- .../workflows/python_pip/test_packager.py | 26 +++++---- 10 files changed, 78 insertions(+), 51 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 6bdd2ed63..1e458eb55 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,7 +7,7 @@ environment: - PYTHON: "C:\\Python27-x64" - PYTHON: "C:\\Python36-x64" -# - PYTHON: "C:\\Python37-x64" + - PYTHON: "C:\\Python37-x64" build: off diff --git a/.travis.yml b/.travis.yml index 27091558d..62494ef7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,11 @@ python: - "2.7" - "3.6" # Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs -#matrix: -# include: -# - python: 3.7 -# dist: xenial -# sudo: true +matrix: + include: + - python: 3.7 + dist: xenial + sudo: true install: # Install the code requirements - make init diff --git a/aws_lambda_builders/validate.py b/aws_lambda_builders/validate.py index 7f4cbeb9b..025a570f2 100644 --- a/aws_lambda_builders/validate.py +++ b/aws_lambda_builders/validate.py @@ -32,7 +32,8 @@ def validate_python_cmd(required_language, required_runtime_version): class RuntimeValidator(object): SUPPORTED_RUNTIMES = [ "python2.7", - "python3.6" + "python3.6", + "python3.7", ] @classmethod diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index cd8a6590e..81d5fe981 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -17,15 +17,14 @@ def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime): self.manifest_path = manifest_path self.scratch_dir = scratch_dir self.runtime = runtime - self.package_builder = PythonPipDependencyBuilder() + self.package_builder = PythonPipDependencyBuilder(runtime=runtime) def execute(self): try: self.package_builder.build_dependencies( self.artifacts_dir, self.manifest_path, - self.scratch_dir, - self.runtime, + self.scratch_dir ) except PackagerError as ex: raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/python_pip/compat.py b/aws_lambda_builders/workflows/python_pip/compat.py index 1a08b645b..64aba2f06 100644 --- a/aws_lambda_builders/workflows/python_pip/compat.py +++ b/aws_lambda_builders/workflows/python_pip/compat.py @@ -1,5 +1,4 @@ import os -import six def pip_import_string(): @@ -101,9 +100,3 @@ def raise_compile_error(*args, **kwargs): pip_no_compile_c_env_vars = { 'CC': '/var/false' } - - -if six.PY3: - lambda_abi = 'cp36m' -else: - lambda_abi = 'cp27mu' diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 9cda81084..a3a2f0130 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -9,7 +9,6 @@ from email.parser import FeedParser -from .compat import lambda_abi from .compat import pip_import_string from .compat import pip_no_compile_c_env_vars from .compat import pip_no_compile_c_shim @@ -59,10 +58,36 @@ class PackageDownloadError(PackagerError): pass +class UnsupportedPythonVersion(PackagerError): + """Generic networking error during a package download.""" + def __init__(self, version): + super(UnsupportedPythonVersion, self).__init__( + "'%s' version of python is not supported" % version + ) + + +def get_lambda_abi(runtime): + supported = { + "python2.7": "cp27mu", + "python3.6": "cp36m", + "python3.7": "cp37m" + } + + if runtime not in supported: + raise UnsupportedPythonVersion(runtime) + + return supported[runtime] + + class PythonPipDependencyBuilder(object): - def __init__(self, osutils=None, dependency_builder=None): + def __init__(self, runtime, osutils=None, dependency_builder=None): """Initialize a PythonPipDependencyBuilder. + :type runtime: str + :param runtime: Python version to build dependencies for. This can + either be python2.7, python3.6 or python3.7. These are currently the + only supported values. + :type osutils: :class:`lambda_builders.utils.OSUtils` :param osutils: A class used for all interactions with the outside OS. @@ -76,11 +101,11 @@ def __init__(self, osutils=None, dependency_builder=None): self.osutils = OSUtils() if dependency_builder is None: - dependency_builder = DependencyBuilder(self.osutils) + dependency_builder = DependencyBuilder(self.osutils, runtime) self._dependency_builder = dependency_builder def build_dependencies(self, artifacts_dir_path, scratch_dir_path, - requirements_path, runtime, ui=None, config=None): + requirements_path, ui=None, config=None): """Builds a python project's dependencies into an artifact directory. :type artifacts_dir_path: str @@ -93,11 +118,6 @@ def build_dependencies(self, artifacts_dir_path, scratch_dir_path, :param requirements_path: Path to a requirements.txt file to inspect for a list of dependencies. - :type runtime: str - :param runtime: Python version to build dependencies for. This can - either be python2.7 or python3.6. These are currently the only - supported values. - :type ui: :class:`lambda_builders.utils.UI` or None :param ui: A class that traps all progress information such as status and errors. If injected by the caller, it can be used to monitor @@ -141,13 +161,16 @@ class DependencyBuilder(object): 'sqlalchemy' } - def __init__(self, osutils, pip_runner=None): + def __init__(self, osutils, runtime, pip_runner=None): """Initialize a DependencyBuilder. :type osutils: :class:`lambda_builders.utils.OSUtils` :param osutils: A class used for all interactions with the outside OS. + :type runtime: str + :param runtime: AWS Lambda Python runtime to build for + :type pip_runner: :class:`PipRunner` :param pip_runner: This class is responsible for executing our pip on our behalf. @@ -156,6 +179,7 @@ def __init__(self, osutils, pip_runner=None): if pip_runner is None: pip_runner = PipRunner(SubprocessPip(osutils)) self._pip = pip_runner + self.runtime = runtime def build_site_packages(self, requirements_filepath, target_directory, @@ -312,8 +336,9 @@ def _download_all_dependencies(self, requirements_filename, directory): def _download_binary_wheels(self, packages, directory): # Try to get binary wheels for each package that isn't compatible. LOG.debug("Downloading missing wheels: %s", packages) + lambda_abi = get_lambda_abi(self.runtime) self._pip.download_manylinux_wheels( - [pkg.identifier for pkg in packages], directory) + [pkg.identifier for pkg in packages], directory, lambda_abi) def _build_sdists(self, sdists, directory, compile_c=True): LOG.debug("Build missing wheels from sdists " @@ -341,6 +366,9 @@ def _is_compatible_wheel_filename(self, filename): # Verify platform is compatible if platform not in self._MANYLINUX_COMPATIBLE_PLATFORM: return False + + lambda_runtime_abi = get_lambda_abi(self.runtime) + # Verify that the ABI is compatible with lambda. Either none or the # correct type for the python version cp27mu for py27 and cp36m for # py36. @@ -351,7 +379,7 @@ def _is_compatible_wheel_filename(self, filename): # Deploying python 3 function which means we need cp36m abi # We can also accept abi3 which is the CPython 3 Stable ABI and # will work on any version of python 3. - return abi == 'cp36m' or abi == 'abi3' + return abi == lambda_runtime_abi or abi == 'abi3' elif prefix_version == 'cp2': # Deploying to python 2 function which means we need cp27mu abi return abi == 'cp27mu' @@ -615,7 +643,7 @@ def download_all_dependencies(self, requirements_filename, directory): # complain at deployment time. self.build_wheel(wheel_package_path, directory) - def download_manylinux_wheels(self, packages, directory): + def download_manylinux_wheels(self, packages, directory, lambda_abi): """Download wheel files for manylinux for all the given packages.""" # If any one of these dependencies fails pip will bail out. Since we # are only interested in all the ones we can download, we need to feed diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index f51c1b4b9..8214ff1be 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -1,3 +1,4 @@ +import sys import os import zipfile import tarfile @@ -16,7 +17,7 @@ from aws_lambda_builders.workflows.python_pip.packager import SDistMetadataFetcher from aws_lambda_builders.workflows.python_pip.packager import \ InvalidSourceDistributionNameError -from aws_lambda_builders.workflows.python_pip.compat import lambda_abi +from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_env_vars from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_shim from aws_lambda_builders.workflows.python_pip.utils import OSUtils @@ -214,7 +215,7 @@ def _write_requirements_txt(self, packages, directory): def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner): appdir = str(_create_app_structure(tmpdir)) self._write_requirements_txt(reqs, appdir) - builder = DependencyBuilder(OSUtils(), runner) + builder = DependencyBuilder(OSUtils(), "python3.6", runner) return appdir, builder def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils): @@ -644,7 +645,7 @@ def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner): expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, 'bar==1.2' ], packages=[ @@ -677,7 +678,7 @@ def test_whitelist_sqlalchemy(self, tmpdir, osutils, pip_runner): expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, 'sqlalchemy==1.1.18' ], packages=[ @@ -839,7 +840,7 @@ def test_build_into_existing_dir_with_preinstalled_packages( expected_args=[ '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', '--implementation', 'cp', - '--abi', lambda_abi, '--dest', mock.ANY, + '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY, 'foo==1.2' ], packages=[ diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index de3b67a6d..51fe6927b 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -47,11 +47,9 @@ def test_must_build_python_project(self): self.assertEquals(expected_files, output_files) def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): - self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, - runtime="python2.8") - expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"}) - output_files = set(os.listdir(self.artifacts_dir)) - self.assertEquals(expected_files, output_files) + with self.assertRaises(WorkflowFailedError): + self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, + runtime="python2.8") def test_must_fail_to_resolve_dependencies(self): diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index 208681a52..1f691efff 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -20,8 +20,7 @@ def test_action_must_call_builder(self, PythonPipDependencyBuilderMock): builder_instance.build_dependencies.assert_called_with("artifacts", "scratch_dir", - "manifest", - "runtime") + "manifest") @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder") def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock): diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 01f81a735..228735e33 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -1,4 +1,3 @@ -import sys from collections import namedtuple import mock @@ -12,6 +11,7 @@ from aws_lambda_builders.workflows.python_pip.packager import Package from aws_lambda_builders.workflows.python_pip.packager import PipRunner from aws_lambda_builders.workflows.python_pip.packager import SubprocessPip +from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi from aws_lambda_builders.workflows.python_pip.packager \ import InvalidSourceDistributionNameError from aws_lambda_builders.workflows.python_pip.packager import NoSuchPackageError @@ -85,6 +85,17 @@ def popen(self, *args, **kwargs): return self._processes.pop() +class TestGetLambdaAbi(object): + def test_get_lambda_abi_python27(self): + assert "cp27mu" == get_lambda_abi("python2.7") + + def test_get_lambda_abi_python36(self): + assert "cp36m" == get_lambda_abi("python3.6") + + def test_get_lambda_abi_python37(self): + assert "cp37m" == get_lambda_abi("python3.7") + + class TestPythonPipDependencyBuilder(object): def test_can_call_dependency_builder(self, osutils): mock_dep_builder = mock.Mock(spec=DependencyBuilder) @@ -92,10 +103,11 @@ def test_can_call_dependency_builder(self, osutils): builder = PythonPipDependencyBuilder( osutils=osutils_mock, dependency_builder=mock_dep_builder, + runtime="runtime" ) builder.build_dependencies( 'artifacts/path/', 'scratch_dir/path/', - 'path/to/requirements.txt', 'python3.6' + 'path/to/requirements.txt' ) mock_dep_builder.build_site_packages.assert_called_once_with( 'path/to/requirements.txt', 'artifacts/path/', 'scratch_dir/path/') @@ -218,14 +230,10 @@ def test_download_wheels(self, pip_factory): # for getting lambda compatible wheels. pip, runner = pip_factory() packages = ['foo', 'bar', 'baz'] - runner.download_manylinux_wheels(packages, 'directory') - if sys.version_info[0] == 2: - abi = 'cp27mu' - else: - abi = 'cp36m' + runner.download_manylinux_wheels(packages, 'directory', "abi") expected_prefix = ['download', '--only-binary=:all:', '--no-deps', '--platform', 'manylinux1_x86_64', - '--implementation', 'cp', '--abi', abi, + '--implementation', 'cp', '--abi', "abi", '--dest', 'directory'] for i, package in enumerate(packages): assert pip.calls[i].args == expected_prefix + [package] @@ -234,7 +242,7 @@ def test_download_wheels(self, pip_factory): def test_download_wheels_no_wheels(self, pip_factory): pip, runner = pip_factory() - runner.download_manylinux_wheels([], 'directory') + runner.download_manylinux_wheels([], 'directory', "abi") assert len(pip.calls) == 0 def test_raise_no_such_package_error(self, pip_factory): From 52f2a22f4cf3a7426ccf91f1139617097805a09d Mon Sep 17 00:00:00 2001 From: Joris Conijn Date: Thu, 22 Nov 2018 17:11:39 +0100 Subject: [PATCH 4/7] fix: Typo in Python_pip design document (#36) --- aws_lambda_builders/workflows/python_pip/DESIGN.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_lambda_builders/workflows/python_pip/DESIGN.md b/aws_lambda_builders/workflows/python_pip/DESIGN.md index 8b1697c4e..d619fa645 100644 --- a/aws_lambda_builders/workflows/python_pip/DESIGN.md +++ b/aws_lambda_builders/workflows/python_pip/DESIGN.md @@ -5,7 +5,7 @@ This package is an effort to port the Chalice packager to a library that can be used to handle the dependency resilution portion of packaging Python code for use in AWS Lambda. The scope for this builder is to take an existing -directory containing customer code, and a top-levle `requirements.txt` file +directory containing customer code, and a top-level `requirements.txt` file specifying third party depedencies. The builder will examine the dependencies and use pip to build and include the dependencies in the customer code bundle in a way that makes them importable. @@ -146,4 +146,4 @@ bundle has an `__init__.py` and is on the `PYTHONPATH`. The dependencies should now be succesfully installed in the target directory. All the temporary/intermediate files can now be deleting including all the -wheel files and sdists. \ No newline at end of file +wheel files and sdists. From 770818763620522aa22d4ad2057f70f32abb5643 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Mon, 26 Nov 2018 14:38:31 -0800 Subject: [PATCH 5/7] fix: astroid version compatible to be compatible with pylint1.7 (#38) --- requirements/dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements/dev.txt b/requirements/dev.txt index 32722b16d..a920601b7 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,6 +2,8 @@ coverage==4.3.4 flake8==3.3.0 tox==2.2.1 pytest-cov==2.4.0 +# astroid > 2.0.4 is not compatible with pylint1.7 +astroid>=1.5.8,<2.1.0 pylint==1.7.2 # Test requirements From cf678d648b5a327ef1d236067e52b480ffe08cd4 Mon Sep 17 00:00:00 2001 From: Sriram Madapusi Vasudevan <3770774+TheSriram@users.noreply.github.com> Date: Mon, 26 Nov 2018 15:44:31 -0800 Subject: [PATCH 6/7] fix: helpful error messages when runtime mismatch for build (#37) --- aws_lambda_builders/exceptions.py | 6 ++++-- aws_lambda_builders/validate.py | 6 ++++-- .../workflows/python_pip/test_python_pip.py | 17 ++++++++++++++++- tests/unit/test_runtime.py | 5 +++-- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/aws_lambda_builders/exceptions.py b/aws_lambda_builders/exceptions.py index f11a30f12..656763667 100644 --- a/aws_lambda_builders/exceptions.py +++ b/aws_lambda_builders/exceptions.py @@ -16,8 +16,10 @@ class UnsupportedManifestError(LambdaBuilderError): class MisMatchRuntimeError(LambdaBuilderError): - MESSAGE = "A runtime version mismatch was found for the given language " \ - "'{language}', required runtime '{required_runtime}'" + MESSAGE = "{language} executable found in your path does not " \ + "match runtime. " \ + "\n Expected version: {required_runtime}, Found version: {found_runtime}. " \ + "\n Possibly related: https://github.com/awslabs/aws-lambda-builders/issues/30" class WorkflowNotFoundError(LambdaBuilderError): diff --git a/aws_lambda_builders/validate.py b/aws_lambda_builders/validate.py index 025a570f2..82d34d3a9 100644 --- a/aws_lambda_builders/validate.py +++ b/aws_lambda_builders/validate.py @@ -17,6 +17,7 @@ def validate_python_cmd(required_language, required_runtime_version): "python", "-c", "import sys; " + "sys.stdout.write('python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)); " "assert sys.version_info.major == {major} " "and sys.version_info.minor == {minor}".format( major=major, @@ -63,10 +64,11 @@ def validate_runtime(cls, required_language, required_runtime): p = subprocess.Popen(cmd, cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - p.communicate() + found_runtime, _ = p.communicate() if p.returncode != 0: raise MisMatchRuntimeError(language=required_language, - required_runtime=required_runtime) + required_runtime=required_runtime, + found_runtime=str(found_runtime.decode('utf-8'))) else: LOG.warning("'%s' runtime has not " "been validated!", required_language) diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 51fe6927b..c4a4f4e41 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -6,7 +6,7 @@ from unittest import TestCase from aws_lambda_builders.builder import LambdaBuilder -from aws_lambda_builders.exceptions import WorkflowFailedError +from aws_lambda_builders.exceptions import WorkflowFailedError, MisMatchRuntimeError class TestPythonPipWorkflow(TestCase): @@ -33,6 +33,12 @@ def setUp(self): language=self.builder.capability.language, major=sys.version_info.major, minor=sys.version_info.minor) + self.runtime_mismatch = { + 'python3.6': 'python2.7', + 'python3.7': 'python2.7', + 'python2.7': 'python3.6' + + } def tearDown(self): shutil.rmtree(self.artifacts_dir) @@ -46,6 +52,15 @@ def test_must_build_python_project(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEquals(expected_files, output_files) + def test_mismatch_runtime_python_project(self): + with self.assertRaises(MisMatchRuntimeError) as mismatch_error: + self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, + runtime=self.runtime_mismatch[self.runtime]) + self.assertEquals(mismatch_error.msg, + MisMatchRuntimeError(language="python", + required_runtime=self.runtime_mismatch[self.runtime], + found_runtime=self.runtime).MESSAGE) + def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): with self.assertRaises(WorkflowFailedError): self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, diff --git a/tests/unit/test_runtime.py b/tests/unit/test_runtime.py index 25799806d..6f9e01168 100644 --- a/tests/unit/test_runtime.py +++ b/tests/unit/test_runtime.py @@ -13,7 +13,7 @@ def __init__(self, returncode): self.returncode = returncode def communicate(self): - pass + return b'python3,6', None class TestRuntime(TestCase): @@ -44,6 +44,7 @@ def test_runtime_validate_mismatch_version_runtime(self): def test_python_command(self): cmd = validate_python_cmd("python", "python2.7") - version_strings = ["sys.version_info.major == 2", "sys.version_info.minor == 7"] + version_strings = ["sys.stdout.write", "sys.version_info.major == 2", + "sys.version_info.minor == 7"] for version_string in version_strings: self.assertTrue(any([part for part in cmd if version_string in part])) From 3744cea731403fc5d5aad36c4f60d9512231fd78 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Mon, 26 Nov 2018 15:54:23 -0800 Subject: [PATCH 7/7] Bumping to v0.0.3 (#39) --- aws_lambda_builders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index f9b942ea8..5f38ffaaa 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = '0.0.3-dev' +__version__ = '0.0.3' RPC_PROTOCOL_VERSION = "0.1"