Revert "feat: Python 37 support (#32)"

This reverts commit 624b6c3.

Author: sanathkr

.appveyor.yml

@@ -7,7 +7,7 @@ environment:
 
   - PYTHON: "C:\\Python27-x64"
   - PYTHON: "C:\\Python36-x64"
-  - PYTHON: "C:\\Python37-x64"
+#  - PYTHON: "C:\\Python37-x64"
 
 
 build: off

.travis.yml

@@ -5,11 +5,11 @@ python:
   - "2.7"
   - "3.6"
 # Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs
-matrix:
-  include:
-    - python: 3.7
-      dist: xenial
-      sudo: true
+#matrix:
+#  include:
+#    - python: 3.7
+#      dist: xenial
+#      sudo: true
 install:
   # Install the code requirements
   - make init

aws_lambda_builders/validate.py

@@ -32,8 +32,7 @@ def validate_python_cmd(required_language, required_runtime_version):
 class RuntimeValidator(object):
     SUPPORTED_RUNTIMES = [
         "python2.7",
-        "python3.6",
-        "python3.7",
+        "python3.6"
     ]
 
     @classmethod

aws_lambda_builders/workflows/python_pip/actions.py

@@ -17,14 +17,15 @@ def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime):
         self.manifest_path = manifest_path
         self.scratch_dir = scratch_dir
         self.runtime = runtime
-        self.package_builder = PythonPipDependencyBuilder(runtime=runtime)
+        self.package_builder = PythonPipDependencyBuilder()
 
     def execute(self):
         try:
             self.package_builder.build_dependencies(
                 self.artifacts_dir,
                 self.manifest_path,
-                self.scratch_dir
+                self.scratch_dir,
+                self.runtime,
             )
         except PackagerError as ex:
             raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/python_pip/compat.py

@@ -1,4 +1,5 @@
 import os
+import six
 
 
 def pip_import_string():
@@ -100,3 +101,9 @@ def raise_compile_error(*args, **kwargs):
     pip_no_compile_c_env_vars = {
         'CC': '/var/false'
     }
+
+
+if six.PY3:
+    lambda_abi = 'cp36m'
+else:
+    lambda_abi = 'cp27mu'

aws_lambda_builders/workflows/python_pip/packager.py

@@ -9,6 +9,7 @@
 from email.parser import FeedParser
 
 
+from .compat import lambda_abi
 from .compat import pip_import_string
 from .compat import pip_no_compile_c_env_vars
 from .compat import pip_no_compile_c_shim
@@ -58,36 +59,10 @@ class PackageDownloadError(PackagerError):
     pass
 
 
-class UnsupportedPythonVersion(PackagerError):
-    """Generic networking error during a package download."""
-    def __init__(self, version):
-        super(UnsupportedPythonVersion, self).__init__(
-            "'%s' version of python is not supported" % version
-        )
-
-
-def get_lambda_abi(runtime):
-    supported = {
-        "python2.7": "cp27mu",
-        "python3.6": "cp36m",
-        "python3.7": "cp37m"
-    }
-
-    if runtime not in supported:
-        raise UnsupportedPythonVersion(runtime)
-
-    return supported[runtime]
-
-
 class PythonPipDependencyBuilder(object):
-    def __init__(self, runtime, osutils=None, dependency_builder=None):
+    def __init__(self, osutils=None, dependency_builder=None):
         """Initialize a PythonPipDependencyBuilder.
 
-        :type runtime: str
-        :param runtime: Python version to build dependencies for. This can
-            either be python2.7, python3.6 or python3.7. These are currently the
-            only supported values.
-
         :type osutils: :class:`lambda_builders.utils.OSUtils`
         :param osutils: A class used for all interactions with the
             outside OS.
@@ -101,11 +76,11 @@ def __init__(self, runtime, osutils=None, dependency_builder=None):
             self.osutils = OSUtils()
 
         if dependency_builder is None:
-            dependency_builder = DependencyBuilder(self.osutils, runtime)
+            dependency_builder = DependencyBuilder(self.osutils)
         self._dependency_builder = dependency_builder
 
     def build_dependencies(self, artifacts_dir_path, scratch_dir_path,
-                           requirements_path, ui=None, config=None):
+                           requirements_path, runtime, ui=None, config=None):
         """Builds a python project's dependencies into an artifact directory.
 
         :type artifacts_dir_path: str
@@ -118,6 +93,11 @@ def build_dependencies(self, artifacts_dir_path, scratch_dir_path,
         :param requirements_path: Path to a requirements.txt file to inspect
             for a list of dependencies.
 
+        :type runtime: str
+        :param runtime: Python version to build dependencies for. This can
+            either be python2.7 or python3.6. These are currently the only
+            supported values.
+
         :type ui: :class:`lambda_builders.utils.UI` or None
         :param ui: A class that traps all progress information such as status
             and errors. If injected by the caller, it can be used to monitor
@@ -161,16 +141,13 @@ class DependencyBuilder(object):
         'sqlalchemy'
     }
 
-    def __init__(self, osutils, runtime, pip_runner=None):
+    def __init__(self, osutils, pip_runner=None):
         """Initialize a DependencyBuilder.
 
         :type osutils: :class:`lambda_builders.utils.OSUtils`
         :param osutils: A class used for all interactions with the
             outside OS.
 
-        :type runtime: str
-        :param runtime: AWS Lambda Python runtime to build for
-
         :type pip_runner: :class:`PipRunner`
         :param pip_runner: This class is responsible for executing our pip
             on our behalf.
@@ -179,7 +156,6 @@ def __init__(self, osutils, runtime, pip_runner=None):
         if pip_runner is None:
             pip_runner = PipRunner(SubprocessPip(osutils))
         self._pip = pip_runner
-        self.runtime = runtime
 
     def build_site_packages(self, requirements_filepath,
                             target_directory,
@@ -336,9 +312,8 @@ def _download_all_dependencies(self, requirements_filename, directory):
     def _download_binary_wheels(self, packages, directory):
         # Try to get binary wheels for each package that isn't compatible.
         LOG.debug("Downloading missing wheels: %s", packages)
-        lambda_abi = get_lambda_abi(self.runtime)
         self._pip.download_manylinux_wheels(
-            [pkg.identifier for pkg in packages], directory, lambda_abi)
+            [pkg.identifier for pkg in packages], directory)
 
     def _build_sdists(self, sdists, directory, compile_c=True):
         LOG.debug("Build missing wheels from sdists "
@@ -366,9 +341,6 @@ def _is_compatible_wheel_filename(self, filename):
         # Verify platform is compatible
         if platform not in self._MANYLINUX_COMPATIBLE_PLATFORM:
             return False
-
-        lambda_runtime_abi = get_lambda_abi(self.runtime)
-
         # Verify that the ABI is compatible with lambda. Either none or the
         # correct type for the python version cp27mu for py27 and cp36m for
         # py36.
@@ -379,7 +351,7 @@ def _is_compatible_wheel_filename(self, filename):
             # Deploying python 3 function which means we need cp36m abi
             # We can also accept abi3 which is the CPython 3 Stable ABI and
             # will work on any version of python 3.
-            return abi == lambda_runtime_abi or abi == 'abi3'
+            return abi == 'cp36m' or abi == 'abi3'
         elif prefix_version == 'cp2':
             # Deploying to python 2 function which means we need cp27mu abi
             return abi == 'cp27mu'
@@ -643,7 +615,7 @@ def download_all_dependencies(self, requirements_filename, directory):
             # complain at deployment time.
             self.build_wheel(wheel_package_path, directory)
 
-    def download_manylinux_wheels(self, packages, directory, lambda_abi):
+    def download_manylinux_wheels(self, packages, directory):
         """Download wheel files for manylinux for all the given packages."""
         # If any one of these dependencies fails pip will bail out. Since we
         # are only interested in all the ones we can download, we need to feed

tests/functional/workflows/python_pip/test_packager.py

@@ -1,4 +1,3 @@
-import sys
 import os
 import zipfile
 import tarfile
@@ -17,7 +16,7 @@
 from aws_lambda_builders.workflows.python_pip.packager import SDistMetadataFetcher
 from aws_lambda_builders.workflows.python_pip.packager import \
     InvalidSourceDistributionNameError
-from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi
+from aws_lambda_builders.workflows.python_pip.compat import lambda_abi
 from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_env_vars
 from aws_lambda_builders.workflows.python_pip.compat import pip_no_compile_c_shim
 from aws_lambda_builders.workflows.python_pip.utils import OSUtils
@@ -215,7 +214,7 @@ def _write_requirements_txt(self, packages, directory):
     def _make_appdir_and_dependency_builder(self, reqs, tmpdir, runner):
         appdir = str(_create_app_structure(tmpdir))
         self._write_requirements_txt(reqs, appdir)
-        builder = DependencyBuilder(OSUtils(), "python3.6", runner)
+        builder = DependencyBuilder(OSUtils(), runner)
         return appdir, builder
 
     def test_can_build_local_dir_as_whl(self, tmpdir, pip_runner, osutils):
@@ -645,7 +644,7 @@ def test_can_replace_incompat_whl(self, tmpdir, osutils, pip_runner):
             expected_args=[
                 '--only-binary=:all:', '--no-deps', '--platform',
                 'manylinux1_x86_64', '--implementation', 'cp',
-                '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY,
+                '--abi', lambda_abi, '--dest', mock.ANY,
                 'bar==1.2'
             ],
             packages=[
@@ -678,7 +677,7 @@ def test_whitelist_sqlalchemy(self, tmpdir, osutils, pip_runner):
             expected_args=[
                 '--only-binary=:all:', '--no-deps', '--platform',
                 'manylinux1_x86_64', '--implementation', 'cp',
-                '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY,
+                '--abi', lambda_abi, '--dest', mock.ANY,
                 'sqlalchemy==1.1.18'
             ],
             packages=[
@@ -840,7 +839,7 @@ def test_build_into_existing_dir_with_preinstalled_packages(
             expected_args=[
                 '--only-binary=:all:', '--no-deps', '--platform',
                 'manylinux1_x86_64', '--implementation', 'cp',
-                '--abi', get_lambda_abi(builder.runtime), '--dest', mock.ANY,
+                '--abi', lambda_abi, '--dest', mock.ANY,
                 'foo==1.2'
             ],
             packages=[

tests/integration/workflows/python_pip/test_python_pip.py

@@ -47,9 +47,11 @@ def test_must_build_python_project(self):
         self.assertEquals(expected_files, output_files)
 
     def test_runtime_validate_python_project_fail_open_unsupported_runtime(self):
-        with self.assertRaises(WorkflowFailedError):
-            self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid,
-                               runtime="python2.8")
+        self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid,
+                           runtime="python2.8")
+        expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"})
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEquals(expected_files, output_files)
 
     def test_must_fail_to_resolve_dependencies(self):
 

tests/unit/workflows/python_pip/test_actions.py

@@ -20,7 +20,8 @@ def test_action_must_call_builder(self, PythonPipDependencyBuilderMock):
 
         builder_instance.build_dependencies.assert_called_with("artifacts",
                                                                "scratch_dir",
-                                                               "manifest")
+                                                               "manifest",
+                                                               "runtime")
 
     @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder")
     def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock):

tests/unit/workflows/python_pip/test_packager.py

@@ -1,3 +1,4 @@
+import sys
 from collections import namedtuple
 
 import mock
@@ -11,7 +12,6 @@
 from aws_lambda_builders.workflows.python_pip.packager import Package
 from aws_lambda_builders.workflows.python_pip.packager import PipRunner
 from aws_lambda_builders.workflows.python_pip.packager import SubprocessPip
-from aws_lambda_builders.workflows.python_pip.packager import get_lambda_abi
 from aws_lambda_builders.workflows.python_pip.packager \
     import InvalidSourceDistributionNameError
 from aws_lambda_builders.workflows.python_pip.packager import NoSuchPackageError
@@ -85,29 +85,17 @@ def popen(self, *args, **kwargs):
         return self._processes.pop()
 
 
-class TestGetLambdaAbi(object):
-    def test_get_lambda_abi_python27(self):
-        assert "cp27mu" == get_lambda_abi("python2.7")
-
-    def test_get_lambda_abi_python36(self):
-        assert "cp36m" == get_lambda_abi("python3.6")
-
-    def test_get_lambda_abi_python37(self):
-        assert "cp37m" == get_lambda_abi("python3.7")
-
-
 class TestPythonPipDependencyBuilder(object):
     def test_can_call_dependency_builder(self, osutils):
         mock_dep_builder = mock.Mock(spec=DependencyBuilder)
         osutils_mock = mock.Mock(spec=osutils)
         builder = PythonPipDependencyBuilder(
             osutils=osutils_mock,
             dependency_builder=mock_dep_builder,
-            runtime="runtime"
         )
         builder.build_dependencies(
             'artifacts/path/', 'scratch_dir/path/',
-            'path/to/requirements.txt'
+            'path/to/requirements.txt', 'python3.6'
         )
         mock_dep_builder.build_site_packages.assert_called_once_with(
             'path/to/requirements.txt', 'artifacts/path/', 'scratch_dir/path/')
@@ -230,10 +218,14 @@ def test_download_wheels(self, pip_factory):
         # for getting lambda compatible wheels.
         pip, runner = pip_factory()
         packages = ['foo', 'bar', 'baz']
-        runner.download_manylinux_wheels(packages, 'directory', "abi")
+        runner.download_manylinux_wheels(packages, 'directory')
+        if sys.version_info[0] == 2:
+            abi = 'cp27mu'
+        else:
+            abi = 'cp36m'
         expected_prefix = ['download', '--only-binary=:all:', '--no-deps',
                            '--platform', 'manylinux1_x86_64',
-                           '--implementation', 'cp', '--abi', "abi",
+                           '--implementation', 'cp', '--abi', abi,
                            '--dest', 'directory']
         for i, package in enumerate(packages):
             assert pip.calls[i].args == expected_prefix + [package]
@@ -242,7 +234,7 @@ def test_download_wheels(self, pip_factory):
 
     def test_download_wheels_no_wheels(self, pip_factory):
         pip, runner = pip_factory()
-        runner.download_manylinux_wheels([], 'directory', "abi")
+        runner.download_manylinux_wheels([], 'directory')
         assert len(pip.calls) == 0
 
     def test_raise_no_such_package_error(self, pip_factory):