feat: Support Python3.8 and Java11 runtimes

.appveyor.yml

@@ -1,48 +1,130 @@
 version: 1.0.{build}
-image: Visual Studio 2017
+image:
+  - Visual Studio 2017
+  - Ubuntu
 
 environment:
   GOPATH: c:\gopath
   GOVERSION: 1.11
   GRADLE_OPTS: -Dorg.gradle.daemon=false
+  nodejs_version: "8.10.0"
 
   matrix:
 
   - PYTHON: "C:\\Python27-x64"
+    PYTHON_VERSION: '2.7.16'
+    PYTHON_ARCH: '64'
+    LINE_COVERAGE: '91'
+    NEW_FLAKE8: 0
+    JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
   - PYTHON: "C:\\Python36-x64"
+    PYTHON_VERSION: '3.6.8'
+    PYTHON_ARCH: '64'
+    LINE_COVERAGE: '91'
+    NEW_FLAKE8: 0
+    JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
   - PYTHON: "C:\\Python37-x64"
+    PYTHON_VERSION: '3.7.4'
+    PYTHON_ARCH: '64'
+    LINE_COVERAGE: '91'
+    NEW_FLAKE8: 0
+    JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
+  - PYTHON: "C:\\Python38-x64"
+    PYTHON_VERSION: '3.8.0'
+    PYTHON_ARCH: '64'
+    LINE_COVERAGE: '72'
+    NEW_FLAKE8: 1
+    JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
 
 
 build: off
 
-install:
-# To run Nodejs workflow integ tests
-- ps: Install-Product node 8.10
-
-- "set PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
-- "%PYTHON%\\python.exe -m pip install -r requirements/dev.txt"
-- "%PYTHON%\\python.exe -m pip install -e ."
-- "set PATH=C:\\Ruby25-x64\\bin;%PATH%"
-- "gem --version"
-- "gem install bundler -v 1.17.3"
-- "bundler --version"
-- "echo %PATH%"
-
-# setup go
-- rmdir c:\go /s /q
-- "choco install golang"
-- "choco install bzr"
-- "choco install dep"
-- setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;"
-- "go version"
-- "go env"
-
-# setup Gradle
-- "choco install gradle"
-
-# Echo final Path
-- "echo %PATH%"
-
-test_script:
-- "%PYTHON%\\python.exe -m pytest --cov aws_lambda_builders --cov-report term-missing tests/unit tests/functional"
-- "%PYTHON%\\python.exe -m pytest tests/integration"
+for:
+  -
+    matrix:
+      only:
+        - image: Visual Studio 2017
+
+    install:
+    # To run Nodejs workflow integ tests
+    - ps: Install-Product node 8.10
+
+    - "set PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%"
+    - "%PYTHON%\\python.exe -m pip install -r requirements/dev.txt"
+    - "%PYTHON%\\python.exe -m pip install -e ."
+    - "set PATH=C:\\Ruby25-x64\\bin;%PATH%"
+    - "gem --version"
+    - "gem install bundler -v 1.17.3"
+    - "bundler --version"
+    - "echo %PATH%"
+
+    # setup go
+    - rmdir c:\go /s /q
+    - "choco install golang"
+    - "choco install bzr"
+    - "choco install dep"
+    - setx PATH "C:\go\bin;C:\gopath\bin;C:\Program Files (x86)\Bazaar\;C:\Program Files\Mercurial;%PATH%;"
+    - "go version"
+    - "go env"
+
+    # setup Gradle
+    - "choco install gradle"
+
+    # Echo final Path
+    - "echo %PATH%"
+
+    test_script:
+    - "%PYTHON%\\python.exe -m pytest --cov aws_lambda_builders --cov-report term-missing tests/unit tests/functional"
+    - "%PYTHON%\\python.exe -m pytest tests/integration"
+
+  -
+    matrix:
+      only:
+        - image: Ubuntu
+
+    install:
+      - sh: "JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64"
+      - sh: "PATH=$JAVA_HOME/bin:$PATH"
+      - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate"
+      - sh: "rvm use 2.5"
+      - sh: "nvm install ${nodejs_version}"
+      - sh: "npm install npm@5.6.0 -g"
+      - sh: "npm -v"
+      - sh: "echo $PATH"
+      - sh: "java --version"
+
+      # Install latest gradle
+      - sh: "sudo apt-get -y remove gradle"
+      - sh: "sudo apt-get install go-dep"
+      - sh: "wget https://services.gradle.org/distributions/gradle-5.5-bin.zip -P /tmp"
+      - sh: "sudo unzip -d /opt/gradle /tmp/gradle-*.zip"
+      - sh: "PATH=/opt/gradle/gradle-5.5/bin:$PATH"
+
+    build_script:
+      - "python -c \"import sys; print(sys.executable)\""
+      - "LAMBDA_BUILDERS_DEV=1 pip install -e \".[dev]\""
+
+    test_script:
+      - "LAMBDA_BUILDERS_DEV=1 pytest --cov aws_lambda_builders --cov-report term-missing --cov-fail-under ${LINE_COVERAGE} tests/unit tests/functional"
+      - "pylint --rcfile .pylintrc aws_lambda_builders"
+
+      # Runs only in Linux
+      - "LAMBDA_BUILDERS_DEV=1 pytest -vv tests/integration"
+
+  -
+    matrix:
+      only:
+        - OLD_FLAKE8: 1
+
+    test_script:
+      - "flake8 lambda_builders"
+      - "flake8 tests/unit tests/integration"
+
+  -
+    matrix:
+      only:
+        - NEW_FLAKE8: 0
+
+    test_script:
+      - "flake8 lambda_builders"
+      - "flake8 tests/unit tests/integration --extend-ignore=W504"

.pylintrc

@@ -59,7 +59,8 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=R0201,W0613,I0021,I0020,W1618,W1619,R0902,R0903,W0231,W0611,R0913,W0703,C0330,R0204,I0011,R0904,C0301
+# R0205,W0107,R1705,R1710,R1719,R1720,R1714 are all disable due to a forced upgrade to support python3.8
+disable=R0201,W0613,I0021,I0020,W1618,W1619,R0902,R0903,W0231,W0611,R0913,W0703,C0330,R0204,I0011,R0904,C0301, R0205,W0107,R1705,R1710,R1719,R1720,R1714
 
 
 [REPORTS]

Makefile

@@ -16,7 +16,7 @@ integ-test:
 flake:
 	# Make sure code conforms to PEP8 standards
 	flake8 lambda_builders
-	flake8 tests/unit tests/integration
+	flake8 tests/unit tests/integration --extend-ignore=W504
 
 lint:
 	# Liner performs static analysis to catch latent bugs

aws_lambda_builders/workflows/java_gradle/gradle_validator.py

@@ -14,22 +14,25 @@ class GradleValidator(object):
     VERSION_STRING_WARNING = "%s failed to return a version string using the '-v' option. The workflow is unable to " \
                              "check that the version of the JVM used is compatible with AWS Lambda."
 
-    MAJOR_VERSION_WARNING = "%s is using a JVM with major version %s which is newer than 8 that is supported by AWS " \
+    MAJOR_VERSION_WARNING = "%s is using a JVM with major version %s which is newer than %s that is supported by AWS " \
                             "Lambda. The compiled function code may not run in AWS Lambda unless the project has " \
-                            "been configured to be compatible with Java 8 using 'targetCompatibility' in Gradle."
+                            "been configured to be compatible with Java %s using 'targetCompatibility' in Gradle."
 
-    def __init__(self, os_utils=None, log=None):
+    def __init__(self, runtime, os_utils=None, log=None):
         self.language = 'java'
         self._valid_binary_path = None
+        self._runtime = runtime
         self.os_utils = OSUtils() if not os_utils else os_utils
         self.log = LOG if not log else log
 
     def validate(self, gradle_path):
         jvm_mv = self._get_major_version(gradle_path)
 
+        language_version = self._runtime.replace('java', '')
+
         if jvm_mv:
-            if int(jvm_mv) > 8:
-                self.log.warning(self.MAJOR_VERSION_WARNING, gradle_path, jvm_mv)
+            if int(jvm_mv) > int(language_version):
+                self.log.warning(self.MAJOR_VERSION_WARNING, gradle_path, jvm_mv, language_version, language_version)
         else:
             self.log.warning(self.VERSION_STRING_WARNING, gradle_path)
 

aws_lambda_builders/workflows/java_gradle/workflow.py

@@ -55,7 +55,7 @@ def get_resolvers(self):
         return [GradleResolver(executable_search_paths=self.executable_search_paths)]
 
     def get_validators(self):
-        return [GradleValidator(self.os_utils)]
+        return [GradleValidator(self.runtime, self.os_utils)]
 
     @property
     def build_output_dir(self):

aws_lambda_builders/workflows/java_maven/maven_validator.py

@@ -14,22 +14,25 @@ class MavenValidator(object):
     VERSION_STRING_WARNING = "%s failed to return a version string using the '-v' option. The workflow is unable to " \
                              "check that the version of the JVM used is compatible with AWS Lambda."
 
-    MAJOR_VERSION_WARNING = "%s is using a JVM with major version %s which is newer than 8 that is supported by AWS " \
+    MAJOR_VERSION_WARNING = "%s is using a JVM with major version %s which is newer than %s that is supported by AWS " \
                             "Lambda. The compiled function code may not run in AWS Lambda unless the project has " \
-                            "been configured to be compatible with Java 8 using 'maven.compiler.target' in Maven."
+                            "been configured to be compatible with Java %s using 'maven.compiler.target' in Maven."
 
-    def __init__(self, os_utils=None, log=None):
+    def __init__(self, runtime, os_utils=None, log=None):
         self.language = 'java'
         self._valid_binary_path = None
+        self._runtime = runtime
         self.os_utils = OSUtils() if not os_utils else os_utils
         self.log = LOG if not log else log
 
     def validate(self, maven_path):
         jvm_mv = self._get_major_version(maven_path)
 
+        language_version = self._runtime.replace('java', '')
+
         if jvm_mv:
-            if int(jvm_mv) > 8:
-                self.log.warning(self.MAJOR_VERSION_WARNING, maven_path, jvm_mv)
+            if int(jvm_mv) > int(language_version):
+                self.log.warning(self.MAJOR_VERSION_WARNING, maven_path, jvm_mv, language_version, language_version)
         else:
             self.log.warning(self.VERSION_STRING_WARNING, maven_path)
 

aws_lambda_builders/workflows/java_maven/workflow.py

@@ -55,4 +55,4 @@ def get_resolvers(self):
         return [MavenResolver(executable_search_paths=self.executable_search_paths)]
 
     def get_validators(self):
-        return [MavenValidator(self.os_utils)]
+        return [MavenValidator(self.runtime, self.os_utils)]

aws_lambda_builders/workflows/python_pip/packager.py

@@ -70,7 +70,8 @@ def get_lambda_abi(runtime):
     supported = {
         "python2.7": "cp27mu",
         "python3.6": "cp36m",
-        "python3.7": "cp37m"
+        "python3.7": "cp37m",
+        "python3.8": "cp38"
     }
 
     if runtime not in supported:

aws_lambda_builders/workflows/python_pip/validator.py

@@ -15,7 +15,8 @@ class PythonRuntimeValidator(object):
     SUPPORTED_RUNTIMES = {
         "python2.7",
         "python3.6",
-        "python3.7"
+        "python3.7",
+        "python3.8"
     }
 
     def __init__(self, runtime):

requirements/dev.txt

@@ -1,10 +1,11 @@
 coverage==4.3.4
-flake8==3.3.0
-tox==2.2.1
+flake8==3.3.0; python_version < '3.8'
+flake8==3.7.9; python_version >= '3.8'
 pytest-cov==2.4.0
 # astroid > 2.0.4 is not compatible with pylint1.7
-astroid>=1.5.8,<2.1.0
-pylint==1.7.2
+astroid>=1.5.8,<2.1.0; python_version < '3.8'
+pylint==1.7.2; python_version < '3.8'
+pylint==2.4.4; python_version >= '3.8'
 
 # Test requirements
 pytest==3.0.7

tests/integration/workflows/java_gradle/test_java_gradle.py

@@ -17,7 +17,7 @@ def setUp(self):
         self.artifacts_dir = tempfile.mkdtemp()
         self.scratch_dir = tempfile.mkdtemp()
         self.builder = LambdaBuilder(language='java', dependency_manager='gradle', application_framework=None)
-        self.runtime = 'java'
+        self.runtime = 'java11'
 
     def tearDown(self):
         shutil.rmtree(self.artifacts_dir)

tests/integration/workflows/java_maven/test_java_maven.py

@@ -15,7 +15,7 @@ def setUp(self):
         self.artifacts_dir = tempfile.mkdtemp()
         self.scratch_dir = tempfile.mkdtemp()
         self.builder = LambdaBuilder(language='java', dependency_manager='maven', application_framework=None)
-        self.runtime = 'java'
+        self.runtime = 'java8'
 
     def tearDown(self):
         shutil.rmtree(self.artifacts_dir)

tests/integration/workflows/python_pip/test_python_pip.py

@@ -36,8 +36,8 @@ def setUp(self):
         self.runtime_mismatch = {
             'python3.6': 'python2.7',
             'python3.7': 'python2.7',
-            'python2.7': 'python3.6'
-
+            'python2.7': 'python3.8',
+            'python3.8': 'python2.7'
         }
 
     def tearDown(self):
@@ -48,7 +48,10 @@ def test_must_build_python_project(self):
         self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid,
                            runtime=self.runtime)
 
-        expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"})
+        if self.runtime == "python2.7":
+            expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"})
+        else:
+            expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"})
         output_files = set(os.listdir(self.artifacts_dir))
         self.assertEquals(expected_files, output_files)
 
@@ -72,7 +75,11 @@ def test_must_fail_to_resolve_dependencies(self):
             self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_invalid,
                                runtime=self.runtime)
 
-        self.assertIn("Invalid requirement: 'adfasf=1.2.3'", str(ctx.exception))
+        # In Python2 a 'u' is now added to the exception string. To account for this, we see if either one is in the
+        # output
+        message_in_exception = "Invalid requirement: 'adfasf=1.2.3'" in str(ctx.exception) or \
+                               "Invalid requirement: u'adfasf=1.2.3'" in str(ctx.exception)
+        self.assertTrue(message_in_exception)
 
     def test_must_fail_if_requirements_not_found(self):
 

tests/integration/workflows/python_pip/testdata/requirements-numpy.txt

@@ -1 +1,2 @@
-numpy==1.15.4
+numpy==1.15.4; python_version == '2.7'
+numpy==1.17.4; python_version >= '3.6'