fix(path-resolver): Add additional binaries to python-pip workflow

aws_lambda_builders/exceptions.py

@@ -19,7 +19,7 @@ class MisMatchRuntimeError(LambdaBuilderError):
     MESSAGE = (
         "{language} executable found in your path does not "
         "match runtime. "
-        "\n Expected version: {required_runtime}, Found version: {runtime_path}. "
+        "\n Expected version: {required_runtime}, Found a different version at {runtime_path}. "
         "\n Possibly related: https://github.com/awslabs/aws-lambda-builders/issues/30"
     )
 

aws_lambda_builders/path_resolver.py

@@ -6,10 +6,14 @@
 
 
 class PathResolver(object):
-    def __init__(self, binary, runtime, executable_search_paths=None):
+    def __init__(self, binary, runtime, additional_binaries=None, executable_search_paths=None):
         self.binary = binary
         self.runtime = runtime
         self.executables = [self.runtime, self.binary]
+        self.additional_binaries = additional_binaries
+        if isinstance(additional_binaries, list):
+            self.executables = self.executables + self.additional_binaries
+
         self.executable_search_paths = executable_search_paths
 
     def _which(self):

aws_lambda_builders/workflows/python_pip/workflow.py

@@ -6,6 +6,7 @@
 from aws_lambda_builders.workflow import BaseWorkflow, Capability
 from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, LinkSourceAction
 from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator
+from aws_lambda_builders.path_resolver import PathResolver
 
 from .actions import PythonPipBuildAction
 from .utils import OSUtils, is_experimental_build_improvements_enabled
@@ -64,6 +65,8 @@ class PythonPipWorkflow(BaseWorkflow):
         ".idea",
     )
 
+    PYTHON_VERSION_THREE = "3"
+
     def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs):
 
         super(PythonPipWorkflow, self).__init__(
@@ -113,5 +116,24 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim
 
         self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES))
 
+    def get_resolvers(self):
+        """
+        Specialized Python path resolver that looks for additional binaries in addition to the language specific binary.
+        """
+        return [
+            PathResolver(
+                runtime=self.runtime,
+                binary=self.CAPABILITY.language,
+                additional_binaries=self._get_additional_binaries(),
+                executable_search_paths=self.executable_search_paths,
+            )
+        ]
+
+    def _get_additional_binaries(self):
+        # python3 is an additional binary that has to be considered in addition to the original python binary, when
+        # the specified python runtime is 3.x
+        major, _ = self.runtime.replace(self.CAPABILITY.language, "").split(".")
+        return [f"{self.CAPABILITY.language}{major}"] if major == self.PYTHON_VERSION_THREE else None
+
     def get_validators(self):
         return [PythonRuntimeValidator(runtime=self.runtime, architecture=self.architecture)]

tests/integration/workflows/python_pip/test_python_pip.py

@@ -1,4 +1,5 @@
 import os
+import pathlib
 import shutil
 import sys
 import platform
@@ -11,6 +12,7 @@
 from aws_lambda_builders.exceptions import WorkflowFailedError
 import logging
 
+from aws_lambda_builders.utils import which
 from aws_lambda_builders.workflows.python_pip.utils import EXPERIMENTAL_FLAG_BUILD_PERFORMANCE
 
 logger = logging.getLogger("aws_lambda_builders.workflows.python_pip.workflow")
@@ -47,7 +49,10 @@ def setUp(self):
             "local-dependencies",
         }
 
-        self.builder = LambdaBuilder(language="python", dependency_manager="pip", application_framework=None)
+        self.dependency_manager = "pip"
+        self.builder = LambdaBuilder(
+            language="python", dependency_manager=self.dependency_manager, application_framework=None
+        )
         self.runtime = "{language}{major}.{minor}".format(
             language=self.builder.capability.language, major=sys.version_info.major, minor=sys.version_info.minor
         )
@@ -98,6 +103,33 @@ def test_must_build_python_project(self):
         output_files = set(os.listdir(self.artifacts_dir))
         self.assertEqual(expected_files, output_files)
 
+    def test_must_build_python_project_python3_binary(self):
+        python_paths = which("python")
+        executable_dir = pathlib.Path(tempfile.gettempdir())
+        new_python_path = executable_dir.joinpath("python3")
+        os.symlink(python_paths[0], new_python_path)
+        # Build with access to the newly symlinked python3 binary.
+        self.builder.build(
+            self.source_dir,
+            self.artifacts_dir,
+            self.scratch_dir,
+            self.manifest_path_valid,
+            runtime=self.runtime,
+            experimental_flags=self.experimental_flags,
+            executable_search_paths=[executable_dir],
+        )
+
+        if self.runtime == "python3.6":
+            self.check_architecture_in("numpy-1.17.4.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"])
+            expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"})
+        else:
+            self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"])
+            expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"})
+
+        output_files = set(os.listdir(self.artifacts_dir))
+        self.assertEqual(expected_files, output_files)
+        os.unlink(new_python_path)
+
     @skipIf(NOT_ARM, "Skip if not running on ARM64")
     def test_must_build_python_project_from_sdist_with_arm(self):
         if self.runtime not in ARM_RUNTIMES:

tests/unit/test_path_resolver.py

@@ -9,11 +9,12 @@
 
 class TestPathResolver(TestCase):
     def setUp(self):
-        self.path_resolver = PathResolver(runtime="chitti2.0", binary="chitti")
+        self.path_resolver = PathResolver(runtime="chitti2.0", binary="chitti", additional_binaries=["chitti2"])
 
     def test_inits(self):
         self.assertEqual(self.path_resolver.runtime, "chitti2.0")
         self.assertEqual(self.path_resolver.binary, "chitti")
+        self.assertEqual(self.path_resolver.executables, ["chitti2.0", "chitti", "chitti2"])
 
     def test_which_fails(self):
         with self.assertRaises(ValueError):

tests/unit/workflows/python_pip/test_workflow.py

@@ -5,6 +5,7 @@
 from parameterized import parameterized_class
 
 from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, LinkSourceAction
+from aws_lambda_builders.path_resolver import PathResolver
 from aws_lambda_builders.workflows.python_pip.utils import OSUtils, EXPERIMENTAL_FLAG_BUILD_PERFORMANCE
 from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator
 from aws_lambda_builders.workflows.python_pip.workflow import PythonPipBuildAction, PythonPipWorkflow
@@ -29,10 +30,13 @@ def setUp(self):
             "artifacts",
             "scratch_dir",
             "manifest",
-            runtime="python3.7",
+            runtime="python3.9",
             osutils=self.osutils_mock,
             experimental_flags=self.experimental_flags,
         )
+        self.python_major_version = "3"
+        self.python_minor_version = "9"
+        self.language = "python"
 
     def test_workflow_sets_up_actions(self):
         self.assertEqual(len(self.workflow.actions), 2)
@@ -46,7 +50,7 @@ def test_workflow_sets_up_actions_without_requirements(self):
             "artifacts",
             "scratch_dir",
             "manifest",
-            runtime="python3.7",
+            runtime="python3.9",
             osutils=self.osutils_mock,
             experimental_flags=self.experimental_flags,
         )
@@ -57,6 +61,18 @@ def test_workflow_validator(self):
         for validator in self.workflow.get_validators():
             self.assertTrue(isinstance(validator, PythonRuntimeValidator))
 
+    def test_workflow_resolver(self):
+        for resolver in self.workflow.get_resolvers():
+            self.assertTrue(isinstance(resolver, PathResolver))
+            self.assertTrue(
+                resolver.executables,
+                [
+                    self.language,
+                    f"{self.language}{self.python_major_version}.{self.python_minor_version}",
+                    f"{self.language}{self.python_major_version}",
+                ],
+            )
+
     def test_workflow_sets_up_actions_without_download_dependencies_with_dependencies_dir(self):
         osutils_mock = Mock(spec=self.osutils)
         osutils_mock.file_exists.return_value = True
@@ -65,7 +81,7 @@ def test_workflow_sets_up_actions_without_download_dependencies_with_dependencie
             "artifacts",
             "scratch_dir",
             "manifest",
-            runtime="python3.7",
+            runtime="python3.9",
             osutils=osutils_mock,
             dependencies_dir="dep",
             download_dependencies=False,
@@ -86,7 +102,7 @@ def test_workflow_sets_up_actions_with_download_dependencies_and_dependencies_di
             "artifacts",
             "scratch_dir",
             "manifest",
-            runtime="python3.7",
+            runtime="python3.9",
             osutils=osutils_mock,
             dependencies_dir="dep",
             download_dependencies=True,
@@ -111,7 +127,7 @@ def test_workflow_sets_up_actions_without_download_dependencies_without_dependen
             "artifacts",
             "scratch_dir",
             "manifest",
-            runtime="python3.7",
+            runtime="python3.9",
             osutils=osutils_mock,
             dependencies_dir=None,
             download_dependencies=False,
@@ -128,7 +144,7 @@ def test_workflow_sets_up_actions_without_combine_dependencies(self):
             "artifacts",
             "scratch_dir",
             "manifest",
-            runtime="python3.7",
+            runtime="python3.9",
             osutils=osutils_mock,
             dependencies_dir="dep",
             download_dependencies=True,
@@ -147,15 +163,15 @@ def test_must_build_with_architecture(self, PythonPipBuildActionMock):
             "artifacts",
             "scratch_dir",
             "manifest",
-            runtime="python3.7",
+            runtime="python3.9",
             architecture="ARM64",
             osutils=self.osutils_mock,
         )
         PythonPipBuildActionMock.assert_called_with(
             "artifacts",
             "scratch_dir",
             "manifest",
-            "python3.7",
+            "python3.9",
             None,
             binaries=ANY,
             architecture="ARM64",