diff --git a/aws_lambda_builders/exceptions.py b/aws_lambda_builders/exceptions.py index 2ed3f3f8e..f1c24eb1a 100644 --- a/aws_lambda_builders/exceptions.py +++ b/aws_lambda_builders/exceptions.py @@ -19,7 +19,7 @@ class MisMatchRuntimeError(LambdaBuilderError): MESSAGE = ( "{language} executable found in your path does not " "match runtime. " - "\n Expected version: {required_runtime}, Found version: {runtime_path}. " + "\n Expected version: {required_runtime}, Found a different version at {runtime_path}. " "\n Possibly related: https://github.com/awslabs/aws-lambda-builders/issues/30" ) diff --git a/aws_lambda_builders/path_resolver.py b/aws_lambda_builders/path_resolver.py index e8f9736ee..6f15bdcbe 100644 --- a/aws_lambda_builders/path_resolver.py +++ b/aws_lambda_builders/path_resolver.py @@ -6,10 +6,14 @@ class PathResolver(object): - def __init__(self, binary, runtime, executable_search_paths=None): + def __init__(self, binary, runtime, additional_binaries=None, executable_search_paths=None): self.binary = binary self.runtime = runtime self.executables = [self.runtime, self.binary] + self.additional_binaries = additional_binaries + if isinstance(additional_binaries, list): + self.executables = self.executables + self.additional_binaries + self.executable_search_paths = executable_search_paths def _which(self): diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index b0da62340..844d02fbd 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -6,6 +6,7 @@ from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, LinkSourceAction from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator +from aws_lambda_builders.path_resolver import PathResolver from .actions import PythonPipBuildAction from .utils import OSUtils, is_experimental_build_improvements_enabled @@ -64,6 +65,8 @@ class PythonPipWorkflow(BaseWorkflow): ".idea", ) + PYTHON_VERSION_THREE = "3" + def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtime=None, osutils=None, **kwargs): super(PythonPipWorkflow, self).__init__( @@ -113,5 +116,24 @@ def __init__(self, source_dir, artifacts_dir, scratch_dir, manifest_path, runtim self.actions.append(CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES)) + def get_resolvers(self): + """ + Specialized Python path resolver that looks for additional binaries in addition to the language specific binary. + """ + return [ + PathResolver( + runtime=self.runtime, + binary=self.CAPABILITY.language, + additional_binaries=self._get_additional_binaries(), + executable_search_paths=self.executable_search_paths, + ) + ] + + def _get_additional_binaries(self): + # python3 is an additional binary that has to be considered in addition to the original python binary, when + # the specified python runtime is 3.x + major, _ = self.runtime.replace(self.CAPABILITY.language, "").split(".") + return [f"{self.CAPABILITY.language}{major}"] if major == self.PYTHON_VERSION_THREE else None + def get_validators(self): return [PythonRuntimeValidator(runtime=self.runtime, architecture=self.architecture)] diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 526320e98..3e1fe6cb3 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -1,4 +1,5 @@ import os +import pathlib import shutil import sys import platform @@ -11,6 +12,7 @@ from aws_lambda_builders.exceptions import WorkflowFailedError import logging +from aws_lambda_builders.utils import which from aws_lambda_builders.workflows.python_pip.utils import EXPERIMENTAL_FLAG_BUILD_PERFORMANCE logger = logging.getLogger("aws_lambda_builders.workflows.python_pip.workflow") @@ -47,7 +49,10 @@ def setUp(self): "local-dependencies", } - self.builder = LambdaBuilder(language="python", dependency_manager="pip", application_framework=None) + self.dependency_manager = "pip" + self.builder = LambdaBuilder( + language="python", dependency_manager=self.dependency_manager, application_framework=None + ) self.runtime = "{language}{major}.{minor}".format( language=self.builder.capability.language, major=sys.version_info.major, minor=sys.version_info.minor ) @@ -98,6 +103,33 @@ def test_must_build_python_project(self): output_files = set(os.listdir(self.artifacts_dir)) self.assertEqual(expected_files, output_files) + def test_must_build_python_project_python3_binary(self): + python_paths = which("python") + executable_dir = pathlib.Path(tempfile.gettempdir()) + new_python_path = executable_dir.joinpath("python3") + os.symlink(python_paths[0], new_python_path) + # Build with access to the newly symlinked python3 binary. + self.builder.build( + self.source_dir, + self.artifacts_dir, + self.scratch_dir, + self.manifest_path_valid, + runtime=self.runtime, + experimental_flags=self.experimental_flags, + executable_search_paths=[executable_dir], + ) + + if self.runtime == "python3.6": + self.check_architecture_in("numpy-1.17.4.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) + expected_files = self.test_data_files.union({"numpy", "numpy-1.17.4.dist-info"}) + else: + self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"]) + expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"}) + + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEqual(expected_files, output_files) + os.unlink(new_python_path) + @skipIf(NOT_ARM, "Skip if not running on ARM64") def test_must_build_python_project_from_sdist_with_arm(self): if self.runtime not in ARM_RUNTIMES: diff --git a/tests/unit/test_path_resolver.py b/tests/unit/test_path_resolver.py index 20d52ae7b..b576af7e4 100644 --- a/tests/unit/test_path_resolver.py +++ b/tests/unit/test_path_resolver.py @@ -9,11 +9,12 @@ class TestPathResolver(TestCase): def setUp(self): - self.path_resolver = PathResolver(runtime="chitti2.0", binary="chitti") + self.path_resolver = PathResolver(runtime="chitti2.0", binary="chitti", additional_binaries=["chitti2"]) def test_inits(self): self.assertEqual(self.path_resolver.runtime, "chitti2.0") self.assertEqual(self.path_resolver.binary, "chitti") + self.assertEqual(self.path_resolver.executables, ["chitti2.0", "chitti", "chitti2"]) def test_which_fails(self): with self.assertRaises(ValueError): diff --git a/tests/unit/workflows/python_pip/test_workflow.py b/tests/unit/workflows/python_pip/test_workflow.py index e7c66b06f..0c653f330 100644 --- a/tests/unit/workflows/python_pip/test_workflow.py +++ b/tests/unit/workflows/python_pip/test_workflow.py @@ -5,6 +5,7 @@ from parameterized import parameterized_class from aws_lambda_builders.actions import CopySourceAction, CleanUpAction, LinkSourceAction +from aws_lambda_builders.path_resolver import PathResolver from aws_lambda_builders.workflows.python_pip.utils import OSUtils, EXPERIMENTAL_FLAG_BUILD_PERFORMANCE from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator from aws_lambda_builders.workflows.python_pip.workflow import PythonPipBuildAction, PythonPipWorkflow @@ -29,10 +30,13 @@ def setUp(self): "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", osutils=self.osutils_mock, experimental_flags=self.experimental_flags, ) + self.python_major_version = "3" + self.python_minor_version = "9" + self.language = "python" def test_workflow_sets_up_actions(self): self.assertEqual(len(self.workflow.actions), 2) @@ -46,7 +50,7 @@ def test_workflow_sets_up_actions_without_requirements(self): "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", osutils=self.osutils_mock, experimental_flags=self.experimental_flags, ) @@ -57,6 +61,18 @@ def test_workflow_validator(self): for validator in self.workflow.get_validators(): self.assertTrue(isinstance(validator, PythonRuntimeValidator)) + def test_workflow_resolver(self): + for resolver in self.workflow.get_resolvers(): + self.assertTrue(isinstance(resolver, PathResolver)) + self.assertTrue( + resolver.executables, + [ + self.language, + f"{self.language}{self.python_major_version}.{self.python_minor_version}", + f"{self.language}{self.python_major_version}", + ], + ) + def test_workflow_sets_up_actions_without_download_dependencies_with_dependencies_dir(self): osutils_mock = Mock(spec=self.osutils) osutils_mock.file_exists.return_value = True @@ -65,7 +81,7 @@ def test_workflow_sets_up_actions_without_download_dependencies_with_dependencie "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", osutils=osutils_mock, dependencies_dir="dep", download_dependencies=False, @@ -86,7 +102,7 @@ def test_workflow_sets_up_actions_with_download_dependencies_and_dependencies_di "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", osutils=osutils_mock, dependencies_dir="dep", download_dependencies=True, @@ -111,7 +127,7 @@ def test_workflow_sets_up_actions_without_download_dependencies_without_dependen "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", osutils=osutils_mock, dependencies_dir=None, download_dependencies=False, @@ -128,7 +144,7 @@ def test_workflow_sets_up_actions_without_combine_dependencies(self): "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", osutils=osutils_mock, dependencies_dir="dep", download_dependencies=True, @@ -147,7 +163,7 @@ def test_must_build_with_architecture(self, PythonPipBuildActionMock): "artifacts", "scratch_dir", "manifest", - runtime="python3.7", + runtime="python3.9", architecture="ARM64", osutils=self.osutils_mock, ) @@ -155,7 +171,7 @@ def test_must_build_with_architecture(self, PythonPipBuildActionMock): "artifacts", "scratch_dir", "manifest", - "python3.7", + "python3.9", None, binaries=ANY, architecture="ARM64",