diff --git a/samcli/lib/build/app_builder.py b/samcli/lib/build/app_builder.py index 0737dc09c6..e656c03ec3 100644 --- a/samcli/lib/build/app_builder.py +++ b/samcli/lib/build/app_builder.py @@ -829,6 +829,17 @@ def _get_build_options( if language == "rust" and "Binary" in build_props: options = options if options else {} options["artifact_executable_name"] = build_props["Binary"] + + if language == "python" and "ParentPackageMode" in build_props: + options = options if options else {} + package_root_mode = build_props["ParentPackageMode"] + if package_root_mode == "auto": + options["parent_python_packages"] = ApplicationBuilder._infer_parent_python_packages( + handler, source_code_path + ) + elif package_root_mode == "explicit": + options["parent_python_packages"] = build_props.get("ParentPackages", None) + return options @staticmethod @@ -1045,3 +1056,51 @@ def _parse_builder_response(stdout_data: str, image_name: str) -> Dict: raise ValueError(msg) return cast(Dict, response) + + @staticmethod + def _infer_parent_python_packages(handler: Optional[str], source_code_path: Optional[str]) -> Optional[str]: + """ + Infers the parent python packages from the handler and source code path. The parent packages are the + packages are the union of the handler's parent packages and the source code path's parent packages. + + Parameters + ---------- + handler: str + The handler value of the function + source_code_path: str + The directory path to the source code of the function + Returns + ------- + str + + """ + MODULE_PART_COUNT = 2 # file name and function name + if not handler or not source_code_path: + LOG.warning( + "Both function Handler and CodeUri must be provided when using PackageRootMode 'auto'." + + "Continuing without parent packages." + ) + return None + if handler.count(".") < MODULE_PART_COUNT: + # Handler does not have any parent packages + LOG.warning("Handler '%s' does not have any parent python packages", handler) + return None + + handler_parts = handler.split(".")[0:-MODULE_PART_COUNT] + code_path_parts = list(pathlib.Path(source_code_path).parts) + + # Remove parts from the start of the path until we find the first part of the handler + while len(code_path_parts) > 0 and code_path_parts[0] != handler_parts[0]: + code_path_parts.pop(0) + + if len(code_path_parts) > 0: + parent_packages = ".".join(code_path_parts[0 : len(handler_parts)]) + LOG.debug("Inferred parent python packages '%s'", parent_packages) + return parent_packages + LOG.warning( + "Could not infer parent python packages from Handler '%s' and CodeUri '%s'." + + "Continuing without parent packages.", + handler, + source_code_path, + ) + return None diff --git a/tests/integration/buildcmd/test_build_cmd_python.py b/tests/integration/buildcmd/test_build_cmd_python.py index 2e7f11dbb6..f822f206e2 100644 --- a/tests/integration/buildcmd/test_build_cmd_python.py +++ b/tests/integration/buildcmd/test_build_cmd_python.py @@ -1,4 +1,5 @@ import logging +import pathlib from typing import Set from unittest import skipIf from uuid import uuid4 @@ -6,22 +7,21 @@ import pytest from parameterized import parameterized, parameterized_class +from tests.integration.buildcmd.build_integ_base import ( + BuildIntegBase, + BuildIntegPythonBase, +) from tests.testing_utils import ( + CI_OVERRIDE, IS_WINDOWS, + RUN_BY_CANARY, RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, - RUN_BY_CANARY, - CI_OVERRIDE, - run_command, - SKIP_DOCKER_TESTS, SKIP_DOCKER_BUILD, SKIP_DOCKER_MESSAGE, + SKIP_DOCKER_TESTS, + run_command, ) -from tests.integration.buildcmd.build_integ_base import ( - BuildIntegBase, - BuildIntegPythonBase, -) - LOG = logging.getLogger(__name__) @@ -476,7 +476,6 @@ class TestBuildCommand_PythonFunctions_With_Specified_Architecture(BuildIntegPyt ] ) def test_with_default_requirements(self, runtime, codeuri, use_container, architecture): - self._test_with_default_requirements( runtime, codeuri, use_container, self.test_data_path, architecture=architecture ) @@ -493,7 +492,6 @@ def test_with_default_requirements(self, runtime, codeuri, use_container, archit ) @pytest.mark.al2023 def test_with_default_requirements_al2023(self, runtime, codeuri, use_container, architecture): - self._test_with_default_requirements( runtime, codeuri, use_container, self.test_data_path, architecture=architecture ) @@ -509,6 +507,100 @@ def test_invalid_architecture(self): self.assertIn("Architecture fake is not supported", str(process_execute.stderr)) +class TestBuildCommand_ParentPackages(BuildIntegPythonBase): + template = "template_with_metadata_python.yaml" + runtime = "python3.12" + use_container = False + prop = "CodeUri" + + logical_id_one = "FunctionOne" + logical_id_two = "FunctionTwo" + parent_packages_one = "src.fnone" + parent_packages_two = "src.fntwo" + codeuri_one = "PythonParentPackages/src/fnone" + codeuri_two = "PythonParentPackages/src/fntwo" + handler_one = f"{parent_packages_one}.main.handler" + handler_two = f"{parent_packages_two}.main.handler" + + overrides = { + "Runtime": runtime, + "CodeUriOne": codeuri_one, + "CodeUriTwo": codeuri_two, + "HandlerOne": handler_one, + "HandlerTwo": handler_two, + } + + expected_files_project_manifest = {"numpy", "src"} + expected_source_files = { + "__init__.py", + "main.py", + "requirements.txt", + } + + def test_parent_package_mode_explicit(self): + # Arrange + cmdlist = self.get_command_list( + use_container=self.use_container, + parameter_overrides={ + **self.overrides, + "ParentPackageMode": "explicit", + "ParentPackagesOne": self.parent_packages_one, + "ParentPackagesTwo": self.parent_packages_two, + }, + ) + + # Act + run_command(cmdlist, cwd=self.working_dir) + + # Assert + self._verify_built_artifacts() + + def test_parent_package_mode_auto(self): + # Arrange + cmdlist = self.get_command_list( + use_container=self.use_container, + parameter_overrides={**self.overrides, "ParentPackageMode": "auto"}, + ) + + # Act + run_command(cmdlist, cwd=self.working_dir) + + # Assert + self._verify_built_artifacts() + + def _verify_built_artifacts(self): + [ + self._verify_built_artifact(self.default_build_dir, logical_id, self.expected_files_project_manifest) + for logical_id in [self.logical_id_one, self.logical_id_two] + ] + [ + self._verify_source_files( + self.default_build_dir, + self.expected_source_files, + logical_id, + parent_packages, + ) + for (logical_id, parent_packages) in [ + (self.logical_id_one, self.parent_packages_one), + (self.logical_id_two, self.parent_packages_two), + ] + ] + + def _verify_source_files( + self, build_dir: pathlib.Path, expected_files: set[str], function_logical_id: str, parent_packages: str + ) -> None: + relative_code_uri = parent_packages.replace(".", "/") + function_artifact_dir = build_dir / function_logical_id / relative_code_uri + all_artifacts = set( + map( + lambda artifact: str(object=artifact.relative_to(function_artifact_dir)), + function_artifact_dir.glob("*"), + ) + ) + actual_files = all_artifacts.intersection(expected_files) + self.assertEqual(actual_files, expected_files) + + class TestBuildCommand_ErrorCases(BuildIntegBase): def test_unsupported_runtime(self): overrides = {"Runtime": "unsupportedpython", "CodeUri": "Python"} diff --git a/tests/integration/testdata/buildcmd/PythonParentPackages/src/__init__.py b/tests/integration/testdata/buildcmd/PythonParentPackages/src/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/buildcmd/PythonParentPackages/src/fnone/__init__.py b/tests/integration/testdata/buildcmd/PythonParentPackages/src/fnone/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/buildcmd/PythonParentPackages/src/fnone/main.py b/tests/integration/testdata/buildcmd/PythonParentPackages/src/fnone/main.py new file mode 100644 index 0000000000..1968a3deab --- /dev/null +++ b/tests/integration/testdata/buildcmd/PythonParentPackages/src/fnone/main.py @@ -0,0 +1,5 @@ +import numpy + + +def handler(event, context): + return {"pi": "{0:.2f}".format(numpy.pi)} diff --git a/tests/integration/testdata/buildcmd/PythonParentPackages/src/fnone/requirements.txt b/tests/integration/testdata/buildcmd/PythonParentPackages/src/fnone/requirements.txt new file mode 100644 index 0000000000..a14e9494b3 --- /dev/null +++ b/tests/integration/testdata/buildcmd/PythonParentPackages/src/fnone/requirements.txt @@ -0,0 +1,8 @@ +# These are some hard packages to build. Using them here helps us verify that building works on various platforms + +# NOTE: Fixing to <1.20.3 as numpy1.20.3 started to use a new wheel naming convention (PEP 600) +numpy<1.20.3; python_version <= '3.9' +numpy==2.1.3; python_version >= '3.10' +# `cryptography` has a dependency on `pycparser` which, for some reason doesn't build inside a Docker container. +# Turning this off until we resolve this issue: https://github.com/awslabs/aws-lambda-builders/issues/29 +# cryptography~=2.4 diff --git a/tests/integration/testdata/buildcmd/PythonParentPackages/src/fntwo/__init__.py b/tests/integration/testdata/buildcmd/PythonParentPackages/src/fntwo/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/buildcmd/PythonParentPackages/src/fntwo/main.py b/tests/integration/testdata/buildcmd/PythonParentPackages/src/fntwo/main.py new file mode 100644 index 0000000000..1968a3deab --- /dev/null +++ b/tests/integration/testdata/buildcmd/PythonParentPackages/src/fntwo/main.py @@ -0,0 +1,5 @@ +import numpy + + +def handler(event, context): + return {"pi": "{0:.2f}".format(numpy.pi)} diff --git a/tests/integration/testdata/buildcmd/PythonParentPackages/src/fntwo/requirements.txt b/tests/integration/testdata/buildcmd/PythonParentPackages/src/fntwo/requirements.txt new file mode 100644 index 0000000000..a14e9494b3 --- /dev/null +++ b/tests/integration/testdata/buildcmd/PythonParentPackages/src/fntwo/requirements.txt @@ -0,0 +1,8 @@ +# These are some hard packages to build. Using them here helps us verify that building works on various platforms + +# NOTE: Fixing to <1.20.3 as numpy1.20.3 started to use a new wheel naming convention (PEP 600) +numpy<1.20.3; python_version <= '3.9' +numpy==2.1.3; python_version >= '3.10' +# `cryptography` has a dependency on `pycparser` which, for some reason doesn't build inside a Docker container. +# Turning this off until we resolve this issue: https://github.com/awslabs/aws-lambda-builders/issues/29 +# cryptography~=2.4 diff --git a/tests/integration/testdata/buildcmd/template_with_metadata_python.yaml b/tests/integration/testdata/buildcmd/template_with_metadata_python.yaml new file mode 100644 index 0000000000..fa7ff37c5e --- /dev/null +++ b/tests/integration/testdata/buildcmd/template_with_metadata_python.yaml @@ -0,0 +1,48 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Parameters: + Runtime: + Type: String + CodeUriOne: + Type: String + HandlerOne: + Type: String + CodeUriTwo: + Type: String + HandlerTwo: + Type: String + ParentPackagesOne: + Type: String + Default: '' + ParentPackagesTwo: + Type: String + Default: '' + ParentPackageMode: + Type: String + +Resources: + + FunctionOne: + Type: AWS::Serverless::Function + Properties: + CodeUri: !Ref CodeUriOne + Handler: !Ref HandlerOne + Runtime: !Ref Runtime + Timeout: 600 + Metadata: + BuildProperties: + ParentPackageMode: !Ref ParentPackageMode + ParentPackages: !Ref ParentPackagesOne + + FunctionTwo: + Type: AWS::Serverless::Function + Properties: + CodeUri: !Ref CodeUriTwo + Handler: !Ref HandlerTwo + Runtime: !Ref Runtime + Timeout: 600 + Metadata: + BuildProperties: + ParentPackageMode: !Ref ParentPackageMode + ParentPackages: !Ref ParentPackagesTwo diff --git a/tests/unit/lib/build_module/test_app_builder.py b/tests/unit/lib/build_module/test_app_builder.py index e2f15ebe22..93c2e5587a 100644 --- a/tests/unit/lib/build_module/test_app_builder.py +++ b/tests/unit/lib/build_module/test_app_builder.py @@ -1,38 +1,35 @@ -from collections import OrderedDict +import json import os import posixpath import sys - -import docker -import json - -from uuid import uuid4 - -from unittest import TestCase -from unittest.mock import Mock, MagicMock, call, mock_open, patch, ANY +from collections import OrderedDict from pathlib import Path, WindowsPath +from unittest import TestCase +from unittest.mock import ANY, MagicMock, Mock, call, mock_open, patch +from uuid import uuid4 +import docker from parameterized import parameterized -from samcli.lib.build.workflow_config import UnsupportedRuntimeException -from samcli.lib.providers.provider import ResourcesToBuildCollector, Function, FunctionBuildInfo +from samcli.commands.local.cli_common.user_exceptions import InvalidFunctionPropertyType from samcli.lib.build.app_builder import ( ApplicationBuilder, - UnsupportedBuilderLibraryVersionError, BuildError, - LambdaBuilderError, BuildInsideContainerError, - DockerfileOutSideOfContext, DockerBuildFailed, DockerConnectionError, + DockerfileOutSideOfContext, + LambdaBuilderError, + UnsupportedBuilderLibraryVersionError, ) -from samcli.commands.local.cli_common.user_exceptions import InvalidFunctionPropertyType -from samcli.lib.telemetry.event import EventName, EventTracker -from samcli.lib.utils.architecture import X86_64, ARM64 +from samcli.lib.build.workflow_config import UnsupportedRuntimeException +from samcli.lib.providers.provider import Function, FunctionBuildInfo, ResourcesToBuildCollector +from samcli.lib.telemetry.event import EventTracker +from samcli.lib.utils.architecture import ARM64, X86_64 from samcli.lib.utils.packagetype import IMAGE, ZIP from samcli.lib.utils.stream_writer import StreamWriter -from samcli.local.docker.manager import DockerImagePullFailedException from samcli.local.docker.container import ContainerContext +from samcli.local.docker.manager import DockerImagePullFailedException from tests.unit.lib.build_module.test_build_graph import generate_function @@ -2956,7 +2953,7 @@ def test_must_raise_on_unsupported_container(self, LambdaBuildContainerMock): container_mock.executable_name = "myexecutable" self.container_manager.run.side_effect = docker.errors.APIError( - "Bad Request: 'lambda-builders' " "executable file not found in $PATH" + "Bad Request: 'lambda-builders' executable file not found in $PATH" ) with self.assertRaises(UnsupportedBuilderLibraryVersionError) as ctx: @@ -3204,6 +3201,31 @@ def test_provided_metadata_get_working_dir_return_None(self): self.assertEqual(options, expected_properties) get_working_directory_path_mock.assert_called_once_with("base_dir", metadata, "source_dir", "scratch_dir") + def test_python_parent_package_mode_explicit(self): + build_properties = {"ParentPackageMode": "explicit", "ParentPackages": "src.function"} + metadata = {"BuildProperties": build_properties} + expected_properties = {"parent_python_packages": "src.function"} + options = ApplicationBuilder._get_build_options("Function", "python", "base_dir", "handler", "pip", metadata) + self.assertEqual(options, expected_properties) + + @parameterized.expand( + [ + ("src.function.index.handler", "src/function", "src.function"), + ("src.index.handler", "../../src", "src"), + ("index.handler", "src/function", None), + ("index.handler", None, None), + (None, "src/function", None), + ("app.function.index.handler", "src/function", None), + ] + ) + def test_python_parent_package_mode_auto(self, handler, source_code_path, expected_parent_packages): + build_properties = {"ParentPackageMode": "auto"} + metadata = {"BuildProperties": build_properties} + options = ApplicationBuilder._get_build_options( + "Function", "python", "base_dir", handler, "pip", metadata, source_code_path + ) + self.assertEqual(options, {"parent_python_packages": expected_parent_packages}) + class TestApplicationBuilderGetWorkingDirectoryPath(TestCase): def test_empty_metadata(self):