feat: add python option for creating parent packages during copy

samcli/lib/build/app_builder.py

@@ -829,6 +829,17 @@ def _get_build_options(
         if language == "rust" and "Binary" in build_props:
             options = options if options else {}
             options["artifact_executable_name"] = build_props["Binary"]
+
+        if language == "python" and "ParentPackageMode" in build_props:
+            options = options if options else {}
+            package_root_mode = build_props["ParentPackageMode"]
+            if package_root_mode == "auto":
+                options["parent_python_packages"] = ApplicationBuilder._infer_parent_python_packages(
+                    handler, source_code_path
+                )
+            elif package_root_mode == "explicit":
+                options["parent_python_packages"] = build_props.get("ParentPackages", None)
+
         return options
 
     @staticmethod
@@ -1045,3 +1056,51 @@ def _parse_builder_response(stdout_data: str, image_name: str) -> Dict:
             raise ValueError(msg)
 
         return cast(Dict, response)
+
+    @staticmethod
+    def _infer_parent_python_packages(handler: Optional[str], source_code_path: Optional[str]) -> Optional[str]:
+        """
+        Infers the parent python packages from the handler and source code path. The parent packages are the
+        packages are the union of the handler's parent packages and the source code path's parent packages.
+
+        Parameters
+        ----------
+        handler: str
+            The handler value of the function
+        source_code_path: str
+            The directory path to the source code of the function
+        Returns
+        -------
+        str
+
+        """
+        MODULE_PART_COUNT = 2  # file name and function name
+        if not handler or not source_code_path:
+            LOG.warning(
+                "Both function Handler and CodeUri must be provided when using PackageRootMode 'auto'."
+                + "Continuing without parent packages."
+            )
+            return None
+        if handler.count(".") < MODULE_PART_COUNT:
+            # Handler does not have any parent packages
+            LOG.warning("Handler '%s' does not have any parent python packages", handler)
+            return None
+
+        handler_parts = handler.split(".")[0:-MODULE_PART_COUNT]
+        code_path_parts = list(pathlib.Path(source_code_path).parts)
+
+        # Remove parts from the start of the path until we find the first part of the handler
+        while len(code_path_parts) > 0 and code_path_parts[0] != handler_parts[0]:
+            code_path_parts.pop(0)
+
+        if len(code_path_parts) > 0:
+            parent_packages = ".".join(code_path_parts[0 : len(handler_parts)])
+            LOG.debug("Inferred parent python packages '%s'", parent_packages)
+            return parent_packages
+        LOG.warning(
+            "Could not infer parent python packages from Handler '%s' and CodeUri '%s'."
+            + "Continuing without parent packages.",
+            handler,
+            source_code_path,
+        )
+        return None

tests/integration/buildcmd/test_build_cmd_python.py

@@ -1,27 +1,27 @@
 import logging
+import pathlib
 from typing import Set
 from unittest import skipIf
 from uuid import uuid4
 
 import pytest
 from parameterized import parameterized, parameterized_class
 
+from tests.integration.buildcmd.build_integ_base import (
+    BuildIntegBase,
+    BuildIntegPythonBase,
+)
 from tests.testing_utils import (
+    CI_OVERRIDE,
     IS_WINDOWS,
+    RUN_BY_CANARY,
     RUNNING_ON_CI,
     RUNNING_TEST_FOR_MASTER_ON_CI,
-    RUN_BY_CANARY,
-    CI_OVERRIDE,
-    run_command,
-    SKIP_DOCKER_TESTS,
     SKIP_DOCKER_BUILD,
     SKIP_DOCKER_MESSAGE,
+    SKIP_DOCKER_TESTS,
+    run_command,
 )
-from tests.integration.buildcmd.build_integ_base import (
-    BuildIntegBase,
-    BuildIntegPythonBase,
-)
-
 
 LOG = logging.getLogger(__name__)
 
@@ -476,7 +476,6 @@ class TestBuildCommand_PythonFunctions_With_Specified_Architecture(BuildIntegPyt
         ]
     )
     def test_with_default_requirements(self, runtime, codeuri, use_container, architecture):
-
         self._test_with_default_requirements(
             runtime, codeuri, use_container, self.test_data_path, architecture=architecture
         )
@@ -493,7 +492,6 @@ def test_with_default_requirements(self, runtime, codeuri, use_container, archit
     )
     @pytest.mark.al2023
     def test_with_default_requirements_al2023(self, runtime, codeuri, use_container, architecture):
-
         self._test_with_default_requirements(
             runtime, codeuri, use_container, self.test_data_path, architecture=architecture
         )
@@ -509,6 +507,100 @@ def test_invalid_architecture(self):
         self.assertIn("Architecture fake is not supported", str(process_execute.stderr))
 
 
+class TestBuildCommand_ParentPackages(BuildIntegPythonBase):
+    template = "template_with_metadata_python.yaml"
+    runtime = "python3.12"
+    use_container = False
+    prop = "CodeUri"
+
+    logical_id_one = "FunctionOne"
+    logical_id_two = "FunctionTwo"
+    parent_packages_one = "src.fnone"
+    parent_packages_two = "src.fntwo"
+    codeuri_one = "PythonParentPackages/src/fnone"
+    codeuri_two = "PythonParentPackages/src/fntwo"
+    handler_one = f"{parent_packages_one}.main.handler"
+    handler_two = f"{parent_packages_two}.main.handler"
+
+    overrides = {
+        "Runtime": runtime,
+        "CodeUriOne": codeuri_one,
+        "CodeUriTwo": codeuri_two,
+        "HandlerOne": handler_one,
+        "HandlerTwo": handler_two,
+    }
+
+    expected_files_project_manifest = {"numpy", "src"}
+    expected_source_files = {
+        "__init__.py",
+        "main.py",
+        "requirements.txt",
+    }
+
+    def test_parent_package_mode_explicit(self):
+        # Arrange
+        cmdlist = self.get_command_list(
+            use_container=self.use_container,
+            parameter_overrides={
+                **self.overrides,
+                "ParentPackageMode": "explicit",
+                "ParentPackagesOne": self.parent_packages_one,
+                "ParentPackagesTwo": self.parent_packages_two,
+            },
+        )
+
+        # Act
+        run_command(cmdlist, cwd=self.working_dir)
+
+        # Assert
+        self._verify_built_artifacts()
+
+    def test_parent_package_mode_auto(self):
+        # Arrange
+        cmdlist = self.get_command_list(
+            use_container=self.use_container,
+            parameter_overrides={**self.overrides, "ParentPackageMode": "auto"},
+        )
+
+        # Act
+        run_command(cmdlist, cwd=self.working_dir)
+
+        # Assert
+        self._verify_built_artifacts()
+
+    def _verify_built_artifacts(self):
+        [
+            self._verify_built_artifact(self.default_build_dir, logical_id, self.expected_files_project_manifest)
+            for logical_id in [self.logical_id_one, self.logical_id_two]
+        ]
+        [
+            self._verify_source_files(
+                self.default_build_dir,
+                self.expected_source_files,
+                logical_id,
+                parent_packages,
+            )
+            for (logical_id, parent_packages) in [
+                (self.logical_id_one, self.parent_packages_one),
+                (self.logical_id_two, self.parent_packages_two),
+            ]
+        ]
+
+    def _verify_source_files(
+        self, build_dir: pathlib.Path, expected_files: set[str], function_logical_id: str, parent_packages: str
+    ) -> None:
+        relative_code_uri = parent_packages.replace(".", "/")
+        function_artifact_dir = build_dir / function_logical_id / relative_code_uri
+        all_artifacts = set(
+            map(
+                lambda artifact: str(object=artifact.relative_to(function_artifact_dir)),
+                function_artifact_dir.glob("*"),
+            )
+        )
+        actual_files = all_artifacts.intersection(expected_files)
+        self.assertEqual(actual_files, expected_files)
+
+
 class TestBuildCommand_ErrorCases(BuildIntegBase):
     def test_unsupported_runtime(self):
         overrides = {"Runtime": "unsupportedpython", "CodeUri": "Python"}

tests/integration/testdata/buildcmd/PythonParentPackages/src/fnone/main.py

@@ -0,0 +1,5 @@
+import numpy
+
+
+def handler(event, context):
+    return {"pi": "{0:.2f}".format(numpy.pi)}

tests/integration/testdata/buildcmd/PythonParentPackages/src/fnone/requirements.txt

@@ -0,0 +1,8 @@
+# These are some hard packages to build. Using them here helps us verify that building works on various platforms
+
+# NOTE: Fixing to <1.20.3 as numpy1.20.3 started to use a new wheel naming convention (PEP 600)
+numpy<1.20.3; python_version <= '3.9'
+numpy==2.1.3; python_version >= '3.10'
+# `cryptography` has a dependency on `pycparser` which, for some reason doesn't build inside a Docker container.
+# Turning this off until we resolve this issue: https://github.com/awslabs/aws-lambda-builders/issues/29
+# cryptography~=2.4

tests/integration/testdata/buildcmd/PythonParentPackages/src/fntwo/main.py

@@ -0,0 +1,5 @@
+import numpy
+
+
+def handler(event, context):
+    return {"pi": "{0:.2f}".format(numpy.pi)}

tests/integration/testdata/buildcmd/PythonParentPackages/src/fntwo/requirements.txt

@@ -0,0 +1,8 @@
+# These are some hard packages to build. Using them here helps us verify that building works on various platforms
+
+# NOTE: Fixing to <1.20.3 as numpy1.20.3 started to use a new wheel naming convention (PEP 600)
+numpy<1.20.3; python_version <= '3.9'
+numpy==2.1.3; python_version >= '3.10'
+# `cryptography` has a dependency on `pycparser` which, for some reason doesn't build inside a Docker container.
+# Turning this off until we resolve this issue: https://github.com/awslabs/aws-lambda-builders/issues/29
+# cryptography~=2.4

tests/integration/testdata/buildcmd/template_with_metadata_python.yaml

@@ -0,0 +1,48 @@
+AWSTemplateFormatVersion : '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+
+Parameters:
+  Runtime:
+    Type: String
+  CodeUriOne:
+    Type: String
+  HandlerOne:
+    Type: String
+  CodeUriTwo:
+    Type: String
+  HandlerTwo:
+    Type: String
+  ParentPackagesOne:
+    Type: String
+    Default: ''
+  ParentPackagesTwo:
+    Type: String
+    Default: ''
+  ParentPackageMode:
+    Type: String
+
+Resources:
+
+  FunctionOne:
+    Type: AWS::Serverless::Function
+    Properties:
+      CodeUri: !Ref CodeUriOne
+      Handler: !Ref HandlerOne
+      Runtime: !Ref Runtime
+      Timeout: 600
+    Metadata:
+      BuildProperties:
+        ParentPackageMode: !Ref ParentPackageMode
+        ParentPackages: !Ref ParentPackagesOne
+
+  FunctionTwo:
+    Type: AWS::Serverless::Function
+    Properties:
+      CodeUri: !Ref CodeUriTwo
+      Handler: !Ref HandlerTwo
+      Runtime: !Ref Runtime
+      Timeout: 600
+    Metadata:
+      BuildProperties:
+        ParentPackageMode: !Ref ParentPackageMode
+        ParentPackages: !Ref ParentPackagesTwo

tests/unit/lib/build_module/test_app_builder.py

@@ -1,38 +1,35 @@
-from collections import OrderedDict
+import json
 import os
 import posixpath
 import sys
-
-import docker
-import json
-
-from uuid import uuid4
-
-from unittest import TestCase
-from unittest.mock import Mock, MagicMock, call, mock_open, patch, ANY
+from collections import OrderedDict
 from pathlib import Path, WindowsPath
+from unittest import TestCase
+from unittest.mock import ANY, MagicMock, Mock, call, mock_open, patch
+from uuid import uuid4
 
+import docker
 from parameterized import parameterized
 
-from samcli.lib.build.workflow_config import UnsupportedRuntimeException
-from samcli.lib.providers.provider import ResourcesToBuildCollector, Function, FunctionBuildInfo
+from samcli.commands.local.cli_common.user_exceptions import InvalidFunctionPropertyType
 from samcli.lib.build.app_builder import (
     ApplicationBuilder,
-    UnsupportedBuilderLibraryVersionError,
     BuildError,
-    LambdaBuilderError,
     BuildInsideContainerError,
-    DockerfileOutSideOfContext,
     DockerBuildFailed,
     DockerConnectionError,
+    DockerfileOutSideOfContext,
+    LambdaBuilderError,
+    UnsupportedBuilderLibraryVersionError,
 )
-from samcli.commands.local.cli_common.user_exceptions import InvalidFunctionPropertyType
-from samcli.lib.telemetry.event import EventName, EventTracker
-from samcli.lib.utils.architecture import X86_64, ARM64
+from samcli.lib.build.workflow_config import UnsupportedRuntimeException
+from samcli.lib.providers.provider import Function, FunctionBuildInfo, ResourcesToBuildCollector
+from samcli.lib.telemetry.event import EventTracker
+from samcli.lib.utils.architecture import ARM64, X86_64
 from samcli.lib.utils.packagetype import IMAGE, ZIP
 from samcli.lib.utils.stream_writer import StreamWriter
-from samcli.local.docker.manager import DockerImagePullFailedException
 from samcli.local.docker.container import ContainerContext
+from samcli.local.docker.manager import DockerImagePullFailedException
 from tests.unit.lib.build_module.test_build_graph import generate_function
 
 
@@ -2956,7 +2953,7 @@ def test_must_raise_on_unsupported_container(self, LambdaBuildContainerMock):
         container_mock.executable_name = "myexecutable"
 
         self.container_manager.run.side_effect = docker.errors.APIError(
-            "Bad Request: 'lambda-builders' " "executable file not found in $PATH"
+            "Bad Request: 'lambda-builders' executable file not found in $PATH"
         )
 
         with self.assertRaises(UnsupportedBuilderLibraryVersionError) as ctx:
@@ -3204,6 +3201,31 @@ def test_provided_metadata_get_working_dir_return_None(self):
         self.assertEqual(options, expected_properties)
         get_working_directory_path_mock.assert_called_once_with("base_dir", metadata, "source_dir", "scratch_dir")
 
+    def test_python_parent_package_mode_explicit(self):
+        build_properties = {"ParentPackageMode": "explicit", "ParentPackages": "src.function"}
+        metadata = {"BuildProperties": build_properties}
+        expected_properties = {"parent_python_packages": "src.function"}
+        options = ApplicationBuilder._get_build_options("Function", "python", "base_dir", "handler", "pip", metadata)
+        self.assertEqual(options, expected_properties)
+
+    @parameterized.expand(
+        [
+            ("src.function.index.handler", "src/function", "src.function"),
+            ("src.index.handler", "../../src", "src"),
+            ("index.handler", "src/function", None),
+            ("index.handler", None, None),
+            (None, "src/function", None),
+            ("app.function.index.handler", "src/function", None),
+        ]
+    )
+    def test_python_parent_package_mode_auto(self, handler, source_code_path, expected_parent_packages):
+        build_properties = {"ParentPackageMode": "auto"}
+        metadata = {"BuildProperties": build_properties}
+        options = ApplicationBuilder._get_build_options(
+            "Function", "python", "base_dir", handler, "pip", metadata, source_code_path
+        )
+        self.assertEqual(options, {"parent_python_packages": expected_parent_packages})
+
 
 class TestApplicationBuilderGetWorkingDirectoryPath(TestCase):
     def test_empty_metadata(self):