Feat: added integration tests, container tag check and layer support

samcli/commands/build/click_container.py

@@ -0,0 +1,19 @@
+"""
+Module to check container based cli parameters
+"""
+import click
+
+
+class ContainerOptions(click.Option):
+    """
+    Preprocessing checks for presence of --use-container flag for container based options.
+    """
+
+    def handle_parse_result(self, ctx, opts, args):
+        if "use_container" not in opts and self.name in opts:
+            opt_name = self.name.replace("_", "-")
+            msg = f"Missing required parameter, need the --use-container flag in order to use --{opt_name} flag."
+            raise click.UsageError(msg)
+        # To make sure no unser input prompting happens
+        self.prompt = None
+        return super().handle_parse_result(ctx, opts, args)

samcli/commands/build/command.py

@@ -20,6 +20,7 @@
 from samcli.cli.cli_config_file import configuration_option, TomlProvider
 from samcli.lib.utils.version_checker import check_newer_version
 from samcli.commands.build.exceptions import InvalidBuildImageException
+from samcli.commands.build.click_container import ContainerOptions
 
 LOG = logging.getLogger(__name__)
 
@@ -117,27 +118,30 @@
     help="Input environment variables through command line to pass into build containers, you can either "
     "input function specific format (FuncName.VarName=Value) or global format (VarName=Value). e.g., "
     "sam build --use-container --container-env-var Func1.VAR1=value1 --container-env-var VAR2=value2",
+    cls=ContainerOptions,
 )
 @click.option(
     "--container-env-var-file",
     "-ef",
     default=None,
     type=click.Path(),  # Must be a json file
     help="Path to environment variable json file (e.g., env_vars.json) to pass into build containers",
+    cls=ContainerOptions,
 )
 @click.option(
     "--build-image",
     "-bi",
     default=None,
     multiple=True,  # Can pass in multiple build images
     required=False,
-    help="Container image URIs for building functions. "
-    "You can specify for all functions with just the image URI "
+    help="Container image URIs for building functions/layers. "
+    "You can specify for all functions/layers with just the image URI "
     "(--build-image public.ecr.aws/sam/build-nodejs14.x:latest). "
     "You can specify for each individual function with "
     "(--build-image FunctionLogicalID=public.ecr.aws/sam/build-nodejs14.x:latest). "
     "A combination of the two can be used. If a function does not have build image specified or "
     "an image URI for all functions, the default SAM CLI build images will be used.",
+    cls=ContainerOptions,
 )
 @click.option(
     "--parallel",

samcli/lib/build/app_builder.py

@@ -67,7 +67,7 @@ def __init__(
         docker_client: Optional[docker.DockerClient] = None,
         container_env_var: Optional[Dict] = None,
         container_env_var_file: Optional[str] = None,
-        build_images: Optional[dict] = None,
+        build_images: Optional[Dict] = None,
     ) -> None:
         """
         Initialize the class
@@ -125,7 +125,7 @@ def __init__(
         self._colored = Colored()
         self._container_env_var = container_env_var
         self._container_env_var_file = container_env_var_file
-        self._build_images = build_images
+        self._build_images = build_images or {}
 
     def build(self) -> Dict[str, str]:
         """
@@ -441,8 +441,11 @@ def _build_layer(
                     # Only set to this value if specified workflow is makefile
                     # which will result in config language as provided
                     build_runtime = compatible_runtimes[0]
+                # None key represents the global build image for all functions/layers
+                global_image = self._build_images.get(None)
+                image = self._build_images.get(layer_name, global_image)
                 self._build_function_on_container(
-                    config, code_dir, artifact_subdir, manifest_path, build_runtime, options, container_env_vars
+                    config, code_dir, artifact_subdir, manifest_path, build_runtime, options, container_env_vars, image
                 )
             else:
                 self._build_function_in_process(
@@ -520,12 +523,9 @@ def _build_function(  # pylint: disable=R1710
                 options = ApplicationBuilder._get_build_options(function_name, config.language, handler)
                 # By default prefer to build in-process for speed
                 if self._container_manager:
-                    image = None
-                    if self._build_images is not None:
-                        if function_name in self._build_images:
-                            image = self._build_images[function_name]
-                        elif None in self._build_images:
-                            image = self._build_images[None]
+                    # None represents the global build image for all functions/layers
+                    global_image = self._build_images.get(None)
+                    image = self._build_images.get(function_name, global_image)
 
                     return self._build_function_on_container(
                         config,
@@ -577,7 +577,7 @@ def _build_function_in_process(
         scratch_dir: str,
         manifest_path: str,
         runtime: str,
-        options: Optional[dict],
+        options: Optional[Dict],
     ) -> str:
 
         builder = LambdaBuilder(

tests/integration/buildcmd/build_integ_base.py

@@ -71,6 +71,7 @@ def get_command_list(
         parallel=False,
         container_env_var=None,
         container_env_var_file=None,
+        build_image=None,
     ):
 
         command_list = [self.cmd, "build"]
@@ -113,6 +114,9 @@ def get_command_list(
         if container_env_var_file:
             command_list += ["--container-env-var-file", container_env_var_file]
 
+        if build_image:
+            command_list += ["--build-image", build_image]
+
         return command_list
 
     def verify_docker_container_cleanedup(self, runtime):

tests/integration/buildcmd/test_build_cmd.py

@@ -7,6 +7,7 @@
 from unittest import skipIf
 from pathlib import Path
 from parameterized import parameterized, parameterized_class
+from subprocess import Popen, PIPE, TimeoutExpired
 
 import pytest
 
@@ -1837,3 +1838,47 @@ def test_nested_build(self, use_container, cached, parallel):
                     ("LocalNestedStack/Function2", {"pi": "3.14"}),
                 ],
             )
+
+
+@skipIf(
+    ((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE),
+    "Skip build tests on windows when running in CI unless overridden",
+)
+class TestBuildWithCustomBuildImage(BuildIntegBase):
+    template = "build_image_function.yaml"
+
+    @parameterized.expand(
+        [
+            ("use_container", None),
+            ("use_container", "amazon/aws-sam-cli-build-image-python3.7:latest"),
+        ]
+    )
+    @pytest.mark.flaky(reruns=3)
+    def test_custom_build_image_succeeds(self, use_container, build_image):
+        if use_container and SKIP_DOCKER_TESTS:
+            self.skipTest(SKIP_DOCKER_MESSAGE)
+
+        cmdlist = self.get_command_list(use_container=use_container, build_image=build_image)
+
+        command_result = run_command(cmdlist, cwd=self.working_dir)
+        stderr = command_result.stderr
+        process_stderr = stderr.strip()
+
+        self._verify_right_image_pulled(build_image, process_stderr)
+        self._verify_build_succeeds(self.default_build_dir)
+
+        self.verify_docker_container_cleanedup("python3.7")
+
+    def _verify_right_image_pulled(self, build_image, process_stderr):
+        image_name = build_image if build_image is not None else "public.ecr.aws/sam/build-python3.7:latest"
+        processed_name = bytes(image_name, encoding="utf-8")
+        self.assertIn(
+            processed_name,
+            process_stderr,
+        )
+
+    def _verify_build_succeeds(self, build_dir):
+        self.assertTrue(build_dir.exists(), "Build directory should be created")
+
+        build_dir_files = os.listdir(str(build_dir))
+        self.assertIn("BuildImageFunction", build_dir_files)

tests/integration/testdata/buildcmd/build_image_function.yaml

@@ -0,0 +1,15 @@
+AWSTemplateFormatVersion : '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+
+Globals:
+  Function:
+    Timeout: 20
+    MemorySize: 512
+
+Resources:
+  BuildImageFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      CodeUri: Python
+      Handler: main.handler
+      Runtime: python3.7

tests/unit/commands/buildcmd/test_container_options.py

@@ -0,0 +1,31 @@
+import click
+
+from unittest import TestCase
+from unittest.mock import Mock, patch, call
+from samcli.commands.build.click_container import ContainerOptions
+
+
+@patch("samcli.commands.build.click_container.ContainerOptions")
+class TestContainerOptionsSucceeds(TestCase):
+    ctx_mock = Mock()
+    opts = {"container_env_var": ["hi=in"], "use_container": True, "resource_logical_id": None}
+    ContainerOptionsMock = Mock()
+    ContainerOptionsMock.handle_parse_result.return_value = "value"
+
+    def test_container_options(self, ContainerOptionsMock):
+        self.assertEqual(self.ContainerOptionsMock.handle_parse_result(self.ctx_mock, self.opts, []), "value")
+
+
+class TestContainerOptionsFails(TestCase):
+    ctx_mock = Mock()
+    opts = {"container_env_var": ["hi=in"], "resource_logical_id": None}
+    args = ["--container-env-var"]
+    container_opt = ContainerOptions(args)
+
+    def test_container_options_failure(self):
+        with self.assertRaises(click.UsageError) as err:
+            self.container_opt.handle_parse_result(self.ctx_mock, self.opts, [])
+        self.assertEqual(
+            str(err.exception),
+            "Missing required parameter, need the --use-container flag in order to use --container-env-var flag.",
+        )

tests/unit/commands/samconfig/test_samconfig.py

@@ -114,6 +114,9 @@ def test_build(self, do_cli_mock):
             "docker_network": "mynetwork",
             "skip_pull_image": True,
             "parameter_overrides": "ParameterKey=Key,ParameterValue=Value ParameterKey=Key2,ParameterValue=Value2",
+            "container_env_var": (""),
+            "container_env_var_file": "file",
+            "build_image": (""),
         }
 
         with samconfig_parameters(["build"], self.scratch_dir, **config_values) as config_path:
@@ -146,7 +149,7 @@ def test_build(self, do_cli_mock):
                 {"Key": "Value", "Key2": "Value2"},
                 None,
                 (),
-                None,
+                "file",
                 (),
             )
 

tests/unit/lib/build_module/test_app_builder.py

@@ -419,6 +419,73 @@ def test_must_build_layer_in_container(self, get_layer_subfolder_mock, osutils_m
             "python3.8",
             None,
             None,
+            None,
+        )
+
+    @patch("samcli.lib.build.app_builder.get_workflow_config")
+    @patch("samcli.lib.build.app_builder.osutils")
+    @patch("samcli.lib.build.app_builder.get_layer_subfolder")
+    def test_must_build_layer_in_container_with_global_build_image(
+        self, get_layer_subfolder_mock, osutils_mock, get_workflow_config_mock
+    ):
+        self.builder._container_manager = self.container_manager
+        get_layer_subfolder_mock.return_value = "python"
+        config_mock = Mock()
+        config_mock.manifest_name = "manifest_name"
+
+        scratch_dir = "scratch"
+        osutils_mock.mkdir_temp.return_value.__enter__ = Mock(return_value=scratch_dir)
+        osutils_mock.mkdir_temp.return_value.__exit__ = Mock()
+
+        get_workflow_config_mock.return_value = config_mock
+        build_function_on_container_mock = Mock()
+
+        build_images = {None: "test_image"}
+        self.builder._build_images = build_images
+        self.builder._build_function_on_container = build_function_on_container_mock
+        self.builder._build_layer("layer_name", "code_uri", "python3.8", ["python3.8"], "full_path")
+        build_function_on_container_mock.assert_called_once_with(
+            config_mock,
+            PathValidator("code_uri"),
+            PathValidator("python"),
+            PathValidator("manifest_name"),
+            "python3.8",
+            None,
+            None,
+            "test_image",
+        )
+
+    @patch("samcli.lib.build.app_builder.get_workflow_config")
+    @patch("samcli.lib.build.app_builder.osutils")
+    @patch("samcli.lib.build.app_builder.get_layer_subfolder")
+    def test_must_build_layer_in_container_with_specific_build_image(
+        self, get_layer_subfolder_mock, osutils_mock, get_workflow_config_mock
+    ):
+        self.builder._container_manager = self.container_manager
+        get_layer_subfolder_mock.return_value = "python"
+        config_mock = Mock()
+        config_mock.manifest_name = "manifest_name"
+
+        scratch_dir = "scratch"
+        osutils_mock.mkdir_temp.return_value.__enter__ = Mock(return_value=scratch_dir)
+        osutils_mock.mkdir_temp.return_value.__exit__ = Mock()
+
+        get_workflow_config_mock.return_value = config_mock
+        build_function_on_container_mock = Mock()
+
+        build_images = {"layer_name": "test_image"}
+        self.builder._build_images = build_images
+        self.builder._build_function_on_container = build_function_on_container_mock
+        self.builder._build_layer("layer_name", "code_uri", "python3.8", ["python3.8"], "full_path")
+        build_function_on_container_mock.assert_called_once_with(
+            config_mock,
+            PathValidator("code_uri"),
+            PathValidator("python"),
+            PathValidator("manifest_name"),
+            "python3.8",
+            None,
+            None,
+            "test_image",
         )