From 36fabad7d494bfbb806f908c80a7b046dfd972d5 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Tue, 21 Dec 2021 22:23:56 -0800 Subject: [PATCH 1/3] - Skip building Functions/Layers that are already built by CDK (Runtime Functions, Bundled Assets) - Add support to flag a function with SkipBuild metadata. --- samcli/commands/_utils/template.py | 10 + samcli/commands/build/build_context.py | 10 +- samcli/lib/providers/provider.py | 20 ++ .../samlib/resource_metadata_normalizer.py | 17 +- .../integration/buildcmd/build_integ_base.py | 2 + tests/integration/buildcmd/test_build_cmd.py | 99 ++++++ .../buildcmd/PreBuiltPython/__init__.py | 0 .../testdata/buildcmd/PreBuiltPython/main.py | 2 + .../__init__.py | 0 .../main.py | 2 + ...on_construct_with_skip_build_metadata.json | 318 ++++++++++++++++++ ...ed_template_python_function_construct.json | 317 +++++++++++++++++ ...on_construct_with_skip_build_metadata.json | 306 +++++++++++++++++ ...ed_template_python_function_construct.json | 305 +++++++++++++++++ ...te_cfn_function_flagged_to_skip_build.yaml | 21 ++ ...mplate_function_flagged_to_skip_build.yaml | 22 ++ tests/unit/commands/_utils/test_template.py | 55 +++ .../commands/buildcmd/test_build_context.py | 25 +- .../unit/commands/local/lib/test_provider.py | 15 + .../test_resource_metadata_normalizer.py | 36 ++ 20 files changed, 1575 insertions(+), 7 deletions(-) create mode 100644 tests/integration/testdata/buildcmd/PreBuiltPython/__init__.py create mode 100644 tests/integration/testdata/buildcmd/PreBuiltPython/main.py create mode 100644 tests/integration/testdata/buildcmd/asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9/__init__.py create mode 100644 tests/integration/testdata/buildcmd/asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9/main.py create mode 100644 tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_function_construct_with_skip_build_metadata.json create mode 100644 tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_python_function_construct.json create mode 100644 tests/integration/testdata/buildcmd/cdk_v2_synthesized_template_function_construct_with_skip_build_metadata.json create mode 100644 tests/integration/testdata/buildcmd/cdk_v2_synthesized_template_python_function_construct.json create mode 100644 tests/integration/testdata/buildcmd/template_cfn_function_flagged_to_skip_build.yaml create mode 100644 tests/integration/testdata/buildcmd/template_function_flagged_to_skip_build.yaml diff --git a/samcli/commands/_utils/template.py b/samcli/commands/_utils/template.py index 234e13f0bf..a8e7b244d9 100644 --- a/samcli/commands/_utils/template.py +++ b/samcli/commands/_utils/template.py @@ -10,6 +10,7 @@ from botocore.utils import set_value_from_jmespath from samcli.commands.exceptions import UserException +from samcli.lib.samlib.resource_metadata_normalizer import ASSET_PATH_METADATA_KEY from samcli.lib.utils.packagetype import ZIP, IMAGE from samcli.yamlhelper import yaml_parse, yaml_dump from samcli.lib.utils.resources import ( @@ -170,6 +171,15 @@ def _update_relative_paths(template_dict, original_root, new_root): set_value_from_jmespath(properties, path_prop_name, updated_path) + metadata = resource.get("Metadata", {}) + if ASSET_PATH_METADATA_KEY in metadata: + path = metadata.get(ASSET_PATH_METADATA_KEY, "") + updated_path = _resolve_relative_to(path, original_root, new_root) + if not updated_path: + # This path does not need to get updated + continue + metadata[ASSET_PATH_METADATA_KEY] = updated_path + # AWS::Includes can be anywhere within the template dictionary. Hence we need to recurse through the # dictionary in a separate method to find and update relative paths in there template_dict = _update_aws_include_relative_path(template_dict, original_root, new_root) diff --git a/samcli/commands/build/build_context.py b/samcli/commands/build/build_context.py index 7571a54e66..54438d85d5 100644 --- a/samcli/commands/build/build_context.py +++ b/samcli/commands/build/build_context.py @@ -429,7 +429,7 @@ def _collect_single_function_and_dependent_layers( return resource_collector.add_function(function) - resource_collector.add_layers([l for l in function.layers if l.build_method is not None]) + resource_collector.add_layers([l for l in function.layers if l.build_method is not None and not l.skip_build]) def _collect_single_buildable_layer( self, resource_identifier: str, resource_collector: ResourcesToBuildCollector @@ -465,6 +465,10 @@ def _is_function_buildable(function: Function): if isinstance(function.codeuri, str) and function.codeuri.endswith(".zip"): LOG.debug("Skip building zip function: %s", function.full_path) return False + # skip build the functions that marked as skip-build + if function.skip_build: + LOG.debug("Skip building pre-built function: %s", function.full_path) + return False return True @staticmethod @@ -477,4 +481,8 @@ def _is_layer_buildable(layer: LayerVersion): if isinstance(layer.codeuri, str) and layer.codeuri.endswith(".zip"): LOG.debug("Skip building zip layer: %s", layer.full_path) return False + # skip build the functions that marked as skip-build + if layer.skip_build: + LOG.debug("Skip building pre-built layer: %s", layer.full_path) + return False return True diff --git a/samcli/lib/providers/provider.py b/samcli/lib/providers/provider.py index 20897a6565..61b1a4f1fc 100644 --- a/samcli/lib/providers/provider.py +++ b/samcli/lib/providers/provider.py @@ -15,6 +15,7 @@ InvalidFunctionPropertyType, ) from samcli.lib.providers.sam_base_provider import SamBaseProvider +from samcli.lib.samlib.resource_metadata_normalizer import SAM_METADATA_SKIP_BUILD_KEY from samcli.lib.utils.architecture import X86_64 if TYPE_CHECKING: # pragma: no cover @@ -83,6 +84,15 @@ def full_path(self) -> str: """ return get_full_path(self.stack_path, self.function_id) + @property + def skip_build(self) -> bool: + """ + Check if the function metadata contains SkipBuild property to determines if SAM should skip building this + resource. It means that the customer is building the Lambda function code outside SAM, and the provided code + path is already built. + """ + return self.metadata.get(SAM_METADATA_SKIP_BUILD_KEY, False) if self.metadata else False + def get_build_dir(self, build_root_dir: str) -> str: """ Return the artifact directory based on the build root dir @@ -192,6 +202,7 @@ def __init__( self._build_architecture = cast(str, metadata.get("BuildArchitecture", X86_64)) self._compatible_architectures = compatible_architectures + self._skip_build = bool(metadata.get(SAM_METADATA_SKIP_BUILD_KEY, False)) @staticmethod def _compute_layer_version(is_defined_within_template: bool, arn: str) -> Optional[int]: @@ -260,6 +271,15 @@ def _compute_layer_name(is_defined_within_template: bool, arn: str) -> str: def stack_path(self) -> str: return self._stack_path + @property + def skip_build(self) -> bool: + """ + Check if the function metadata contains SkipBuild property to determines if SAM should skip building this + resource. It means that the customer is building the Lambda function code outside SAM, and the provided code + path is already built. + """ + return self._skip_build + @property def arn(self) -> str: return self._arn diff --git a/samcli/lib/samlib/resource_metadata_normalizer.py b/samcli/lib/samlib/resource_metadata_normalizer.py index 01e6ab8d90..7e1c745cb0 100644 --- a/samcli/lib/samlib/resource_metadata_normalizer.py +++ b/samcli/lib/samlib/resource_metadata_normalizer.py @@ -20,6 +20,9 @@ SAM_METADATA_DOCKER_CONTEXT_KEY = "DockerContext" SAM_METADATA_DOCKER_BUILD_ARGS_KEY = "DockerBuildArgs" +ASSET_BUNDLED_METADATA_KEY = "aws:asset:is-bundled" +SAM_METADATA_SKIP_BUILD_KEY = "SkipBuild" + LOG = logging.getLogger(__name__) @@ -45,7 +48,7 @@ def normalize(template_dict): if asset_property == IMAGE_ASSET_PROPERTY: asset_metadata = ResourceMetadataNormalizer._extract_image_asset_metadata(resource_metadata) - ResourceMetadataNormalizer._update_resource_image_asset_metadata(resource_metadata, asset_metadata) + ResourceMetadataNormalizer._update_resource_metadata(resource_metadata, asset_metadata) # For image-type functions, the asset path is expected to be the name of the Docker image. # When building, we set the name of the image to be the logical id of the function. asset_path = logical_id.lower() @@ -54,6 +57,16 @@ def normalize(template_dict): ResourceMetadataNormalizer._replace_property(asset_property, asset_path, resource, logical_id) + # Set SkipBuild metadata iff is-bundled metadata exists, and value is True + skip_build = resource_metadata.get(ASSET_BUNDLED_METADATA_KEY, False) + if skip_build: + ResourceMetadataNormalizer._update_resource_metadata( + resource_metadata, + { + SAM_METADATA_SKIP_BUILD_KEY: True, + }, + ) + @staticmethod def _replace_property(property_key, property_value, resource, logical_id): """ @@ -115,7 +128,7 @@ def _extract_image_asset_metadata(metadata): } @staticmethod - def _update_resource_image_asset_metadata(metadata, updated_values): + def _update_resource_metadata(metadata, updated_values): """ Update the metadata values for image-type lambda functions diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py index f0ab85714f..06485abc9e 100644 --- a/tests/integration/buildcmd/build_integ_base.py +++ b/tests/integration/buildcmd/build_integ_base.py @@ -193,6 +193,8 @@ def _verify_invoke_built_function(self, template_path, function_logical_id, over overrides, ] + LOG.info("Running invoke Command: {}".format(cmdlist)) + process_execute = run_command(cmdlist) process_execute.process.wait() diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index 4b11c600c6..2f7863437f 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -7,10 +7,12 @@ from typing import Set from unittest import skipIf +import jmespath import pytest from parameterized import parameterized, parameterized_class from samcli.lib.utils import osutils +from samcli.yamlhelper import yaml_parse from tests.testing_utils import ( IS_WINDOWS, RUNNING_ON_CI, @@ -83,6 +85,103 @@ def test_with_default_requirements(self, runtime, use_container): ) +@skipIf( + # Hits public ECR pull limitation, move it to canary tests + ((not RUN_BY_CANARY) or (IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE), + "Skip build tests on windows when running in CI unless overridden", +) +@parameterized_class( + ("template", "SKIPPED_FUNCTION_LOGICAL_ID", "src_code_path", "src_code_prop", "metadata_key"), + [ + ("template_function_flagged_to_skip_build.yaml", "SkippedFunction", "PreBuiltPython", "CodeUri", None), + ("template_cfn_function_flagged_to_skip_build.yaml", "SkippedFunction", "PreBuiltPython", "Code", None), + ( + "cdk_v1_synthesized_template_python_function_construct.json", + "SkippedFunctionDA0220D7", + "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + None, + "aws:asset:path", + ), + ( + "cdk_v1_synthesized_template_function_construct_with_skip_build_metadata.json", + "SkippedFunctionDA0220D7", + "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + None, + "aws:asset:path", + ), + ( + "cdk_v2_synthesized_template_python_function_construct.json", + "SkippedFunctionDA0220D7", + "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + None, + "aws:asset:path", + ), + ( + "cdk_v2_synthesized_template_function_construct_with_skip_build_metadata.json", + "RandomSpaceFunction4F8564D0", + "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + None, + "aws:asset:path", + ), + ], +) +class TestSkipBuildingFlaggedFunctions(BuildIntegPythonBase): + template = "template_cfn_function_flagged_to_skip_build.yaml" + SKIPPED_FUNCTION_LOGICAL_ID = "SkippedFunction" + src_code_path = "PreBuiltPython" + src_code_prop = "Code" + metadata_key = None + + @pytest.mark.flaky(reruns=3) + def test_with_default_requirements(self): + self._validate_skipped_built_function( + self.default_build_dir, + self.SKIPPED_FUNCTION_LOGICAL_ID, + self.test_data_path, + self.src_code_path, + self.src_code_prop, + self.metadata_key, + ) + + def _validate_skipped_built_function( + self, build_dir, skipped_function_logical_id, relative_path, src_code_path, src_code_prop, metadata_key + ): + + cmdlist = self.get_command_list() + + LOG.info("Running Command: {}".format(cmdlist)) + run_command(cmdlist, cwd=self.working_dir) + + self.assertTrue(build_dir.exists(), "Build directory should be created") + + build_dir_files = os.listdir(str(build_dir)) + self.assertNotIn(skipped_function_logical_id, build_dir_files) + + expected_value = os.path.relpath( + os.path.normpath(os.path.join(str(relative_path), src_code_path)), + str(self.default_build_dir), + ) + + with open(self.built_template, "r") as fp: + template_dict = yaml_parse(fp.read()) + if src_code_prop: + self.assertEqual( + expected_value, + jmespath.search( + f"Resources.{skipped_function_logical_id}.Properties.{src_code_prop}", template_dict + ), + ) + if metadata_key: + metadata = jmespath.search(f"Resources.{skipped_function_logical_id}.Metadata", template_dict) + metadata = metadata if metadata else {} + self.assertEqual(expected_value, metadata.get(metadata_key, "")) + expected = "Hello World" + if not SKIP_DOCKER_TESTS: + self._verify_invoke_built_function( + self.built_template, skipped_function_logical_id, self._make_parameter_override_arg({}), expected + ) + + @skipIf( ((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE), "Skip build tests on windows when running in CI unless overridden", diff --git a/tests/integration/testdata/buildcmd/PreBuiltPython/__init__.py b/tests/integration/testdata/buildcmd/PreBuiltPython/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/buildcmd/PreBuiltPython/main.py b/tests/integration/testdata/buildcmd/PreBuiltPython/main.py new file mode 100644 index 0000000000..a4e0923dda --- /dev/null +++ b/tests/integration/testdata/buildcmd/PreBuiltPython/main.py @@ -0,0 +1,2 @@ +def handler(event, context): + return "Hello World" \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9/__init__.py b/tests/integration/testdata/buildcmd/asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/buildcmd/asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9/main.py b/tests/integration/testdata/buildcmd/asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9/main.py new file mode 100644 index 0000000000..a4e0923dda --- /dev/null +++ b/tests/integration/testdata/buildcmd/asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9/main.py @@ -0,0 +1,2 @@ +def handler(event, context): + return "Hello World" \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_function_construct_with_skip_build_metadata.json b/tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_function_construct_with_skip_build_metadata.json new file mode 100644 index 0000000000..7021b71865 --- /dev/null +++ b/tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_function_construct_with_skip_build_metadata.json @@ -0,0 +1,318 @@ +{ + "Resources": { + "SkippedFunctionServiceRole1AB2E270": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + }, + "Metadata": { + "aws:cdk:path": "CDKV1SupportDemoStack/SkippedFunction/ServiceRole/Resource" + } + }, + "SkippedFunctionDA0220D7": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParameters5e7469917894bd27674659946e1ff4b27e12f4fcec7ccf03deda8762fe1618ccS3Bucket7D9D92ED" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters5e7469917894bd27674659946e1ff4b27e12f4fcec7ccf03deda8762fe1618ccS3VersionKey53E96C82" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters5e7469917894bd27674659946e1ff4b27e12f4fcec7ccf03deda8762fe1618ccS3VersionKey53E96C82" + } + ] + } + ] + } + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "SkippedFunctionServiceRole1AB2E270", + "Arn" + ] + }, + "Handler": "main.handler", + "Runtime": "python3.7", + "Timeout": 120 + }, + "DependsOn": [ + "SkippedFunctionServiceRole1AB2E270" + ], + "Metadata": { + "aws:cdk:path": "CDKV1SupportDemoStack/SkippedFunction/Resource", + "aws:asset:path": "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + "aws:asset:is-bundled": false, + "aws:asset:property": "Code", + "SkipBuild": true + } + }, + "CDKMetadata": { + "Type": "AWS::CDK::Metadata", + "Properties": { + "Analytics": "v2:deflate64:H4sIAAAAAAAA/zWNwQ6CMBBEv4V7WSTAXTHxbPAL1nYlBdom3VYPTf9divE0byazOy203QCn6owfrqVamySdJ0iPgHIVF2YKO87azuLqLAcfZRDXl72jR0OBfDETsYteUuG9pXTQzmZRXqYNzVMhpFu0ssSl8+csNBpIk9uO06JZcFdjmWU41ncPY5QrhRGZchbWKYKFm3fbQztAXy2sde2jDdoQTD/9AhqmcqjXAAAA" + }, + "Metadata": { + "aws:cdk:path": "CDKV1SupportDemoStack/CDKMetadata/Default" + }, + "Condition": "CDKMetadataAvailable" + } + }, + "Parameters": { + "AssetParameters5e7469917894bd27674659946e1ff4b27e12f4fcec7ccf03deda8762fe1618ccS3Bucket7D9D92ED": { + "Type": "String", + "Description": "S3 bucket for asset \"5e7469917894bd27674659946e1ff4b27e12f4fcec7ccf03deda8762fe1618cc\"" + }, + "AssetParameters5e7469917894bd27674659946e1ff4b27e12f4fcec7ccf03deda8762fe1618ccS3VersionKey53E96C82": { + "Type": "String", + "Description": "S3 key for asset version \"5e7469917894bd27674659946e1ff4b27e12f4fcec7ccf03deda8762fe1618cc\"" + }, + "AssetParameters5e7469917894bd27674659946e1ff4b27e12f4fcec7ccf03deda8762fe1618ccArtifactHashC9EC1F43": { + "Type": "String", + "Description": "Artifact hash for asset \"5e7469917894bd27674659946e1ff4b27e12f4fcec7ccf03deda8762fe1618cc\"" + } + }, + "Conditions": { + "CDKMetadataAvailable": { + "Fn::Or": [ + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "af-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ca-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-northwest-1" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-3" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "me-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "sa-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-2" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-2" + ] + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_python_function_construct.json b/tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_python_function_construct.json new file mode 100644 index 0000000000..81a26d5243 --- /dev/null +++ b/tests/integration/testdata/buildcmd/cdk_v1_synthesized_template_python_function_construct.json @@ -0,0 +1,317 @@ +{ + "Resources": { + "SkippedFunctionServiceRole1AB2E270": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + }, + "Metadata": { + "aws:cdk:path": "CDKV1SupportDemoStack/SkippedFunction/ServiceRole/Resource" + } + }, + "SkippedFunctionDA0220D7": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParameters7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9S3BucketD34C40A8" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9S3VersionKeyA6BD1318" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9S3VersionKeyA6BD1318" + } + ] + } + ] + } + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "SkippedFunctionServiceRole1AB2E270", + "Arn" + ] + }, + "FunctionName": "SkippedFunction", + "Handler": "main.handler", + "Runtime": "python3.9" + }, + "DependsOn": [ + "SkippedFunctionServiceRole1AB2E270" + ], + "Metadata": { + "aws:cdk:path": "CDKV1SupportDemoStack/SkippedFunction/Resource", + "aws:asset:path": "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + "aws:asset:is-bundled": true, + "aws:asset:property": "Code" + } + }, + "CDKMetadata": { + "Type": "AWS::CDK::Metadata", + "Properties": { + "Analytics": "v2:deflate64:H4sIAAAAAAAA/zWNwQ6CMBBEv4V7WSTCXTHxbPAL1nbFgrSmu9UYwr9LIZ5mJnkzU0K5r2GXHfDDuTZDMWkfCKaroB7UkZlksZ11nTp5xxKiFnW6uwsGHEkopNAS+xg0Jb9Qxor1blZpcnrieDOYv77y8A4uq5yj0wlRFkeYWv9cm0lnxfsc0yvDer5kaKIeSBpkUtsaLPR/Y56V84ag5+JdVlDWUGU9W5uH6MSOBO2mP+nUMXrpAAAA" + }, + "Metadata": { + "aws:cdk:path": "CDKV1SupportDemoStack/CDKMetadata/Default" + }, + "Condition": "CDKMetadataAvailable" + } + }, + "Parameters": { + "AssetParameters7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9S3BucketD34C40A8": { + "Type": "String", + "Description": "S3 bucket for asset \"7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9\"" + }, + "AssetParameters7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9S3VersionKeyA6BD1318": { + "Type": "String", + "Description": "S3 key for asset version \"7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9\"" + }, + "AssetParameters7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9ArtifactHashF30CA7AA": { + "Type": "String", + "Description": "Artifact hash for asset \"7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9\"" + } + }, + "Conditions": { + "CDKMetadataAvailable": { + "Fn::Or": [ + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "af-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ca-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-northwest-1" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-3" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "me-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "sa-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-2" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-2" + ] + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/cdk_v2_synthesized_template_function_construct_with_skip_build_metadata.json b/tests/integration/testdata/buildcmd/cdk_v2_synthesized_template_function_construct_with_skip_build_metadata.json new file mode 100644 index 0000000000..a1314c468f --- /dev/null +++ b/tests/integration/testdata/buildcmd/cdk_v2_synthesized_template_function_construct_with_skip_build_metadata.json @@ -0,0 +1,306 @@ +{ + "Resources": { + "RandomSpaceFunctionServiceRole223B050F": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + }, + "Metadata": { + "aws:cdk:path": "CDKV2SupportDemoStack/RandomSpaceFunction/ServiceRole/Resource" + } + }, + "RandomSpaceFunction4F8564D0": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "5e7469917894bd27674659946e1ff4b27e12f4fcec7ccf03deda8762fe1618cc.zip" + }, + "Role": { + "Fn::GetAtt": [ + "RandomSpaceFunctionServiceRole223B050F", + "Arn" + ] + }, + "Handler": "main.handler", + "Runtime": "python3.7", + "Timeout": 120 + }, + "DependsOn": [ + "RandomSpaceFunctionServiceRole223B050F" + ], + "Metadata": { + "aws:cdk:path": "CDKV2SupportDemoStack/RandomSpaceFunction/Resource", + "aws:asset:path": "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + "aws:asset:is-bundled": false, + "aws:asset:property": "Code", + "SkipBuild": true + } + }, + "CDKMetadata": { + "Type": "AWS::CDK::Metadata", + "Properties": { + "Analytics": "v2:deflate64:H4sIAAAAAAAA/zWMyw2DMAxAZ+Ee3ELpAAWpA9ABkElSZD6OhJP2ELF7A6inZz/Lr4QSrhl+JddmymfqIb486kkl1cUZl94gxGdg7cmxat78nzdFuEBs3Wx3vXNTcutQxHqBx460Qx30ZH2Nks6HTP2BeDierLiw6iPQODZ0htkZC6NcPkUFxR2qbBSifA3sabHQnvwBxHdcX7oAAAA=" + }, + "Metadata": { + "aws:cdk:path": "CDKV2SupportDemoStack/CDKMetadata/Default" + }, + "Condition": "CDKMetadataAvailable" + } + }, + "Conditions": { + "CDKMetadataAvailable": { + "Fn::Or": [ + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "af-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ca-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-northwest-1" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-3" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "me-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "sa-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-2" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-2" + ] + } + ] + } + ] + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} diff --git a/tests/integration/testdata/buildcmd/cdk_v2_synthesized_template_python_function_construct.json b/tests/integration/testdata/buildcmd/cdk_v2_synthesized_template_python_function_construct.json new file mode 100644 index 0000000000..51de72fa52 --- /dev/null +++ b/tests/integration/testdata/buildcmd/cdk_v2_synthesized_template_python_function_construct.json @@ -0,0 +1,305 @@ +{ + "Resources": { + "SkippedFunctionServiceRole1AB2E270": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + }, + "Metadata": { + "aws:cdk:path": "CDKV2SupportDemoStack/SkippedFunction/ServiceRole/Resource" + } + }, + "SkippedFunctionDA0220D7": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9.zip" + }, + "Role": { + "Fn::GetAtt": [ + "SkippedFunctionServiceRole1AB2E270", + "Arn" + ] + }, + "FunctionName": "SkippedFunction", + "Handler": "main.handler", + "Runtime": "python3.9" + }, + "DependsOn": [ + "SkippedFunctionServiceRole1AB2E270" + ], + "Metadata": { + "aws:cdk:path": "CDKV2SupportDemoStack/SkippedFunction/Resource", + "aws:asset:path": "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + "aws:asset:is-bundled": true, + "aws:asset:property": "Code" + } + }, + "CDKMetadata": { + "Type": "AWS::CDK::Metadata", + "Properties": { + "Analytics": "v2:deflate64:H4sIAAAAAAAA/0WOSw6CMBBAz8K+HT7iXjFxbfAAZCgVyqclTKsxhLtbPomrl5m8vJkE5gSiAD/ERdXxXpUwPy2KjvlVMSscYM5NL9ntpVcujE4FEklLcF3hZ8ic6KTNkCTrcSgrBG/fnRZWGb2wzfPRWul660gybhJb82Z0pXYtgohjPzYIcXA5HgpX7k0+fm1j9KE8tuF/Q5tKQkvhO04hPkMatKQUn5y2apCQ7/wBHa5B2e0AAAA=" + }, + "Metadata": { + "aws:cdk:path": "CDKV2SupportDemoStack/CDKMetadata/Default" + }, + "Condition": "CDKMetadataAvailable" + } + }, + "Conditions": { + "CDKMetadataAvailable": { + "Fn::Or": [ + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "af-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-northeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ap-southeast-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "ca-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "cn-northwest-1" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-central-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-north-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-2" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "eu-west-3" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "me-south-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "sa-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-east-2" + ] + } + ] + }, + { + "Fn::Or": [ + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-1" + ] + }, + { + "Fn::Equals": [ + { + "Ref": "AWS::Region" + }, + "us-west-2" + ] + } + ] + } + ] + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/template_cfn_function_flagged_to_skip_build.yaml b/tests/integration/testdata/buildcmd/template_cfn_function_flagged_to_skip_build.yaml new file mode 100644 index 0000000000..0d914a47cb --- /dev/null +++ b/tests/integration/testdata/buildcmd/template_cfn_function_flagged_to_skip_build.yaml @@ -0,0 +1,21 @@ +AWSTemplateFormatVersion : '2010-09-09' + +Resources: + + Function: + Type: AWS::Lambda::Function + Properties: + Handler: main.handler + Runtime: python3.7 + Code: Python + Timeout: 600 + + SkippedFunction: + Type: AWS::Lambda::Function + Properties: + Handler: main.handler + Runtime: python3.7 + Code: PreBuiltPython + Timeout: 600 + Metadata: + SkipBuild: true diff --git a/tests/integration/testdata/buildcmd/template_function_flagged_to_skip_build.yaml b/tests/integration/testdata/buildcmd/template_function_flagged_to_skip_build.yaml new file mode 100644 index 0000000000..c45159e755 --- /dev/null +++ b/tests/integration/testdata/buildcmd/template_function_flagged_to_skip_build.yaml @@ -0,0 +1,22 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Resources: + + Function: + Type: AWS::Serverless::Function + Properties: + Handler: main.handler + Runtime: python3.7 + CodeUri: Python + Timeout: 600 + + SkippedFunction: + Type: AWS::Serverless::Function + Properties: + Handler: main.handler + Runtime: python3.7 + CodeUri: PreBuiltPython + Timeout: 600 + Metadata: + SkipBuild: true diff --git a/tests/unit/commands/_utils/test_template.py b/tests/unit/commands/_utils/test_template.py index 7262ea261c..ee2eef8bd0 100644 --- a/tests/unit/commands/_utils/test_template.py +++ b/tests/unit/commands/_utils/test_template.py @@ -200,6 +200,61 @@ def test_must_update_relative_resource_paths(self, resource_type, properties): self.maxDiff = None self.assertEqual(result, expected_template_dict) + @parameterized.expand([(resource_type, props) for resource_type, props in RESOURCES_WITH_LOCAL_PATHS.items()]) + def test_must_update_relative_resource_metadata_paths(self, resource_type, properties): + for propname in properties: + template_dict = { + "Resources": { + "MyResourceWithRelativePath": { + "Type": resource_type, + "Properties": {}, + "Metadata": {"aws:asset:path": self.curpath}, + }, + "MyResourceWithS3Path": { + "Type": resource_type, + "Properties": {propname: self.s3path}, + "Metadata": {}, + }, + "MyResourceWithAbsolutePath": { + "Type": resource_type, + "Properties": {propname: self.abspath}, + "Metadata": {"aws:asset:path": self.abspath}, + }, + "MyResourceWithInvalidPath": { + "Type": resource_type, + "Properties": { + # Path is not a string + propname: {"foo": "bar"} + }, + }, + "MyResourceWithoutProperties": {"Type": resource_type}, + "UnsupportedResourceType": {"Type": "AWS::Ec2::Instance", "Properties": {"Code": "bar"}}, + "ResourceWithoutType": {"foo": "bar"}, + }, + "Parameters": {"a": "b"}, + } + + set_value_from_jmespath( + template_dict, f"Resources.MyResourceWithRelativePath.Properties.{propname}", self.curpath + ) + + expected_template_dict = copy.deepcopy(template_dict) + + set_value_from_jmespath( + expected_template_dict, + f"Resources.MyResourceWithRelativePath.Properties.{propname}", + self.expected_result, + ) + + expected_template_dict["Resources"]["MyResourceWithRelativePath"]["Metadata"][ + "aws:asset:path" + ] = self.expected_result + + result = _update_relative_paths(template_dict, self.src, self.dest) + + self.maxDiff = None + self.assertEqual(result, expected_template_dict) + @parameterized.expand([(resource_type, props) for resource_type, props in RESOURCES_WITH_IMAGE_COMPONENT.items()]) def test_must_skip_image_components(self, resource_type, properties): for propname in properties: diff --git a/tests/unit/commands/buildcmd/test_build_context.py b/tests/unit/commands/buildcmd/test_build_context.py index c30e17cb20..c7f7d3a629 100644 --- a/tests/unit/commands/buildcmd/test_build_context.py +++ b/tests/unit/commands/buildcmd/test_build_context.py @@ -356,17 +356,25 @@ def test_must_return_many_functions_to_build( func2 = DummyFunction("func2") func3_skipped = DummyFunction("func3", inlinecode="def handler(): pass", codeuri=None) func4_skipped = DummyFunction("func4", codeuri="packaged_function.zip") + func7_skipped = DummyFunction("func7", skip_build=True) func_provider_mock = Mock() - func_provider_mock.get_all.return_value = [func1, func2, func3_skipped, func4_skipped] + func_provider_mock.get_all.return_value = [ + func1, + func2, + func3_skipped, + func4_skipped, + func7_skipped, + ] funcprovider = SamFunctionProviderMock.return_value = func_provider_mock layer1 = DummyLayer("layer1", "buildMethod") layer2_skipped = DummyLayer("layer1", None) layer3_skipped = DummyLayer("layer1", "buildMethod", codeuri="packaged_function.zip") + layer4_skipped = DummyLayer("layer4", "buildMethod", skip_build=True) layer_provider_mock = Mock() - layer_provider_mock.get_all.return_value = [layer1, layer2_skipped, layer3_skipped] + layer_provider_mock.get_all.return_value = [layer1, layer2_skipped, layer3_skipped, layer4_skipped] layerprovider = SamLayerProviderMock.return_value = layer_provider_mock base_dir = pathlib_mock.Path.return_value.resolve.return_value.parent = "basedir" @@ -919,17 +927,26 @@ def test_must_catch_function_not_found_exception( class DummyLayer: - def __init__(self, name, build_method, codeuri="layer_src"): + def __init__(self, name, build_method, codeuri="layer_src", skip_build=False): self.name = name self.build_method = build_method self.codeuri = codeuri self.full_path = Mock() + self.skip_build = skip_build class DummyFunction: - def __init__(self, name, layers=[], inlinecode=None, codeuri="src"): + def __init__( + self, + name, + layers=[], + inlinecode=None, + codeuri="src", + skip_build=False, + ): self.name = name self.layers = layers self.inlinecode = inlinecode self.codeuri = codeuri self.full_path = Mock() + self.skip_build = skip_build diff --git a/tests/unit/commands/local/lib/test_provider.py b/tests/unit/commands/local/lib/test_provider.py index eba0e8da45..12144ecc31 100644 --- a/tests/unit/commands/local/lib/test_provider.py +++ b/tests/unit/commands/local/lib/test_provider.py @@ -98,6 +98,21 @@ def test_invalid_architecture(self): self.function.architecture self.assertEqual(str(e.exception), "Function name property Architectures should be a list of length 1") + def test_skip_build_is_false_if_metadata_is_None(self): + self.assertFalse(self.function.skip_build) + + def test_skip_build_is_false_if_metadata_is_empty(self): + self.function = self.function._replace(metadata={}) + self.assertFalse(self.function.skip_build) + + def test_skip_build_is_false_if_skip_build_metadata_flag_is_false(self): + self.function = self.function._replace(metadata={"SkipBuild": False}) + self.assertFalse(self.function.skip_build) + + def test_skip_build_is_false_if_skip_build_metadata_flag_is_true(self): + self.function = self.function._replace(metadata={"SkipBuild": True}) + self.assertTrue(self.function.skip_build) + class TestLayerVersion(TestCase): @parameterized.expand( diff --git a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py index c03f42ccdb..320d800cf1 100644 --- a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py +++ b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py @@ -151,3 +151,39 @@ def test_replace_of_property_that_does_not_exist(self): ResourceMetadataNormalizer.normalize(template_data) self.assertEqual("new path", template_data["Resources"]["Function1"]["Properties"]["Code"]) + + def test_set_skip_build_metadata_for_bundled_assets_metadata_equals_true(self): + template_data = { + "Resources": { + "Function1": { + "Properties": {"Code": "some value"}, + "Metadata": { + "aws:asset:path": "new path", + "aws:asset:property": "Code", + "aws:asset:is-bundled": True, + }, + } + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + self.assertTrue(template_data["Resources"]["Function1"]["Metadata"]["SkipBuild"]) + + def test_no_skip_build_metadata_for_bundled_assets_metadata_equals_false(self): + template_data = { + "Resources": { + "Function1": { + "Properties": {"Code": "some value"}, + "Metadata": { + "aws:asset:path": "new path", + "aws:asset:property": "Code", + "aws:asset:is-bundled": False, + }, + } + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + self.assertIsNone(template_data["Resources"]["Function1"]["Metadata"].get("SkipBuild")) From 96df2b1d80983260abaf9e17539211a3cc205702 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Tue, 21 Dec 2021 22:23:56 -0800 Subject: [PATCH 2/3] - Skip building Functions/Layers that are already built by CDK (Runtime Functions, Bundled Assets) - Add support to flag a function with SkipBuild metadata. --- tests/integration/buildcmd/test_build_cmd.py | 98 ++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index f533dddb4d..51f8ce8e05 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -9,6 +9,7 @@ import jmespath import docker +import jmespath import pytest from parameterized import parameterized, parameterized_class @@ -235,6 +236,103 @@ def test_with_default_requirements(self, runtime): ) +@skipIf( + # Hits public ECR pull limitation, move it to canary tests + ((not RUN_BY_CANARY) or (IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE), + "Skip build tests on windows when running in CI unless overridden", +) +@parameterized_class( + ("template", "SKIPPED_FUNCTION_LOGICAL_ID", "src_code_path", "src_code_prop", "metadata_key"), + [ + ("template_function_flagged_to_skip_build.yaml", "SkippedFunction", "PreBuiltPython", "CodeUri", None), + ("template_cfn_function_flagged_to_skip_build.yaml", "SkippedFunction", "PreBuiltPython", "Code", None), + ( + "cdk_v1_synthesized_template_python_function_construct.json", + "SkippedFunctionDA0220D7", + "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + None, + "aws:asset:path", + ), + ( + "cdk_v1_synthesized_template_function_construct_with_skip_build_metadata.json", + "SkippedFunctionDA0220D7", + "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + None, + "aws:asset:path", + ), + ( + "cdk_v2_synthesized_template_python_function_construct.json", + "SkippedFunctionDA0220D7", + "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + None, + "aws:asset:path", + ), + ( + "cdk_v2_synthesized_template_function_construct_with_skip_build_metadata.json", + "RandomSpaceFunction4F8564D0", + "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", + None, + "aws:asset:path", + ), + ], +) +class TestSkipBuildingFlaggedFunctions(BuildIntegPythonBase): + template = "template_cfn_function_flagged_to_skip_build.yaml" + SKIPPED_FUNCTION_LOGICAL_ID = "SkippedFunction" + src_code_path = "PreBuiltPython" + src_code_prop = "Code" + metadata_key = None + + @pytest.mark.flaky(reruns=3) + def test_with_default_requirements(self): + self._validate_skipped_built_function( + self.default_build_dir, + self.SKIPPED_FUNCTION_LOGICAL_ID, + self.test_data_path, + self.src_code_path, + self.src_code_prop, + self.metadata_key, + ) + + def _validate_skipped_built_function( + self, build_dir, skipped_function_logical_id, relative_path, src_code_path, src_code_prop, metadata_key + ): + + cmdlist = self.get_command_list() + + LOG.info("Running Command: {}".format(cmdlist)) + run_command(cmdlist, cwd=self.working_dir) + + self.assertTrue(build_dir.exists(), "Build directory should be created") + + build_dir_files = os.listdir(str(build_dir)) + self.assertNotIn(skipped_function_logical_id, build_dir_files) + + expected_value = os.path.relpath( + os.path.normpath(os.path.join(str(relative_path), src_code_path)), + str(self.default_build_dir), + ) + + with open(self.built_template, "r") as fp: + template_dict = yaml_parse(fp.read()) + if src_code_prop: + self.assertEqual( + expected_value, + jmespath.search( + f"Resources.{skipped_function_logical_id}.Properties.{src_code_prop}", template_dict + ), + ) + if metadata_key: + metadata = jmespath.search(f"Resources.{skipped_function_logical_id}.Metadata", template_dict) + metadata = metadata if metadata else {} + self.assertEqual(expected_value, metadata.get(metadata_key, "")) + expected = "Hello World" + if not SKIP_DOCKER_TESTS: + self._verify_invoke_built_function( + self.built_template, skipped_function_logical_id, self._make_parameter_override_arg({}), expected + ) + + @skipIf( ((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE), "Skip build tests on windows when running in CI unless overridden", From bcfdc888f25ded59a4798ba160a4f92c1189fb5f Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar <71043312+moelasmar@users.noreply.github.com> Date: Wed, 22 Dec 2021 14:03:37 -0800 Subject: [PATCH 3/3] merge and do black changes --- tests/integration/buildcmd/test_build_cmd.py | 97 ------------------- .../commands/buildcmd/test_build_context.py | 1 + 2 files changed, 1 insertion(+), 97 deletions(-) diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index 51f8ce8e05..a503881b9f 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -87,103 +87,6 @@ def test_with_default_requirements(self, runtime, use_container): ) -@skipIf( - # Hits public ECR pull limitation, move it to canary tests - ((not RUN_BY_CANARY) or (IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE), - "Skip build tests on windows when running in CI unless overridden", -) -@parameterized_class( - ("template", "SKIPPED_FUNCTION_LOGICAL_ID", "src_code_path", "src_code_prop", "metadata_key"), - [ - ("template_function_flagged_to_skip_build.yaml", "SkippedFunction", "PreBuiltPython", "CodeUri", None), - ("template_cfn_function_flagged_to_skip_build.yaml", "SkippedFunction", "PreBuiltPython", "Code", None), - ( - "cdk_v1_synthesized_template_python_function_construct.json", - "SkippedFunctionDA0220D7", - "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", - None, - "aws:asset:path", - ), - ( - "cdk_v1_synthesized_template_function_construct_with_skip_build_metadata.json", - "SkippedFunctionDA0220D7", - "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", - None, - "aws:asset:path", - ), - ( - "cdk_v2_synthesized_template_python_function_construct.json", - "SkippedFunctionDA0220D7", - "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", - None, - "aws:asset:path", - ), - ( - "cdk_v2_synthesized_template_function_construct_with_skip_build_metadata.json", - "RandomSpaceFunction4F8564D0", - "asset.7023fd47c81480184154c6e0e870d6920c50e35d8fae977873016832e127ded9", - None, - "aws:asset:path", - ), - ], -) -class TestSkipBuildingFlaggedFunctions(BuildIntegPythonBase): - template = "template_cfn_function_flagged_to_skip_build.yaml" - SKIPPED_FUNCTION_LOGICAL_ID = "SkippedFunction" - src_code_path = "PreBuiltPython" - src_code_prop = "Code" - metadata_key = None - - @pytest.mark.flaky(reruns=3) - def test_with_default_requirements(self): - self._validate_skipped_built_function( - self.default_build_dir, - self.SKIPPED_FUNCTION_LOGICAL_ID, - self.test_data_path, - self.src_code_path, - self.src_code_prop, - self.metadata_key, - ) - - def _validate_skipped_built_function( - self, build_dir, skipped_function_logical_id, relative_path, src_code_path, src_code_prop, metadata_key - ): - - cmdlist = self.get_command_list() - - LOG.info("Running Command: {}".format(cmdlist)) - run_command(cmdlist, cwd=self.working_dir) - - self.assertTrue(build_dir.exists(), "Build directory should be created") - - build_dir_files = os.listdir(str(build_dir)) - self.assertNotIn(skipped_function_logical_id, build_dir_files) - - expected_value = os.path.relpath( - os.path.normpath(os.path.join(str(relative_path), src_code_path)), - str(self.default_build_dir), - ) - - with open(self.built_template, "r") as fp: - template_dict = yaml_parse(fp.read()) - if src_code_prop: - self.assertEqual( - expected_value, - jmespath.search( - f"Resources.{skipped_function_logical_id}.Properties.{src_code_prop}", template_dict - ), - ) - if metadata_key: - metadata = jmespath.search(f"Resources.{skipped_function_logical_id}.Metadata", template_dict) - metadata = metadata if metadata else {} - self.assertEqual(expected_value, metadata.get(metadata_key, "")) - expected = "Hello World" - if not SKIP_DOCKER_TESTS: - self._verify_invoke_built_function( - self.built_template, skipped_function_logical_id, self._make_parameter_override_arg({}), expected - ) - - @skipIf( # Hits public ECR pull limitation, move it to canary tests ((not RUN_BY_CANARY) or (IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE), diff --git a/tests/unit/commands/buildcmd/test_build_context.py b/tests/unit/commands/buildcmd/test_build_context.py index 567ba9006a..ec2c7f6e8e 100644 --- a/tests/unit/commands/buildcmd/test_build_context.py +++ b/tests/unit/commands/buildcmd/test_build_context.py @@ -16,6 +16,7 @@ UnsupportedBuilderLibraryVersionError, BuildInsideContainerError, ContainerBuildNotSupported, + ApplicationBuildResult, ) from samcli.lib.build.workflow_config import UnsupportedRuntimeException from samcli.local.lambdafn.exceptions import FunctionNotFound