diff --git a/samcli/lib/intrinsic_resolver/intrinsics_symbol_table.py b/samcli/lib/intrinsic_resolver/intrinsics_symbol_table.py index e5d704214e..bd9b5e7e3d 100644 --- a/samcli/lib/intrinsic_resolver/intrinsics_symbol_table.py +++ b/samcli/lib/intrinsic_resolver/intrinsics_symbol_table.py @@ -325,6 +325,19 @@ def get_translation(self, logical_id, resource_attributes=IntrinsicResolver.REF) return None return logical_id_item + if logical_id in self._parameters and isinstance(logical_id_item, dict): + if "Ref" in logical_id_item: + # It is a parameter passed down from the parent + # here we use a special function "CrossStackRef" to refer a logicalID from other stacks + # ".." means the parent stack + return {"Ref": {"CrossStackRef": ["..", logical_id_item.get("Ref")]}} + if "Fn::Join" in logical_id_item: + delimiter, items = logical_id_item.get("Fn::Join") + if delimiter == "," and all("Ref" in item for item in items): + return [{"Ref": {"CrossStackRef": ["..", item.get("Ref")]}} for item in items] + + LOG.warning('"sam build" currently does not support the resolution of parameter %s.', logical_id_item) + return logical_id_item.get(resource_attributes) @staticmethod diff --git a/samcli/lib/providers/sam_function_provider.py b/samcli/lib/providers/sam_function_provider.py index b78b8f6c68..e12d9dcdd0 100644 --- a/samcli/lib/providers/sam_function_provider.py +++ b/samcli/lib/providers/sam_function_provider.py @@ -43,14 +43,14 @@ def __init__( """ self.stacks = stacks + self._use_raw_codeuri = use_raw_codeuri + self._ignore_code_extraction_warnings = ignore_code_extraction_warnings for stack in stacks: LOG.debug("%d resources found in the stack %s", len(stack.resources), stack.stack_path) # Store a map of function full_path to function information for quick reference - self.functions = SamFunctionProvider._extract_functions( - self.stacks, use_raw_codeuri, ignore_code_extraction_warnings - ) + self.functions = self._extract_functions() self._deprecated_runtimes = {"nodejs4.3", "nodejs6.10", "nodejs8.10", "dotnetcore2.0"} self._colored = Colored() @@ -103,23 +103,17 @@ def get_all(self) -> Iterator[Function]: for _, function in self.functions.items(): yield function - @staticmethod - def _extract_functions( - stacks: List[Stack], use_raw_codeuri: bool = False, ignore_code_extraction_warnings: bool = False - ) -> Dict[str, Function]: + def _extract_functions(self) -> Dict[str, Function]: """ Extracts and returns function information from the given dictionary of SAM/CloudFormation resources. This method supports functions defined with AWS::Serverless::Function and AWS::Lambda::Function - :param stacks: List of SAM/CloudFormation stacks to extract functions from - :param bool use_raw_codeuri: Do not resolve adjust core_uri based on the template path, use the raw uri. - :param bool ignore_code_extraction_warnings: suppress log statements on code extraction from resources. :return dict(string : samcli.commands.local.lib.provider.Function): Dictionary of function full_path to the Function configuration object """ result: Dict[str, Function] = {} # a dict with full_path as key and extracted function as value - for stack in stacks: + for stack in self.stacks: for name, resource in stack.resources.items(): resource_type = resource.get("Type") @@ -130,46 +124,36 @@ def _extract_functions( resource_properties["Metadata"] = resource_metadata if resource_type == SamFunctionProvider.SERVERLESS_FUNCTION: - layers = SamFunctionProvider._parse_layer_info( + layers = self._parse_layer_info( stack, resource_properties.get("Layers", []), - use_raw_codeuri, - ignore_code_extraction_warnings=ignore_code_extraction_warnings, ) - function = SamFunctionProvider._convert_sam_function_resource( + function = self._convert_sam_function_resource( stack, name, resource_properties, layers, - use_raw_codeuri, - ignore_code_extraction_warnings=ignore_code_extraction_warnings, ) result[function.full_path] = function elif resource_type == SamFunctionProvider.LAMBDA_FUNCTION: - layers = SamFunctionProvider._parse_layer_info( + layers = self._parse_layer_info( stack, resource_properties.get("Layers", []), - use_raw_codeuri, - ignore_code_extraction_warnings=ignore_code_extraction_warnings, - ) - function = SamFunctionProvider._convert_lambda_function_resource( - stack, name, resource_properties, layers, use_raw_codeuri ) + function = self._convert_lambda_function_resource(stack, name, resource_properties, layers) result[function.full_path] = function # We don't care about other resource types. Just ignore them return result - @staticmethod def _convert_sam_function_resource( + self, stack: Stack, name: str, resource_properties: Dict, layers: List[LayerVersion], - use_raw_codeuri: bool = False, - ignore_code_extraction_warnings: bool = False, ) -> Function: """ Converts a AWS::Serverless::Function resource to a Function configuration usable by the provider. @@ -201,7 +185,7 @@ def _convert_sam_function_resource( name, resource_properties, "CodeUri", - ignore_code_extraction_warnings=ignore_code_extraction_warnings, + ignore_code_extraction_warnings=self._ignore_code_extraction_warnings, ) LOG.debug("Found Serverless function with name='%s' and CodeUri='%s'", name, codeuri) elif packagetype == IMAGE: @@ -209,12 +193,11 @@ def _convert_sam_function_resource( LOG.debug("Found Serverless function with name='%s' and ImageUri='%s'", name, imageuri) return SamFunctionProvider._build_function_configuration( - stack, name, codeuri, resource_properties, layers, inlinecode, imageuri, use_raw_codeuri + stack, name, codeuri, resource_properties, layers, inlinecode, imageuri, self._use_raw_codeuri ) - @staticmethod def _convert_lambda_function_resource( - stack: Stack, name: str, resource_properties: Dict, layers: List[LayerVersion], use_raw_codeuri: bool = False + self, stack: Stack, name: str, resource_properties: Dict, layers: List[LayerVersion] ) -> Function: """ Converts a AWS::Lambda::Function resource to a Function configuration usable by the provider. @@ -227,8 +210,6 @@ def _convert_lambda_function_resource( Properties of this resource layers List(samcli.commands.local.lib.provider.Layer) List of the Layer objects created from the template and layer list defined on the function. - use_raw_codeuri - Do not resolve adjust core_uri based on the template path, use the raw uri. Returns ------- @@ -259,7 +240,7 @@ def _convert_lambda_function_resource( LOG.debug("Found Lambda function with name='%s' and Imageuri='%s'", name, imageuri) return SamFunctionProvider._build_function_configuration( - stack, name, codeuri, resource_properties, layers, inlinecode, imageuri, use_raw_codeuri + stack, name, codeuri, resource_properties, layers, inlinecode, imageuri, self._use_raw_codeuri ) @staticmethod @@ -330,12 +311,10 @@ def _build_function_configuration( codesign_config_arn=resource_properties.get("CodeSigningConfigArn", None), ) - @staticmethod def _parse_layer_info( + self, stack: Stack, list_of_layers: List[Any], - use_raw_codeuri: bool = False, - ignore_code_extraction_warnings: bool = False, ) -> List[LayerVersion]: """ Creates a list of Layer objects that are represented by the resources and the list of layers @@ -347,10 +326,6 @@ def _parse_layer_info( list_of_layers : List[Any] List of layers that are defined within the Layers Property on a function, layer can be defined as string or Dict, in case customers define it in other types, use "Any" here. - use_raw_codeuri : bool - Do not resolve adjust core_uri based on the template path, use the raw uri. - ignore_code_extraction_warnings : bool - Whether to print warning when codeuri is not a local pth Returns ------- @@ -387,8 +362,25 @@ def _parse_layer_info( # In the list of layers that is defined within a template, you can reference a LayerVersion resource. # When running locally, we need to follow that Ref so we can extract the local path to the layer code. if isinstance(layer, dict) and layer.get("Ref"): - layer_logical_id = cast(str, layer.get("Ref")) - layer_resource = stack.resources.get(layer_logical_id) + # Here layer refers to another resources. + # in a multi stack system, the layer might locate in the current stack or passed down from parents. + # for example, {"Ref": "layerLogicalID"} refers to the "layerLogicalID" in the current stack. + # {"Ref": {"CrossStackRef": ["..", "layerLogicalID"]} + # refers to the "layerLogicalID" in the parent stack. + # {"Ref": {"CrossStackRef": ["..", {"CrossStackRef": ["..", "layerLogicalID"]})} + # refers to the layer in the grandparent stack. + # here we use a while loop to remove all "CrossStackRef" keys + # and locate the underlying layer resource ID. + ref_stack = stack + layer_ref = layer.get("Ref") + while isinstance(layer_ref, dict) and "CrossStackRef" in layer_ref: + relative_stack_path, layer_ref = layer_ref["CrossStackRef"] + ref_stack = SamLocalStackProvider.resolve_stack(self.stacks, ref_stack, relative_stack_path) + + layer_logical_id = cast(str, layer_ref) + LOG.debug("resolved layer from %s to %s in stack %s", layer, layer_logical_id, ref_stack.stack_path) + + layer_resource = ref_stack.resources.get(layer_logical_id) if not layer_resource or layer_resource.get("Type", "") not in ( SamFunctionProvider.SERVERLESS_LAYER, SamFunctionProvider.LAMBDA_LAYER, @@ -405,12 +397,14 @@ def _parse_layer_info( if resource_type == SamFunctionProvider.SERVERLESS_LAYER: codeuri = SamFunctionProvider._extract_sam_function_codeuri( - layer_logical_id, layer_properties, "ContentUri", ignore_code_extraction_warnings + layer_logical_id, layer_properties, "ContentUri", self._ignore_code_extraction_warnings ) - if codeuri and not use_raw_codeuri: - LOG.debug("--base-dir is presented not, adjusting uri %s relative to %s", codeuri, stack.location) - codeuri = SamLocalStackProvider.normalize_resource_path(stack.location, codeuri) + if codeuri and not self._use_raw_codeuri: + LOG.debug( + "--base-dir is presented not, adjusting uri %s relative to %s", codeuri, ref_stack.location + ) + codeuri = SamLocalStackProvider.normalize_resource_path(ref_stack.location, codeuri) layers.append( LayerVersion( @@ -418,7 +412,7 @@ def _parse_layer_info( codeuri, compatible_runtimes, layer_resource.get("Metadata", None), - stack_path=stack.stack_path, + stack_path=ref_stack.stack_path, ) ) diff --git a/samcli/lib/providers/sam_stack_provider.py b/samcli/lib/providers/sam_stack_provider.py index 25053a1e52..a7bc988cb3 100644 --- a/samcli/lib/providers/sam_stack_provider.py +++ b/samcli/lib/providers/sam_stack_provider.py @@ -3,6 +3,7 @@ """ import logging import os +import posixpath from typing import Optional, Dict, cast, List, Iterator, Tuple from urllib.parse import unquote, urlparse @@ -340,3 +341,14 @@ def normalize_resource_path(stack_file_path: str, path: str) -> str: stack_file_path = os.path.relpath(os.path.realpath(stack_file_path)) return os.path.normpath(os.path.join(os.path.dirname(stack_file_path), path)) + + @staticmethod + def resolve_stack(stacks: List[Stack], base_stack: Stack, relative_stack_path: str) -> Stack: + target_stack_path = posixpath.normpath(posixpath.join(base_stack.stack_path, relative_stack_path)) + if target_stack_path == ".": + # root stack + target_stack_path = "" + try: + return next(stack for stack in stacks if stack.stack_path == target_stack_path) + except StopIteration as ex: + raise ValueError(f"Cannot find stack with path {target_stack_path}") from ex diff --git a/tests/unit/commands/local/lib/test_sam_function_provider.py b/tests/unit/commands/local/lib/test_sam_function_provider.py index 146dd3a512..e67fadd4e6 100644 --- a/tests/unit/commands/local/lib/test_sam_function_provider.py +++ b/tests/unit/commands/local/lib/test_sam_function_provider.py @@ -594,7 +594,7 @@ def test_must_extract_functions(self, get_template_mock, extract_mock): stack = make_root_stack(template, self.parameter_overrides) provider = SamFunctionProvider([stack]) - extract_mock.assert_called_with([stack], False, False) + extract_mock.assert_called_once() get_template_mock.assert_called_with(template, self.parameter_overrides) self.assertEqual(provider.functions, extract_result) @@ -609,7 +609,7 @@ def test_must_default_to_empty_resources(self, get_template_mock, extract_mock): stack = make_root_stack(template, self.parameter_overrides) provider = SamFunctionProvider([stack]) - extract_mock.assert_called_with([stack], False, False) # Empty Resources value must be passed + extract_mock.assert_called_once() self.assertEqual(provider.functions, extract_result) @@ -625,9 +625,9 @@ def test_must_work_for_sam_function(self, convert_mock, resources_mock): expected = {"A/B/C/Func1": convertion_result} stack = make_root_stack(None) - result = SamFunctionProvider._extract_functions([stack]) + result = SamFunctionProvider([stack]).functions self.assertEqual(expected, result) - convert_mock.assert_called_with(stack, "Func1", {"a": "b"}, [], False, ignore_code_extraction_warnings=False) + convert_mock.assert_called_with(stack, "Func1", {"a": "b"}, []) @patch("samcli.lib.providers.sam_function_provider.Stack.resources", new_callable=PropertyMock) @patch.object(SamFunctionProvider, "_convert_sam_function_resource") @@ -646,9 +646,9 @@ def test_must_work_with_no_properties(self, convert_mock, resources_mock): expected = {"A/B/C/Func1": convertion_result} stack = make_root_stack(None) - result = SamFunctionProvider._extract_functions([stack]) + result = SamFunctionProvider([stack]).functions self.assertEqual(expected, result) - convert_mock.assert_called_with(stack, "Func1", {}, [], False, ignore_code_extraction_warnings=False) + convert_mock.assert_called_with(stack, "Func1", {}, []) @patch("samcli.lib.providers.sam_function_provider.Stack.resources", new_callable=PropertyMock) @patch.object(SamFunctionProvider, "_convert_lambda_function_resource") @@ -662,9 +662,9 @@ def test_must_work_for_lambda_function(self, convert_mock, resources_mock): expected = {"A/B/C/Func1": convertion_result} stack = make_root_stack(None) - result = SamFunctionProvider._extract_functions([stack]) + result = SamFunctionProvider([stack]).functions self.assertEqual(expected, result) - convert_mock.assert_called_with(stack, "Func1", {"a": "b"}, [], False) + convert_mock.assert_called_with(stack, "Func1", {"a": "b"}, []) @patch("samcli.lib.providers.sam_function_provider.Stack.resources", new_callable=PropertyMock) def test_must_skip_unknown_resource(self, resources_mock): @@ -672,7 +672,7 @@ def test_must_skip_unknown_resource(self, resources_mock): expected = {} - result = SamFunctionProvider._extract_functions([make_root_stack(None)]) + result = SamFunctionProvider([make_root_stack(None)]).functions self.assertEqual(expected, result) @patch.object(SamFunctionProvider, "_convert_lambda_function_resource") @@ -701,12 +701,12 @@ def test_must_work_for_multiple_functions_with_name_but_in_different_stacks( expected = {"Func1": function_root, "C/Func1": function_child} - result = SamFunctionProvider._extract_functions([stack_root, stack_child]) + result = SamFunctionProvider([stack_root, stack_child]).functions self.assertEqual(expected, result) convert_mock.assert_has_calls( [ - call(stack_root, "Func1", {"a": "b"}, [], False), - call(stack_child, "Func1", {"a": "b"}, [], False), + call(stack_root, "Func1", {"a": "b"}, []), + call(stack_child, "Func1", {"a": "b"}, []), ] ) @@ -747,7 +747,9 @@ def test_must_convert_zip(self): stack_path=STACK_PATH, ) - result = SamFunctionProvider._convert_sam_function_resource(STACK, name, properties, ["Layer1", "Layer2"]) + result = SamFunctionProvider._convert_sam_function_resource( + Mock(), STACK, name, properties, ["Layer1", "Layer2"] + ) self.assertEqual(expected, result) @@ -787,7 +789,7 @@ def test_must_convert_image(self): stack_path=STACK_PATH, ) - result = SamFunctionProvider._convert_sam_function_resource(STACK, name, properties, []) + result = SamFunctionProvider._convert_sam_function_resource(Mock(), STACK, name, properties, []) self.assertEqual(expected, result) @@ -817,7 +819,7 @@ def test_must_skip_non_existent_properties(self): stack_path=STACK_PATH, ) - result = SamFunctionProvider._convert_sam_function_resource(STACK, name, properties, []) + result = SamFunctionProvider._convert_sam_function_resource(Mock(), STACK, name, properties, []) self.assertEqual(expected, result) @@ -826,7 +828,7 @@ def test_must_default_missing_code_uri(self): name = "myname" properties = {"Runtime": "myruntime"} - result = SamFunctionProvider._convert_sam_function_resource(STACK, name, properties, []) + result = SamFunctionProvider._convert_sam_function_resource(Mock(), STACK, name, properties, []) self.assertEqual(result.codeuri, ".") # Default value def test_must_use_inlinecode(self): @@ -861,7 +863,7 @@ def test_must_use_inlinecode(self): stack_path=STACK_PATH, ) - result = SamFunctionProvider._convert_sam_function_resource(STACK, name, properties, []) + result = SamFunctionProvider._convert_sam_function_resource(Mock(), STACK, name, properties, []) self.assertEqual(expected, result) @@ -898,7 +900,7 @@ def test_must_prioritize_inlinecode(self): stack_path=STACK_PATH, ) - result = SamFunctionProvider._convert_sam_function_resource(STACK, name, properties, []) + result = SamFunctionProvider._convert_sam_function_resource(Mock(), STACK, name, properties, []) self.assertEqual(expected, result) @@ -912,7 +914,7 @@ def test_must_handle_code_dict(self): } } - result = SamFunctionProvider._convert_sam_function_resource(STACK, name, properties, []) + result = SamFunctionProvider._convert_sam_function_resource(Mock(), STACK, name, properties, []) self.assertEqual(result.codeuri, ".") # Default value def test_must_handle_code_s3_uri(self): @@ -920,7 +922,7 @@ def test_must_handle_code_s3_uri(self): name = "myname" properties = {"CodeUri": "s3://bucket/key"} - result = SamFunctionProvider._convert_sam_function_resource(STACK, name, properties, []) + result = SamFunctionProvider._convert_sam_function_resource(Mock(), STACK, name, properties, []) self.assertEqual(result.codeuri, ".") # Default value @@ -960,7 +962,9 @@ def test_must_convert(self): stack_path=STACK_PATH, ) - result = SamFunctionProvider._convert_lambda_function_resource(STACK, name, properties, ["Layer1", "Layer2"]) + result = SamFunctionProvider._convert_lambda_function_resource( + Mock(), STACK, name, properties, ["Layer1", "Layer2"] + ) self.assertEqual(expected, result) @@ -997,7 +1001,7 @@ def test_must_use_inlinecode(self): stack_path=STACK_PATH, ) - result = SamFunctionProvider._convert_lambda_function_resource(STACK, name, properties, []) + result = SamFunctionProvider._convert_lambda_function_resource(Mock(), STACK, name, properties, []) self.assertEqual(expected, result) @@ -1027,7 +1031,7 @@ def test_must_skip_non_existent_properties(self): stack_path=STACK_PATH, ) - result = SamFunctionProvider._convert_lambda_function_resource(STACK, name, properties, []) + result = SamFunctionProvider._convert_lambda_function_resource(Mock(), STACK, name, properties, []) self.assertEqual(expected, result) @@ -1041,7 +1045,7 @@ class TestSamFunctionProvider_parse_layer_info(TestCase): ) def test_raise_on_invalid_layer_resource(self, resources, layer_reference): with self.assertRaises(InvalidLayerReference): - SamFunctionProvider._parse_layer_info(STACK, [layer_reference], resources) + SamFunctionProvider._parse_layer_info(Mock(), Mock(resources=resources), [layer_reference]) @parameterized.expand( [ @@ -1053,7 +1057,7 @@ def test_raise_on_invalid_layer_resource(self, resources, layer_reference): ) def test_raise_on_AmazonLinux1703_layer_provided(self, resources, layer_reference): with self.assertRaises(InvalidLayerVersionArn): - SamFunctionProvider._parse_layer_info(STACK, [layer_reference], resources) + SamFunctionProvider._parse_layer_info(Mock(), Mock(resources=resources), [layer_reference]) def test_must_ignore_opt_in_AmazonLinux1803_layer(self): resources = {} @@ -1063,7 +1067,7 @@ def test_must_ignore_opt_in_AmazonLinux1803_layer(self): "arn:aws:lambda:::awslayer:AmazonLinux1803", ] actual = SamFunctionProvider._parse_layer_info( - Mock(stack_path=STACK_PATH, location="template.yaml", resources=resources), list_of_layers + Mock(), Mock(stack_path=STACK_PATH, location="template.yaml", resources=resources), list_of_layers ) for (actual_layer, expected_layer) in zip( @@ -1084,7 +1088,7 @@ def test_layers_created_from_template_resources(self): {"NonRef": "Something"}, ] actual = SamFunctionProvider._parse_layer_info( - Mock(stack_path=STACK_PATH, location="template.yaml", resources=resources), list_of_layers + Mock(), Mock(stack_path=STACK_PATH, location="template.yaml", resources=resources), list_of_layers ) for (actual_layer, expected_layer) in zip( @@ -1101,7 +1105,7 @@ def test_return_empty_list_on_no_layers(self): resources = {"Function": {"Type": "AWS::Serverless::Function", "Properties": {}}} actual = SamFunctionProvider._parse_layer_info( - Mock(stack_path=STACK_PATH, location="template.yaml", resources=resources), [] + Mock(), Mock(stack_path=STACK_PATH, location="template.yaml", resources=resources), [] ) self.assertEqual(actual, [])