Skip to content

WIP: sam build Resolve layer ARN passed from to parent stacks (not covering all intrinsic functions) #2732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
aahung wants to merge 4 commits into from
Closed

WIP: sam build Resolve layer ARN passed from to parent stacks (not covering all intrinsic functions) #2732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6f4f69c
refactor: Convert some staticmethods to normal methods to avoid passi…
aahung Mar 18, 2021
c8a44d4
Update function provider unit test after refactoring
aahung Mar 18, 2021
a92d301
Resolve layer ARN passed (directly using a single !Ref) from to paren…
aahung Mar 18, 2021
f265ff6
Handle !Join
aahung Mar 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions samcli/lib/intrinsic_resolver/intrinsics_symbol_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,19 @@ def get_translation(self, logical_id, resource_attributes=IntrinsicResolver.REF)
return None
return logical_id_item

if logical_id in self._parameters and isinstance(logical_id_item, dict):
if "Ref" in logical_id_item:
# It is a parameter passed down from the parent
# here we use a special function "CrossStackRef" to refer a logicalID from other stacks
# ".." means the parent stack
Comment on lines +330 to +332
Copy link
Contributor

@mndeveci mndeveci Mar 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: can we move the comment under if statement?

return {"Ref": {"CrossStackRef": ["..", logical_id_item.get("Ref")]}}
if "Fn::Join" in logical_id_item:
delimiter, items = logical_id_item.get("Fn::Join")
if delimiter == "," and all("Ref" in item for item in items):
return [{"Ref": {"CrossStackRef": ["..", item.get("Ref")]}} for item in items]

LOG.warning('"sam build" currently does not support the resolution of parameter %s.', logical_id_item)

return logical_id_item.get(resource_attributes)

@staticmethod
Expand Down
88 changes: 41 additions & 47 deletions samcli/lib/providers/sam_function_provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,14 @@ def __init__(
"""

self.stacks = stacks
self._use_raw_codeuri = use_raw_codeuri
self._ignore_code_extraction_warnings = ignore_code_extraction_warnings

for stack in stacks:
LOG.debug("%d resources found in the stack %s", len(stack.resources), stack.stack_path)

# Store a map of function full_path to function information for quick reference
self.functions = SamFunctionProvider._extract_functions(
self.stacks, use_raw_codeuri, ignore_code_extraction_warnings
)
self.functions = self._extract_functions()

self._deprecated_runtimes = {"nodejs4.3", "nodejs6.10", "nodejs8.10", "dotnetcore2.0"}
self._colored = Colored()
Expand Down Expand Up @@ -103,23 +103,17 @@ def get_all(self) -> Iterator[Function]:
for _, function in self.functions.items():
yield function

@staticmethod
def _extract_functions(
stacks: List[Stack], use_raw_codeuri: bool = False, ignore_code_extraction_warnings: bool = False
) -> Dict[str, Function]:
def _extract_functions(self) -> Dict[str, Function]:
"""
Extracts and returns function information from the given dictionary of SAM/CloudFormation resources. This
method supports functions defined with AWS::Serverless::Function and AWS::Lambda::Function

:param stacks: List of SAM/CloudFormation stacks to extract functions from
:param bool use_raw_codeuri: Do not resolve adjust core_uri based on the template path, use the raw uri.
:param bool ignore_code_extraction_warnings: suppress log statements on code extraction from resources.
:return dict(string : samcli.commands.local.lib.provider.Function): Dictionary of function full_path to the
Function configuration object
"""

result: Dict[str, Function] = {} # a dict with full_path as key and extracted function as value
for stack in stacks:
for stack in self.stacks:
for name, resource in stack.resources.items():

resource_type = resource.get("Type")
Expand All @@ -130,46 +124,36 @@ def _extract_functions(
resource_properties["Metadata"] = resource_metadata

if resource_type == SamFunctionProvider.SERVERLESS_FUNCTION:
layers = SamFunctionProvider._parse_layer_info(
layers = self._parse_layer_info(
stack,
resource_properties.get("Layers", []),
use_raw_codeuri,
ignore_code_extraction_warnings=ignore_code_extraction_warnings,
)
function = SamFunctionProvider._convert_sam_function_resource(
function = self._convert_sam_function_resource(
stack,
name,
resource_properties,
layers,
use_raw_codeuri,
ignore_code_extraction_warnings=ignore_code_extraction_warnings,
)
result[function.full_path] = function

elif resource_type == SamFunctionProvider.LAMBDA_FUNCTION:
layers = SamFunctionProvider._parse_layer_info(
layers = self._parse_layer_info(
stack,
resource_properties.get("Layers", []),
use_raw_codeuri,
ignore_code_extraction_warnings=ignore_code_extraction_warnings,
)
function = SamFunctionProvider._convert_lambda_function_resource(
stack, name, resource_properties, layers, use_raw_codeuri
)
function = self._convert_lambda_function_resource(stack, name, resource_properties, layers)
result[function.full_path] = function

# We don't care about other resource types. Just ignore them

return result

@staticmethod
def _convert_sam_function_resource(
self,
stack: Stack,
name: str,
resource_properties: Dict,
layers: List[LayerVersion],
use_raw_codeuri: bool = False,
ignore_code_extraction_warnings: bool = False,
) -> Function:
"""
Converts a AWS::Serverless::Function resource to a Function configuration usable by the provider.
Expand Down Expand Up @@ -201,20 +185,19 @@ def _convert_sam_function_resource(
name,
resource_properties,
"CodeUri",
ignore_code_extraction_warnings=ignore_code_extraction_warnings,
ignore_code_extraction_warnings=self._ignore_code_extraction_warnings,
)
LOG.debug("Found Serverless function with name='%s' and CodeUri='%s'", name, codeuri)
elif packagetype == IMAGE:
imageuri = SamFunctionProvider._extract_sam_function_imageuri(resource_properties, "ImageUri")
LOG.debug("Found Serverless function with name='%s' and ImageUri='%s'", name, imageuri)

return SamFunctionProvider._build_function_configuration(
stack, name, codeuri, resource_properties, layers, inlinecode, imageuri, use_raw_codeuri
stack, name, codeuri, resource_properties, layers, inlinecode, imageuri, self._use_raw_codeuri
)

@staticmethod
def _convert_lambda_function_resource(
stack: Stack, name: str, resource_properties: Dict, layers: List[LayerVersion], use_raw_codeuri: bool = False
self, stack: Stack, name: str, resource_properties: Dict, layers: List[LayerVersion]
) -> Function:
"""
Converts a AWS::Lambda::Function resource to a Function configuration usable by the provider.
Expand All @@ -227,8 +210,6 @@ def _convert_lambda_function_resource(
Properties of this resource
layers List(samcli.commands.local.lib.provider.Layer)
List of the Layer objects created from the template and layer list defined on the function.
use_raw_codeuri
Do not resolve adjust core_uri based on the template path, use the raw uri.

Returns
-------
Expand Down Expand Up @@ -259,7 +240,7 @@ def _convert_lambda_function_resource(
LOG.debug("Found Lambda function with name='%s' and Imageuri='%s'", name, imageuri)

return SamFunctionProvider._build_function_configuration(
stack, name, codeuri, resource_properties, layers, inlinecode, imageuri, use_raw_codeuri
stack, name, codeuri, resource_properties, layers, inlinecode, imageuri, self._use_raw_codeuri
)

@staticmethod
Expand Down Expand Up @@ -330,12 +311,10 @@ def _build_function_configuration(
codesign_config_arn=resource_properties.get("CodeSigningConfigArn", None),
)

@staticmethod
def _parse_layer_info(
self,
stack: Stack,
list_of_layers: List[Any],
use_raw_codeuri: bool = False,
ignore_code_extraction_warnings: bool = False,
) -> List[LayerVersion]:
"""
Creates a list of Layer objects that are represented by the resources and the list of layers
Expand All @@ -347,10 +326,6 @@ def _parse_layer_info(
list_of_layers : List[Any]
List of layers that are defined within the Layers Property on a function,
layer can be defined as string or Dict, in case customers define it in other types, use "Any" here.
use_raw_codeuri : bool
Do not resolve adjust core_uri based on the template path, use the raw uri.
ignore_code_extraction_warnings : bool
Whether to print warning when codeuri is not a local pth

Returns
-------
Expand Down Expand Up @@ -387,8 +362,25 @@ def _parse_layer_info(
# In the list of layers that is defined within a template, you can reference a LayerVersion resource.
# When running locally, we need to follow that Ref so we can extract the local path to the layer code.
if isinstance(layer, dict) and layer.get("Ref"):
layer_logical_id = cast(str, layer.get("Ref"))
layer_resource = stack.resources.get(layer_logical_id)
# Here layer refers to another resources.
# in a multi stack system, the layer might locate in the current stack or passed down from parents.
# for example, {"Ref": "layerLogicalID"} refers to the "layerLogicalID" in the current stack.
# {"Ref": {"CrossStackRef": ["..", "layerLogicalID"]}
Copy link
Contributor

@CoshUS CoshUS Mar 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn't adding a new SAM CLI specific intrinsic function break existing translation? The template will not be compatible with uploading to CFN directly.

Copy link
Contributor Author

@aahung aahung Mar 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The built template won't contain this special function.

The output template because it is only affected by builder.update_template()(only update codeuri and other path properties) and template.move_template() (only move the file).

# refers to the "layerLogicalID" in the parent stack.
# {"Ref": {"CrossStackRef": ["..", {"CrossStackRef": ["..", "layerLogicalID"]})}
Copy link
Contributor

@mndeveci mndeveci Mar 18, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should carefully clear this CrossStackRef from the template.

And what happens if there are more than one level between the layer definition vs the function. Like RootStack (LayerDefinition) -> ChildStack -> GrandChildStack (Function which uses LayerDefinition from RootStack)?

Copy link
Contributor Author

@aahung aahung Mar 18, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should carefully clear this CrossStackRef from the template.

Yes, but this does not affect the output template because it is only affected by builder.update_template()(only update codeuri and other path properties) and template.move_template() (only move the file).

And what happens if there are more than one level between the layer definition vs the function.

this is the last example. I tested this with

# template.yaml
Resources:
  MyLayerVersion:
    Type: AWS::Serverless::LayerVersion
    Properties:
      LayerName: MyLayer
      Description: Layer description
      ContentUri: MyLayerVersion
      CompatibleRuntimes:
        - python3.8

  App:
    Type: AWS::Serverless::Application
    Properties:
      Location: ./child.yaml
      Parameters:
        Layer: !Ref MyLayerVersion

# child.yaml
Parameters:
  Layer:
    Type: String

Resources:
  Function:
    Type: AWS::Serverless::Function
    Properties:
      CodeUri: function
      Handler: app.lambda_handler
      Runtime: python3.8
      Layers:
        - !Ref Layer
  
  App:
    Type: AWS::Serverless::Application
    Properties:
      Location: ./child2.yaml
      Parameters:
        Layer: !Ref Layer

# child2.yaml
Parameters:
  Layer:
    Type: String

Resources:
  Function2:
    Type: AWS::Serverless::Function
    Properties:
      CodeUri: function
      Handler: app.lambda_handler
      Runtime: python3.8
      Layers:
        - !Ref Layer

# refers to the layer in the grandparent stack.
# here we use a while loop to remove all "CrossStackRef" keys
# and locate the underlying layer resource ID.
ref_stack = stack
layer_ref = layer.get("Ref")
while isinstance(layer_ref, dict) and "CrossStackRef" in layer_ref:
relative_stack_path, layer_ref = layer_ref["CrossStackRef"]
ref_stack = SamLocalStackProvider.resolve_stack(self.stacks, ref_stack, relative_stack_path)

layer_logical_id = cast(str, layer_ref)
LOG.debug("resolved layer from %s to %s in stack %s", layer, layer_logical_id, ref_stack.stack_path)

layer_resource = ref_stack.resources.get(layer_logical_id)
if not layer_resource or layer_resource.get("Type", "") not in (
SamFunctionProvider.SERVERLESS_LAYER,
SamFunctionProvider.LAMBDA_LAYER,
Expand All @@ -405,20 +397,22 @@ def _parse_layer_info(

if resource_type == SamFunctionProvider.SERVERLESS_LAYER:
codeuri = SamFunctionProvider._extract_sam_function_codeuri(
layer_logical_id, layer_properties, "ContentUri", ignore_code_extraction_warnings
layer_logical_id, layer_properties, "ContentUri", self._ignore_code_extraction_warnings
)

if codeuri and not use_raw_codeuri:
LOG.debug("--base-dir is presented not, adjusting uri %s relative to %s", codeuri, stack.location)
codeuri = SamLocalStackProvider.normalize_resource_path(stack.location, codeuri)
if codeuri and not self._use_raw_codeuri:
LOG.debug(
"--base-dir is presented not, adjusting uri %s relative to %s", codeuri, ref_stack.location
)
codeuri = SamLocalStackProvider.normalize_resource_path(ref_stack.location, codeuri)

layers.append(
LayerVersion(
layer_logical_id,
codeuri,
compatible_runtimes,
layer_resource.get("Metadata", None),
stack_path=stack.stack_path,
stack_path=ref_stack.stack_path,
)
)

Expand Down
12 changes: 12 additions & 0 deletions samcli/lib/providers/sam_stack_provider.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"""
import logging
import os
import posixpath
from typing import Optional, Dict, cast, List, Iterator, Tuple
from urllib.parse import unquote, urlparse

Expand Down Expand Up @@ -340,3 +341,14 @@ def normalize_resource_path(stack_file_path: str, path: str) -> str:
stack_file_path = os.path.relpath(os.path.realpath(stack_file_path))

return os.path.normpath(os.path.join(os.path.dirname(stack_file_path), path))

@staticmethod
def resolve_stack(stacks: List[Stack], base_stack: Stack, relative_stack_path: str) -> Stack:
Copy link
Contributor

@mndeveci mndeveci Mar 18, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add some documentation to the method

target_stack_path = posixpath.normpath(posixpath.join(base_stack.stack_path, relative_stack_path))
if target_stack_path == ".":
# root stack
target_stack_path = ""
try:
return next(stack for stack in stacks if stack.stack_path == target_stack_path)
except StopIteration as ex:
raise ValueError(f"Cannot find stack with path {target_stack_path}") from ex
Loading