feat: Support Runtime management controls (#420) (#4609)

* feat: Support Runtime management controls (#420)

* feat: Support Runtime management controls

- Modify ECR repo from `/sam/emulation-<runtime>` to `/lambda/<runtime>`.
- Keep RAPID image up-to-date with the remote environment.
- Remove SAM cli version from RAPID image tag locally.
  - These modifications change the rapid image tag
    from `python3.7-rapid-1.2.0-x86_64` to `3.7-rapid-x86_64`,
    since now the runtime name is part of the image *name* and not the tag.
- Support a `RuntimeManagementConfig` in SAM templates and show a message
  if a runtime version is pinned in the template when invoking.
- Rename some variables when building the rapid image, for more clarity.

* PR feedback. Add placeholder text

* Last updates for RuntimeManagementConfiguration

- Change from RuntimeVersion to RuntimeVersion Arn
- Update the message when user has a RuntimeVersion in their template
and wants to invoke locally.

* Update with latest tranform changes

* update tests with latest transform changes

* rollback dev version of SAMT

Co-authored-by: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com>

* Update local_uri_plugin.py

* Update local_uri_plugin.py

Co-authored-by: Renato Valenzuela <37676028+valerena@users.noreply.github.com>

samcli/commands/local/lib/local_lambda.py

@@ -210,6 +210,7 @@ def get_invoke_config(self, function: Function) -> FunctionConfig:
             memory=function.memory,
             timeout=function_timeout,
             env_vars=env_vars,
+            runtime_management_config=function.runtime_management_config,
         )
 
     def _make_env_vars(self, function: Function) -> EnvironmentVariables:

samcli/lib/providers/provider.py

@@ -84,6 +84,8 @@ class Function(NamedTuple):
     function_url_config: Optional[Dict]
     # The path of the stack relative to the root stack, it is empty for functions in root stack
     stack_path: str = ""
+    # Configuration for runtime management. Includes the fields `UpdateRuntimeOn` and `RuntimeVersionArn` (optional).
+    runtime_management_config: Optional[Dict] = None
 
     @property
     def full_path(self) -> str:

samcli/lib/providers/sam_function_provider.py

@@ -467,6 +467,7 @@ def _build_function_configuration(
             codesign_config_arn=resource_properties.get("CodeSigningConfigArn", None),
             architectures=resource_properties.get("Architectures", None),
             function_url_config=resource_properties.get("FunctionUrlConfig"),
+            runtime_management_config=resource_properties.get("RuntimeManagementConfig"),
         )
 
     @staticmethod

samcli/local/docker/lambda_image.py

@@ -1,13 +1,15 @@
 """
 Generates a Docker Image to be used for invoking a function locally
 """
+from typing import Optional
 import uuid
 import logging
 import hashlib
 from enum import Enum
 from pathlib import Path
 
 import sys
+import re
 import platform
 import docker
 
@@ -55,10 +57,49 @@ def has_value(cls, value):
         """
         return any(value == item.value for item in cls)
 
+    @classmethod
+    def get_image_name_tag(cls, runtime: str, architecture: str) -> str:
+        """
+        Returns the image name and tag for a particular runtime
+
+        Parameters
+        ----------
+        runtime : str
+            AWS Lambda runtime
+        architecture : str
+            Architecture for the runtime
+
+        Returns
+        -------
+        str
+            Image name and tag for the runtime's base image, like `python:3.7` or `provided:al2`
+        """
+        runtime_image_tag = ""
+        if runtime == cls.provided.value:
+            # There's a special tag for `provided` not al2 (provided:alami)
+            runtime_image_tag = "provided:alami"
+        elif runtime.startswith("provided"):
+            # `provided.al2` becomes `provided:al2``
+            runtime_image_tag = runtime.replace(".", ":")
+        elif runtime.startswith("dotnet"):
+            # dotnetcore3.1 becomes dotnet:core3.1 and dotnet6 becomes dotnet:6
+            runtime_image_tag = runtime.replace("dotnet", "dotnet:")
+        else:
+            # This fits most runtimes format: `nameN.M` becomes `name:N.M` (python3.9 -> python:3.9)
+            runtime_image_tag = re.sub(r"^([a-z]+)([0-9][a-z0-9\.]*)$", r"\1:\2", runtime)
+            # nodejs14.x, go1.x, etc don't have the `.x` part.
+            runtime_image_tag = runtime_image_tag.replace(".x", "")
+
+        # Runtime image tags contain the architecture only if more than one is supported for that runtime
+        if has_runtime_multi_arch_image(runtime):
+            runtime_image_tag = f"{runtime_image_tag}-{architecture}"
+        return runtime_image_tag
+
 
 class LambdaImage:
     _LAYERS_DIR = "/opt"
-    _INVOKE_REPO_PREFIX = "public.ecr.aws/sam/emulation"
+    _INVOKE_REPO_PREFIX = "public.ecr.aws/lambda"
+    _SAM_INVOKE_REPO_PREFIX = "public.ecr.aws/sam/emulation"
     _SAM_CLI_REPO_NAME = "samcli/lambda"
     _RAPID_SOURCE_PATH = Path(__file__).parent.joinpath("..", "rapid").resolve()
 
@@ -108,49 +149,58 @@ def build(self, runtime, packagetype, image, layers, architecture, stream=None,
         str
             The image to be used (REPOSITORY:TAG)
         """
-        image_name = None
+        base_image = None
+        tag_prefix = ""
 
         if packagetype == IMAGE:
-            image_name = image
+            base_image = image
         elif packagetype == ZIP:
             if self.invoke_images:
-                image_name = self.invoke_images.get(function_name, self.invoke_images.get(None))
-            if not image_name:
-                tag_name = f"latest-{architecture}" if has_runtime_multi_arch_image(runtime) else "latest"
-                image_name = f"{self._INVOKE_REPO_PREFIX}-{runtime}:{tag_name}"
-
-        if not image_name:
+                base_image = self.invoke_images.get(function_name, self.invoke_images.get(None))
+            if not base_image:
+                # Gets the ECR image format like `python:3.7` or `nodejs:16-x86_64`
+                runtime_image_tag = Runtime.get_image_name_tag(runtime, architecture)
+                runtime_only_number = re.split("[:-]", runtime_image_tag)[1]
+                tag_prefix = f"{runtime_only_number}-"
+                base_image = f"{self._INVOKE_REPO_PREFIX}/{runtime_image_tag}"
+
+        if not base_image:
             raise InvalidIntermediateImageError(f"Invalid PackageType, PackageType needs to be one of [{ZIP}, {IMAGE}]")
 
         if image:
             self.skip_pull_image = True
 
-        # Default image tag to be the base image with a tag of 'rapid' instead of latest.
         # If the image name had a digest, removing the @ so that a valid image name can be constructed
         # to use for the local invoke image name.
-        image_repo = image_name.split(":")[0].replace("@", "")
-        image_tag = f"{image_repo}:{RAPID_IMAGE_TAG_PREFIX}-{version}-{architecture}"
+        image_repo = base_image.split(":")[0].replace("@", "")
+        rapid_image = f"{image_repo}:{tag_prefix}{RAPID_IMAGE_TAG_PREFIX}-{architecture}"
 
         downloaded_layers = []
 
         if layers and packagetype == ZIP:
             downloaded_layers = self.layer_downloader.download_all(layers, self.force_image_build)
 
-            docker_image_version = self._generate_docker_image_version(downloaded_layers, runtime, architecture)
-            image_tag = f"{self._SAM_CLI_REPO_NAME}:{docker_image_version}"
+            docker_image_version = self._generate_docker_image_version(downloaded_layers, runtime_image_tag)
+            rapid_image = f"{self._SAM_CLI_REPO_NAME}-{docker_image_version}"
 
         image_not_found = False
 
         # If we are not using layers, build anyways to ensure any updates to rapid get added
         try:
-            self.docker_client.images.get(image_tag)
+            self.docker_client.images.get(rapid_image)
+            # Check if the base image is up-to-date locally and modify build/pull parameters accordingly
+            self._check_base_image_is_current(base_image)
         except docker.errors.ImageNotFound:
-            LOG.info("Image was not found.")
+            LOG.info("Local image was not found.")
             image_not_found = True
 
-        # If building a new rapid image, delete older rapid images of the same repo
-        if image_not_found and image_tag == f"{image_repo}:{RAPID_IMAGE_TAG_PREFIX}-{version}-{architecture}":
-            self._remove_rapid_images(image_repo)
+        # If building a new rapid image, delete older rapid images
+        if image_not_found and rapid_image == f"{image_repo}:{tag_prefix}{RAPID_IMAGE_TAG_PREFIX}-{architecture}":
+            if tag_prefix:
+                # ZIP functions with new RAPID format. Delete images from the old ecr/sam repository
+                self._remove_rapid_images(f"{self._SAM_INVOKE_REPO_PREFIX}-{runtime}")
+            else:
+                self._remove_rapid_images(image_repo)
 
         if (
             self.force_image_build
@@ -162,10 +212,10 @@ def build(self, runtime, packagetype, image, layers, architecture, stream=None,
             stream_writer.write("Building image...")
             stream_writer.flush()
             self._build_image(
-                image if image else image_name, image_tag, downloaded_layers, architecture, stream=stream_writer
+                image if image else base_image, rapid_image, downloaded_layers, architecture, stream=stream_writer
             )
 
-        return image_tag
+        return rapid_image
 
     def get_config(self, image_tag):
         config = {}
@@ -176,7 +226,7 @@ def get_config(self, image_tag):
             return config
 
     @staticmethod
-    def _generate_docker_image_version(layers, runtime, architecture):
+    def _generate_docker_image_version(layers, runtime_image_tag):
         """
         Generate the Docker TAG that will be used to create the image
 
@@ -185,11 +235,8 @@ def _generate_docker_image_version(layers, runtime, architecture):
         layers list(samcli.commands.local.lib.provider.Layer)
             List of the layers
 
-        runtime str
-            Runtime of the image to create
-
-        architecture str
-            Architecture type either x86_64 or arm64 on AWS lambda
+        runtime_image_tag str
+            Runtime version format to generate image name and tag (including architecture, e.g. "python:3.7-x86_64")
 
         Returns
         -------
@@ -204,9 +251,7 @@ def _generate_docker_image_version(layers, runtime, architecture):
         # same order), SAM CLI will only produce one image and use this image across both functions for invoke.
 
         return (
-            runtime
-            + "-"
-            + architecture
+            runtime_image_tag
             + "-"
             + hashlib.sha256("-".join([layer.name for layer in layers]).encode("utf-8")).hexdigest()[0:25]
         )
@@ -219,10 +264,14 @@ def _build_image(self, base_image, docker_tag, layers, architecture, stream=None
         ----------
         base_image str
             Base Image to use for the new image
-        docker_tag
+        docker_tag str
             Docker tag (REPOSITORY:TAG) to use when building the image
         layers list(samcli.commands.local.lib.provider.Layer)
             List of Layers to be use to mount in the image
+        architecture str
+            Architecture, either x86_64 or arm64
+        stream samcli.lib.utils.stream_writer.StreamWriter
+            Stream to write the build output
 
         Raises
         ------
@@ -289,9 +338,9 @@ def set_item_permission(tar_info):
     @staticmethod
     def _generate_dockerfile(base_image, layers, architecture):
         """
-        FROM amazon/aws-sam-cli-emulation-image-python3.6:latest
+        FROM public.ecr.aws/lambda/python:3.6-x86_64
 
-        ADD init /var/rapid
+        ADD aws-lambda-rie /var/rapid
 
         ADD layer1 /opt
         ADD layer2 /opt
@@ -334,7 +383,7 @@ def _remove_rapid_images(self, repo: str) -> None:
         try:
             for image in self.docker_client.images.list(name=repo):
                 for tag in image.tags:
-                    if self.is_rapid_image(tag) and not self.is_image_current(tag):
+                    if self.is_rapid_image(tag) and not self.is_rapid_image_current(tag):
                         try:
                             self.docker_client.images.remove(image.id)
                         except docker.errors.APIError as ex:
@@ -348,20 +397,29 @@ def is_rapid_image(image_name: str) -> bool:
         """
         Is the image tagged as a RAPID clone?
 
-        : param string image_name: Name of the image
-        : return bool: True, if the image name ends with rapid-$SAM_CLI_VERSION. False, otherwise
+        Parameters
+        ----------
+        image_name : str
+            Name of the image
+
+        Returns
+        -------
+        bool
+            True if the image tag starts with the rapid prefix or contains it in between. False, otherwise
         """
 
         try:
-            return image_name.split(":")[1].startswith(f"{RAPID_IMAGE_TAG_PREFIX}-")
+            tag = image_name.split(":")[1]
+            return tag.startswith(f"{RAPID_IMAGE_TAG_PREFIX}-") or f"-{RAPID_IMAGE_TAG_PREFIX}-" in tag
         except (IndexError, AttributeError):
             # split() returned 1 or less items or image_name is None
             return False
 
     @staticmethod
-    def is_image_current(image_name: str) -> bool:
+    def is_rapid_image_current(image_name: str) -> bool:
         """
-        Verify if an image is current or the latest image for the version of samcli
+        Verify if an image has the latest format.
+        The current format doesn't include the SAM version and has the RAPID prefix between dashes.
 
         Parameters
         ----------
@@ -373,4 +431,85 @@ def is_image_current(image_name: str) -> bool:
         bool
             return True if it is current and vice versa
         """
-        return bool(f"-{version}" in image_name)
+        return f"-{RAPID_IMAGE_TAG_PREFIX}-" in image_name
+
+    def _check_base_image_is_current(self, image_name: str) -> None:
+        """
+        Check if the existing base image is up-to-date and update modifier parameters
+        (skip_pull_image, force_image_build) accordingly, printing an informative
+        message depending on the case.
+
+        Parameters
+        ----------
+        image_name : str
+            Base image name to check
+        """
+
+        # No need to check if the overriding parameters are already set
+        if self.skip_pull_image or self.force_image_build:
+            return
+
+        if self.is_base_image_current(image_name):
+            self.skip_pull_image = True
+            LOG.info("Local image is up-to-date")
+        else:
+            self.force_image_build = True
+            LOG.info(
+                "Local image is out of date and will be updated to the latest runtime. "
+                "To skip this, pass in the parameter --skip-pull-image"
+            )
+
+    def is_base_image_current(self, image_name: str) -> bool:
+        """
+        Return True if the base image is up-to-date with the remote environment by comparing the image digests
+
+        Parameters
+        ----------
+        image_name : str
+            Base image name to check
+
+        Returns
+        -------
+        bool
+            True if local image digest is the same as the remote image digest
+        """
+        return self.get_local_image_digest(image_name) == self.get_remote_image_digest(image_name)
+
+    def get_remote_image_digest(self, image_name: str) -> Optional[str]:
+        """
+        Get the digest of the remote version of an image
+
+        Parameters
+        ----------
+        image_name : str
+            Name of the image to get the digest
+
+        Returns
+        -------
+        str
+            Image digest, including `sha256:` prefix
+        """
+        remote_info = self.docker_client.images.get_registry_data(image_name)
+        digest: Optional[str] = remote_info.attrs.get("Descriptor", {}).get("digest")
+        return digest
+
+    def get_local_image_digest(self, image_name: str) -> Optional[str]:
+        """
+        Get the digest of the local version of an image
+
+        Parameters
+        ----------
+        image_name : str
+            Name of the image to get the digest
+
+        Returns
+        -------
+        str
+            Image digest, including `sha256:` prefix
+        """
+        image_info = self.docker_client.images.get(image_name)
+        full_digest: str = image_info.attrs.get("RepoDigests", [None])[0]
+        try:
+            return full_digest.split("@")[1]
+        except (AttributeError, IndexError):
+            return None

samcli/local/docker/manager.py

@@ -79,7 +79,7 @@ def create(self, container):
         if is_image_local and self.skip_pull_image:
             LOG.info("Requested to skip pulling images ...\n")
         elif image_name.startswith("samcli/lambda") or (is_image_local and LambdaImage.is_rapid_image(image_name)):
-            LOG.info("Skip pulling image and use local one: %s.\n", image_name)
+            LOG.info("Using local image: %s.\n", image_name)
         else:
             try:
                 self.pull_image(image_name)