feat: configuration file for sam cli commands

requirements/base.txt

@@ -13,4 +13,5 @@ requests==2.22.0
 serverlessrepo==0.1.9
 aws_lambda_builders==0.5.0
 # https://github.com/mhammond/pywin32/issues/1439
-pywin32 < 226; sys_platform == 'win32'
+pywin32 < 226; sys_platform == 'win32'
+toml==0.10.0

requirements/isolated.txt

@@ -32,6 +32,7 @@ requests==2.22.0
 s3transfer==0.2.1
 serverlessrepo==0.1.9
 six==1.11.0
+toml==0.10.0
 tzlocal==2.0.0
 urllib3==1.25.3
 websocket-client==0.56.0

samcli/cli/cli_config_file.py

@@ -0,0 +1,188 @@
+"""
+CLI configuration decorator to use TOML configuration files for click commands.
+"""
+
+## This section contains code copied and modified from [click_config_file][https://github.com/phha/click_config_file/blob/master/click_config_file.py]
+## SPDX-License-Identifier: MIT
+
+import functools
+import os
+import logging
+
+import click
+import toml
+
+__all__ = ("TomlProvider", "configuration_option", "get_ctx_defaults")
+
+LOG = logging.getLogger("samcli")
+DEFAULT_CONFIG_FILE_NAME = "samconfig.toml"
+DEFAULT_IDENTIFER = "default"
+
+
+class TomlProvider:
+    """
+    A parser for toml configuration files
+    :param cmd: sam command name as defined by click
+    :param section: section defined in the configuration file nested within `cmd`
+    """
+
+    def __init__(self, section=None):
+        self.section = section
+
+    def __call__(self, file_path, config_env, cmd_name):
+        """
+        Get resolved config based on the `file_path` for the configuration file,
+        `config_env` targeted inside the config file and corresponding `cmd_name`
+        as denoted by `click`.
+
+        :param file_path: The path to the configuration file
+        :param config_env: The name of the sectional config_env within configuration file.
+        :param cmd_name: sam command name as defined by click
+        :returns dictionary containing the configuration parameters under specified config_env
+        """
+        resolved_config = {}
+        try:
+            config = toml.load(file_path)
+        except Exception as ex:
+            LOG.error("Error reading configuration file :%s %s", file_path, str(ex))
+            return resolved_config
+        if self.section:
+            try:
+                resolved_config = self._get_config_env(config, config_env)[cmd_name][self.section]
+            except KeyError:
+                LOG.debug(
+                    "Error reading configuration file at %s with config_env %s, command %s, section %s",
+                    file_path,
+                    config_env,
+                    cmd_name,
+                    self.section,
+                )
+        return resolved_config
+
+    def _get_config_env(self, config, config_env):
+        """
+
+        :param config: loaded TOML configuration file into dictionary representation
+        :param config_env: top level section defined within TOML configuration file
+        :return:
+        """
+        return config.get(config_env, config.get(DEFAULT_IDENTIFER, {}))
+
+
+def configuration_callback(cmd_name, option_name, config_env_name, saved_callback, provider, ctx, param, value):
+    """
+    Callback for reading the config file.
+
+    Also takes care of calling user specified custom callback afterwards.
+
+    :param cmd_name: `sam` command name derived from click.
+    :param option_name: The name of the option. This is used for error messages.
+    :param config_env_name: `top` level section within configuration file
+    :param saved_callback: User-specified callback to be called later.
+    :param provider:  A callable that parses the configuration file and returns a dictionary
+        of the configuration parameters. Will be called as
+        `provider(file_path, config_env, cmd_name)`.
+    :param ctx: Click context
+    :param param: Click parameter
+    :param value: Specified value for config_env
+    :returns specified callback or the specified value for config_env.
+    """
+
+    # ctx, param and value are default arguments for click specified callbacks.
+    ctx.default_map = ctx.default_map or {}
+    cmd_name = cmd_name or ctx.info_name
+    param.default = DEFAULT_IDENTIFER
+    config_env_name = value or config_env_name
+    config = get_ctx_defaults(cmd_name, provider, ctx, config_env_name=config_env_name)
+    ctx.default_map.update(config)
+
+    return saved_callback(ctx, param, value) if saved_callback else value
+
+
+def get_ctx_defaults(cmd_name, provider, ctx, config_env_name=DEFAULT_IDENTIFER):
+    """
+    Get the set of the parameters that are needed to be set into the click command.
+    This function also figures out the command name by looking up current click context's parent
+    and constructing the parsed command name that is used in default configuration file.
+    If a given cmd_name is start-api, the parsed name is "local_start_api".
+    provider is called with `config_file`, `config_env_name` and `parsed_cmd_name`.
+
+    :param cmd_name: `sam` command name
+    :param provider: provider to be called for reading configuration file
+    :param ctx: Click context
+    :param config_env_name: config-env within configuration file
+    :return: dictionary of defaults for parameters
+    """
+
+    cwd = getattr(ctx, "config_path", None)
+    config_file = os.path.join(cwd if cwd else os.getcwd(), DEFAULT_CONFIG_FILE_NAME)
+    config = {}
+    if os.path.isfile(config_file):
+        LOG.debug("Config file location: %s", os.path.abspath(config_file))
+
+        # Find parent of current context
+        _parent = ctx.parent
+        _cmd_names = []
+        # Need to find the total set of commands that current command is part of.
+        if cmd_name != ctx.info_name:
+            _cmd_names = [cmd_name]
+        _cmd_names.append(ctx.info_name)
+        # Go through all parents till a parent of a context exists.
+        while _parent.parent:
+            info_name = _parent.info_name
+            _cmd_names.append(info_name)
+            _parent = _parent.parent
+
+        # construct a parsed name that is of the format: a_b_c_d
+        parsed_cmd_name = "_".join(reversed([cmd.replace("-", "_").replace(" ", "_") for cmd in _cmd_names]))
+
+        config = provider(config_file, config_env_name, parsed_cmd_name)
+
+    return config
+
+
+def configuration_option(*param_decls, **attrs):
+    """
+    Adds configuration file support to a click application.
+
+    This will create an option of type `STRING` expecting the config_env in the
+    configuration file, by default this config_env is `default`. When specified,
+    the requisite portion of the configuration file is considered as the
+    source of truth.
+
+    The default name of the option is `--config-env`.
+
+    This decorator accepts the same arguments as `click.option`.
+    In addition, the following keyword arguments are available:
+    :param cmd_name: The command name. Default: `ctx.info_name`
+    :param config_env_name: The config_env name. This is used to determine which part of the configuration
+        needs to be read.
+    :param provider:         A callable that parses the configuration file and returns a dictionary
+        of the configuration parameters. Will be called as
+        `provider(file_path, config_env, cmd_name)
+    """
+    param_decls = param_decls or ("--config-env",)
+    option_name = param_decls[0]
+
+    def decorator(f):
+
+        attrs.setdefault("is_eager", True)
+        attrs.setdefault("help", "Read config-env from Configuration File.")
+        attrs.setdefault("expose_value", False)
+        # --config-env is hidden and can potentially be opened up in the future.
+        attrs.setdefault("hidden", True)
+        # explicitly ignore values passed to --config-env, can be opened up in the future.
+        config_env_name = DEFAULT_IDENTIFER
+        provider = attrs.pop("provider")
+        attrs["type"] = click.STRING
+        saved_callback = attrs.pop("callback", None)
+        partial_callback = functools.partial(
+            configuration_callback, None, option_name, config_env_name, saved_callback, provider
+        )
+        attrs["callback"] = partial_callback
+        return click.option(*param_decls, **attrs)(f)
+
+    return decorator
+
+
+# End section copied from [[click_config_file][https://github.com/phha/click_config_file/blob/master/click_config_file.py]

samcli/cli/types.py

@@ -18,9 +18,17 @@ class CfnParameterOverridesType(click.ParamType):
     __EXAMPLE_1 = "ParameterKey=KeyPairName,ParameterValue=MyKey ParameterKey=InstanceType,ParameterValue=t1.micro"
     __EXAMPLE_2 = "KeyPairName=MyKey InstanceType=t1.micro"
 
-    # Regex that parses CloudFormation parameter key-value pairs: https://regex101.com/r/xqfSjW/2
-    _pattern_1 = r"(?:ParameterKey=([A-Za-z0-9\"]+),ParameterValue=(\"(?:\\.|[^\"\\]+)*\"|(?:\\.|[^ \"\\]+)+))"
-    _pattern_2 = r"(?:([A-Za-z0-9\"]+)=(\"(?:\\.|[^\"\\]+)*\"|(?:\\.|[^ \"\\]+)+))"
+    # Regex that parses CloudFormation parameter key-value pairs:
+    # https://regex101.com/r/xqfSjW/2
+    # https://regex101.com/r/xqfSjW/5
+
+    # If Both ParameterKey pattern and KeyPairName=MyKey, should not be fixed. if they are it can
+    # result in unpredicatable behavior.
+    KEY_REGEX = '([A-Za-z0-9\\"]+)'
+    VALUE_REGEX = '(\\"(?:\\\\.|[^\\"\\\\]+)*\\"|(?:\\\\.|[^ \\"\\\\]+)+))'
+
+    _pattern_1 = r"(?:ParameterKey={key},ParameterValue={value}".format(key=KEY_REGEX, value=VALUE_REGEX)
+    _pattern_2 = r"(?:(?: ){key}={value}".format(key=KEY_REGEX, value=VALUE_REGEX)
 
     ordered_pattern_match = [_pattern_1, _pattern_2]
 
@@ -34,7 +42,11 @@ def convert(self, value, param, ctx):
         if value == ("",):
             return result
 
+        value = (value,) if isinstance(value, str) else value
         for val in value:
+            val.strip()
+            # Add empty string to start of the string to help match `_pattern2`
+            val = " " + val
 
             try:
                 # NOTE(TheSriram): find the first regex that matched.
@@ -159,6 +171,9 @@ def convert(self, value, param, ctx):
         if value == ("",):
             return result
 
+        # if value comes in a via configuration file, we should still convert it.
+        # value = (value, ) if not isinstance(value, tuple) else value
+
         for val in value:
 
             groups = re.findall(self._pattern, val)

samcli/commands/_utils/options.py

@@ -7,6 +7,7 @@
 from functools import partial
 
 import click
+from click.types import FuncParamType
 from samcli.cli.types import CfnParameterOverridesType, CfnMetadataType, CfnTags
 from samcli.commands._utils.custom_options.option_nargs import OptionNargs
 
@@ -43,8 +44,10 @@ def get_or_default_template_file_name(ctx, param, provided_value, include_build)
             if os.path.exists(option):
                 provided_value = option
                 break
-
     result = os.path.abspath(provided_value)
+
+    if ctx:
+        setattr(ctx, "config_path", os.path.dirname(result))
     LOG.debug("Using SAM Template at %s", result)
     return result
 
@@ -74,13 +77,15 @@ def template_click_option(include_build=True):
     Click Option for template option
     """
     return click.option(
+        "--template-file",
         "--template",
         "-t",
         default=_TEMPLATE_OPTION_DEFAULT_VALUE,
         type=click.Path(),
         envvar="SAM_TEMPLATE_FILE",
         callback=partial(get_or_default_template_file_name, include_build=include_build),
         show_default=True,
+        is_eager=True,
         help="AWS SAM template file",
     )
 
@@ -143,7 +148,7 @@ def capabilities_click_option():
     return click.option(
         "--capabilities",
         cls=OptionNargs,
-        type=click.STRING,
+        type=FuncParamType(lambda value: value.split(" ")),
         required=True,
         help="A list of  capabilities  that  you  must  specify"
         "before  AWS  Cloudformation  can create certain stacks. Some stack tem-"
@@ -182,7 +187,7 @@ def notification_arns_click_option():
     return click.option(
         "--notification-arns",
         cls=OptionNargs,
-        type=click.STRING,
+        type=FuncParamType(lambda value: value.split(" ")),
         required=False,
         help="Amazon  Simple  Notification  Service  topic"
         "Amazon  Resource  Names  (ARNs) that AWS CloudFormation associates with"

samcli/commands/build/command.py

@@ -10,6 +10,7 @@
     parameter_override_option
 from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options
 from samcli.lib.telemetry.metrics import track_command
+from samcli.cli.cli_config_file import configuration_option, TomlProvider
 
 
 LOG = logging.getLogger(__name__)
@@ -53,6 +54,7 @@
 """
 
 
+@configuration_option(provider=TomlProvider(section="parameters"))
 @click.command("build", help=HELP_TEXT, short_help="Build your Lambda function code")
 @click.option('--build-dir', '-b',
               default=DEFAULT_BUILD_DIR,
@@ -82,7 +84,7 @@
 @track_command
 def cli(ctx,
         function_identifier,
-        template,
+        template_file,
         base_dir,
         build_dir,
         use_container,
@@ -95,7 +97,7 @@ def cli(ctx,
 
     mode = _get_mode_value_from_envvar("SAM_BUILD_MODE", choices=["debug"])
 
-    do_cli(function_identifier, template, base_dir, build_dir, True, use_container, manifest, docker_network,
+    do_cli(function_identifier, template_file, base_dir, build_dir, True, use_container, manifest, docker_network,
            skip_pull_image, parameter_overrides, mode)  # pragma: no cover
 
 

samcli/commands/deploy/command.py

@@ -4,12 +4,13 @@
 
 import click
 
-
+from samcli.cli.cli_config_file import configuration_option, TomlProvider
 from samcli.commands._utils.options import (
     parameter_override_option,
     capabilities_override_option,
     tags_override_option,
     notification_arns_override_option,
+    template_click_option,
 )
 from samcli.cli.main import pass_context, common_options, aws_creds_options
 from samcli.lib.telemetry.metrics import track_command
@@ -27,20 +28,14 @@
 """
 
 
+@configuration_option(provider=TomlProvider(section="parameters"))
 @click.command(
     "deploy",
     short_help=SHORT_HELP,
     context_settings={"ignore_unknown_options": False, "allow_interspersed_args": True, "allow_extra_args": True},
     help=HELP_TEXT,
 )
-@click.option(
-    "--template-file",
-    "--template",
-    "-t",
-    required=True,
-    type=click.Path(),
-    help="The path where your AWS SAM template is located",
-)
+@template_click_option(include_build=False)
 @click.option(
     "--stack-name",
     required=True,

samcli/commands/init/__init__.py

@@ -8,6 +8,7 @@
 
 import click
 
+from samcli.cli.cli_config_file import configuration_option, TomlProvider
 from samcli.commands.exceptions import UserException
 from samcli.cli.main import pass_context, common_options, global_cfg
 from samcli.local.common.runtime_template import RUNTIMES, SUPPORTED_DEP_MANAGERS
@@ -55,6 +56,7 @@
 """
 
 
+@configuration_option(provider=TomlProvider(section="parameters"))
 @click.command(
     "init",
     help=HELP_TEXT,