diff --git a/samcli/cli/cli_config_file.py b/samcli/cli/cli_config_file.py index e4606e4555..bfc295c01b 100644 --- a/samcli/cli/cli_config_file.py +++ b/samcli/cli/cli_config_file.py @@ -10,49 +10,67 @@ import logging import os from pathlib import Path +from typing import Any, Callable, Dict, List, Optional import click +from click.core import ParameterSource from samcli.cli.context import get_cmd_names from samcli.commands.exceptions import ConfigException from samcli.lib.config.samconfig import DEFAULT_CONFIG_FILE_NAME, DEFAULT_ENV, SamConfig -__all__ = ("TomlProvider", "configuration_option", "get_ctx_defaults") +__all__ = ("ConfigProvider", "configuration_option", "get_ctx_defaults") LOG = logging.getLogger(__name__) -class TomlProvider: +class ConfigProvider: """ - A parser for toml configuration files + A parser for sam configuration files """ def __init__(self, section=None, cmd_names=None): """ - The constructor for TomlProvider class - :param section: section defined in the configuration file nested within `cmd` - :param cmd_names: cmd_name defined in the configuration file + The constructor for ConfigProvider class + + Parameters + ---------- + section + The section defined in the configuration file nested within `cmd` + cmd_names + The cmd_name defined in the configuration file """ self.section = section self.cmd_names = cmd_names - def __call__(self, config_path, config_env, cmd_names): + def __call__(self, config_path: Path, config_env: str, cmd_names: List[str]) -> dict: """ Get resolved config based on the `file_path` for the configuration file, `config_env` targeted inside the config file and corresponding `cmd_name` as denoted by `click`. - :param config_path: The path of configuration file. - :param config_env: The name of the sectional config_env within configuration file. - :param list cmd_names: sam command name as defined by click - :returns dictionary containing the configuration parameters under specified config_env + Parameters + ---------- + config_path: Path + The path of configuration file. + config_env: str + The name of the sectional config_env within configuration file. + cmd_names: List[str] + The sam command name as defined by click. + + Returns + ------- + dict + A dictionary containing the configuration parameters under specified config_env. """ - resolved_config = {} + resolved_config: dict = {} # Use default sam config file name if config_path only contain the directory config_file_path = ( - Path(os.path.abspath(config_path)) if config_path else Path(os.getcwd(), DEFAULT_CONFIG_FILE_NAME) + Path(os.path.abspath(config_path)) + if config_path + else Path(os.getcwd(), SamConfig.get_default_file(os.getcwd())) ) config_file_name = config_file_path.name config_file_dir = config_file_path.parents[0] @@ -105,32 +123,56 @@ def __call__(self, config_path, config_env, cmd_names): return resolved_config -def configuration_callback(cmd_name, option_name, saved_callback, provider, ctx, param, value): +def configuration_callback( + cmd_name: str, + option_name: str, + saved_callback: Optional[Callable], + provider: Callable, + ctx: click.Context, + param: click.Parameter, + value, +): """ Callback for reading the config file. Also takes care of calling user specified custom callback afterwards. - :param cmd_name: `sam` command name derived from click. - :param option_name: The name of the option. This is used for error messages. - :param saved_callback: User-specified callback to be called later. - :param provider: A callable that parses the configuration file and returns a dictionary + Parameters + ---------- + cmd_name: str + The `sam` command name derived from click. + option_name: str + The name of the option. This is used for error messages. + saved_callback: Optional[Callable] + User-specified callback to be called later. + provider: Callable + A callable that parses the configuration file and returns a dictionary of the configuration parameters. Will be called as `provider(file_path, config_env, cmd_name)`. - :param ctx: Click context - :param param: Click parameter - :param value: Specified value for config_env - :returns specified callback or the specified value for config_env. + ctx: click.Context + Click context + param: click.Parameter + Click parameter + value + Specified value for config_env + + Returns + ------- + The specified callback or the specified value for config_env. """ # ctx, param and value are default arguments for click specified callbacks. ctx.default_map = ctx.default_map or {} - cmd_name = cmd_name or ctx.info_name + cmd_name = cmd_name or str(ctx.info_name) param.default = None config_env_name = ctx.params.get("config_env") or DEFAULT_ENV - config_file = ctx.params.get("config_file") or DEFAULT_CONFIG_FILE_NAME config_dir = getattr(ctx, "samconfig_dir", None) or os.getcwd() + config_file = ( # If given by default, check for other `samconfig` extensions first. Else use user-provided value + SamConfig.get_default_file(config_dir=config_dir) + if getattr(ctx.get_parameter_source("config_file"), "name", "") == ParameterSource.DEFAULT.name + else ctx.params.get("config_file") or SamConfig.get_default_file(config_dir=config_dir) + ) # If --config-file is an absolute path, use it, if not, start from config_dir config_file_path = config_file if os.path.isabs(config_file) else os.path.join(config_dir, config_file) if ( @@ -154,21 +196,35 @@ def configuration_callback(cmd_name, option_name, saved_callback, provider, ctx, return saved_callback(ctx, param, config_env_name) if saved_callback else config_env_name -def get_ctx_defaults(cmd_name, provider, ctx, config_env_name, config_file=None): +def get_ctx_defaults( + cmd_name: str, provider: Callable, ctx: click.Context, config_env_name: str, config_file: Optional[str] = None +) -> Any: """ Get the set of the parameters that are needed to be set into the click command. + This function also figures out the command name by looking up current click context's parent and constructing the parsed command name that is used in default configuration file. If a given cmd_name is start-api, the parsed name is "local_start_api". provider is called with `config_file`, `config_env_name` and `parsed_cmd_name`. - :param cmd_name: `sam` command name - :param provider: provider to be called for reading configuration file - :param ctx: Click context - :param config_env_name: config-env within configuration file, sam configuration file will be relative to the - supplied original template if its path is not specified - :param config_file: configuration file name - :return: dictionary of defaults for parameters + Parameters + ---------- + cmd_name: str + The `sam` command name. + provider: Callable + The provider to be called for reading configuration file. + ctx: click.Context + Click context + config_env_name: str + The config-env within configuration file, sam configuration file will be relative to the + supplied original template if its path is not specified. + config_file: Optional[str] + The configuration file name. + + Returns + ------- + Any + A dictionary of defaults for parameters. """ return provider(config_file, config_env_name, get_cmd_names(cmd_name, ctx)) @@ -180,30 +236,38 @@ def configuration_option(*param_decls, **attrs): """ Adds configuration file support to a click application. - NOTE: This decorator should be added to the top of parameter chain, right below click.command, before - any options are declared. - - Example: - >>> @click.command("hello") - @configuration_option(provider=TomlProvider(section="parameters")) - @click.option('--name', type=click.String) - def hello(name): - print("Hello " + name) - This will create a hidden click option whose callback function loads configuration parameters from default configuration environment [default] in default configuration file [samconfig.toml] in the template file directory. - :param preconfig_decorator_list: A list of click option decorator which need to place before this function. For - exmple, if we want to add option "--config-file" and "--config-env" to allow customized configuration file + + Note + ---- + This decorator should be added to the top of parameter chain, right below click.command, before + any options are declared. + + Example + ------- + >>> @click.command("hello") + @configuration_option(provider=ConfigProvider(section="parameters")) + @click.option('--name', type=click.String) + def hello(name): + print("Hello " + name) + + Parameters + ---------- + preconfig_decorator_list: list + A list of click option decorator which need to place before this function. For + example, if we want to add option "--config-file" and "--config-env" to allow customized configuration file and configuration environment, we will use configuration_option as below: @configuration_option( preconfig_decorator_list=[decorator_customize_config_file, decorator_customize_config_env], - provider=TomlProvider(section=CONFIG_SECTION), + provider=ConfigProvider(section=CONFIG_SECTION), ) By default, we enable these two options. - :param provider: A callable that parses the configuration file and returns a dictionary + provider: Callable + A callable that parses the configuration file and returns a dictionary of the configuration parameters. Will be called as - `provider(file_path, config_env, cmd_name) + `provider(file_path, config_env, cmd_name)` """ def decorator_configuration_setup(f): @@ -240,17 +304,25 @@ def decorator(f): return composed_decorator(decorator_list) -def decorator_customize_config_file(f): +def decorator_customize_config_file(f: Callable) -> Callable: """ CLI option to customize configuration file name. By default it is 'samconfig.toml' in project directory. Ex: --config-file samconfig.toml - :param f: Callback function passed by Click - :return: Callback function + + Parameters + ---------- + f: Callable + Callback function passed by Click + + Returns + ------- + Callable + A Callback function """ - config_file_attrs = {} + config_file_attrs: Dict[str, Any] = {} config_file_param_decls = ("--config-file",) config_file_attrs["help"] = "Configuration file containing default parameter values." - config_file_attrs["default"] = "samconfig.toml" + config_file_attrs["default"] = DEFAULT_CONFIG_FILE_NAME config_file_attrs["show_default"] = True config_file_attrs["is_eager"] = True config_file_attrs["required"] = False @@ -258,17 +330,25 @@ def decorator_customize_config_file(f): return click.option(*config_file_param_decls, **config_file_attrs)(f) -def decorator_customize_config_env(f): +def decorator_customize_config_env(f: Callable) -> Callable: """ CLI option to customize configuration environment name. By default it is 'default'. Ex: --config-env default - :param f: Callback function passed by Click - :return: Callback function + + Parameters + ---------- + f: Callable + Callback function passed by Click + + Returns + ------- + Callable + A Callback function """ - config_env_attrs = {} + config_env_attrs: Dict[str, Any] = {} config_env_param_decls = ("--config-env",) config_env_attrs["help"] = "Environment name specifying default parameter values in the configuration file." - config_env_attrs["default"] = "default" + config_env_attrs["default"] = DEFAULT_ENV config_env_attrs["show_default"] = True config_env_attrs["is_eager"] = True config_env_attrs["required"] = False diff --git a/samcli/commands/_utils/custom_options/hook_name_option.py b/samcli/commands/_utils/custom_options/hook_name_option.py index 745ab6c64a..a2cb334157 100644 --- a/samcli/commands/_utils/custom_options/hook_name_option.py +++ b/samcli/commands/_utils/custom_options/hook_name_option.py @@ -141,7 +141,7 @@ def _get_customer_input_beta_features_option(default_map, experimental_entry, op if beta_features is not None: return beta_features - # Get the beta-features flag value from the SamConfig toml file if provided. + # Get the beta-features flag value from the SamConfig file if provided. beta_features = default_map.get("beta_features") if beta_features is not None: return beta_features diff --git a/samcli/commands/build/command.py b/samcli/commands/build/command.py index a60f9954e1..86327d411d 100644 --- a/samcli/commands/build/command.py +++ b/samcli/commands/build/command.py @@ -27,7 +27,7 @@ from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options, print_cmdline_args from samcli.commands.build.core.command import BuildCommand from samcli.lib.telemetry.metric import track_command -from samcli.cli.cli_config_file import configuration_option, TomlProvider +from samcli.cli.cli_config_file import configuration_option, ConfigProvider from samcli.lib.utils.version_checker import check_newer_version from samcli.commands.build.click_container import ContainerOptions from samcli.commands.build.utils import MountMode @@ -69,7 +69,7 @@ short_help=HELP_TEXT, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @hook_name_click_option( force_prepare=True, invalid_coexist_options=["t", "template-file", "template", "parameter-overrides"], diff --git a/samcli/commands/delete/delete_context.py b/samcli/commands/delete/delete_context.py index 08d327eceb..1424c87f4a 100644 --- a/samcli/commands/delete/delete_context.py +++ b/samcli/commands/delete/delete_context.py @@ -9,7 +9,7 @@ from botocore.exceptions import NoCredentialsError, NoRegionError from click import confirm, prompt -from samcli.cli.cli_config_file import TomlProvider +from samcli.cli.cli_config_file import ConfigProvider from samcli.commands.delete.exceptions import CfDeleteFailedStatusError from samcli.commands.exceptions import AWSServiceClientError, RegionError from samcli.lib.bootstrap.companion_stack.companion_stack_builder import CompanionStack @@ -82,8 +82,8 @@ def parse_config_file(self): """ Read the provided config file if it exists and assign the options values. """ - toml_provider = TomlProvider(CONFIG_SECTION, [CONFIG_COMMAND]) - config_options = toml_provider( + config_provider = ConfigProvider(CONFIG_SECTION, [CONFIG_COMMAND]) + config_options = config_provider( config_path=self.config_file, config_env=self.config_env, cmd_names=[CONFIG_COMMAND] ) if not config_options: diff --git a/samcli/commands/deploy/command.py b/samcli/commands/deploy/command.py index aeefe4d25f..557a601261 100644 --- a/samcli/commands/deploy/command.py +++ b/samcli/commands/deploy/command.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.cdk_support_decorators import unsupported_command_cdk from samcli.commands._utils.click_mutex import ClickMutex @@ -75,7 +75,7 @@ description=DESCRIPTION, requires_credentials=True, ) -@configuration_option(provider=TomlProvider(section=CONFIG_SECTION)) +@configuration_option(provider=ConfigProvider(section=CONFIG_SECTION)) @click.option( "--guided", "-g", diff --git a/samcli/commands/deploy/guided_config.py b/samcli/commands/deploy/guided_config.py index b9d8ea59b5..78866944cd 100644 --- a/samcli/commands/deploy/guided_config.py +++ b/samcli/commands/deploy/guided_config.py @@ -7,7 +7,8 @@ from samcli.cli.context import get_cmd_names from samcli.commands.deploy.exceptions import GuidedDeployFailedError -from samcli.lib.config.samconfig import DEFAULT_CONFIG_FILE_NAME, DEFAULT_ENV, SamConfig +from samcli.lib.config.exceptions import SamConfigFileReadException +from samcli.lib.config.samconfig import DEFAULT_ENV, SamConfig class GuidedConfig: @@ -19,20 +20,25 @@ def get_config_ctx(self, config_file=None): ctx = click.get_current_context() samconfig_dir = getattr(ctx, "samconfig_dir", None) + config_dir = samconfig_dir if samconfig_dir else SamConfig.config_dir(template_file_path=self.template_file) samconfig = SamConfig( - config_dir=samconfig_dir if samconfig_dir else SamConfig.config_dir(template_file_path=self.template_file), - filename=config_file or DEFAULT_CONFIG_FILE_NAME, + config_dir=config_dir, + filename=config_file or SamConfig.get_default_file(config_dir=config_dir), ) return ctx, samconfig def read_config_showcase(self, config_file=None): - _, samconfig = self.get_config_ctx(config_file) - - status = "Found" if samconfig.exists() else "Not found" msg = ( "Syntax invalid in samconfig.toml; save values " "through sam deploy --guided to overwrite file with a valid set of values." ) + try: + _, samconfig = self.get_config_ctx(config_file) + except SamConfigFileReadException: + raise GuidedDeployFailedError(msg) + + status = "Found" if samconfig.exists() else "Not found" + config_sanity = samconfig.sanity_check() click.secho("\nConfiguring SAM deploy\n======================", fg="yellow") click.echo(f"\n\tLooking for config file [{config_file}] : {status}") diff --git a/samcli/commands/init/command.py b/samcli/commands/init/command.py index 8702b3f9a9..f31e967a04 100644 --- a/samcli/commands/init/command.py +++ b/samcli/commands/init/command.py @@ -7,7 +7,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import common_options, pass_context, print_cmdline_args from samcli.commands._utils.click_mutex import ClickMutex from samcli.commands.init.core.command import InitCommand @@ -112,7 +112,7 @@ def wrapped(*args, **kwargs): description=DESCRIPTION, requires_credentials=False, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option( "--no-interactive", is_flag=True, diff --git a/samcli/commands/list/endpoints/command.py b/samcli/commands/list/endpoints/command.py index 6de1c41bb2..f11543ef8c 100644 --- a/samcli/commands/list/endpoints/command.py +++ b/samcli/commands/list/endpoints/command.py @@ -4,7 +4,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.command_exception_handler import command_exception_handler from samcli.commands._utils.options import parameter_override_option, template_option_without_build @@ -21,7 +21,7 @@ @click.command(name="endpoints", help=HELP_TEXT) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @parameter_override_option @stack_name_option @output_option diff --git a/samcli/commands/list/resources/command.py b/samcli/commands/list/resources/command.py index 5dd2b41034..dacfac30e2 100644 --- a/samcli/commands/list/resources/command.py +++ b/samcli/commands/list/resources/command.py @@ -4,7 +4,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.command_exception_handler import command_exception_handler from samcli.commands._utils.options import parameter_override_option, template_option_without_build @@ -20,7 +20,7 @@ @click.command(name="resources", help=HELP_TEXT) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @parameter_override_option @stack_name_option @output_option diff --git a/samcli/commands/list/stack_outputs/command.py b/samcli/commands/list/stack_outputs/command.py index 3800c009b2..e988f98045 100644 --- a/samcli/commands/list/stack_outputs/command.py +++ b/samcli/commands/list/stack_outputs/command.py @@ -4,7 +4,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.command_exception_handler import command_exception_handler from samcli.commands.list.cli_common.options import output_option @@ -23,7 +23,7 @@ required=True, type=click.STRING, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @output_option @aws_creds_options @common_options diff --git a/samcli/commands/local/generate_event/event_generation.py b/samcli/commands/local/generate_event/event_generation.py index 5715bded73..9bf8e49e7a 100644 --- a/samcli/commands/local/generate_event/event_generation.py +++ b/samcli/commands/local/generate_event/event_generation.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.options import debug_option from samcli.lib.generated_sample_events import events from samcli.lib.telemetry.metric import track_command @@ -160,7 +160,7 @@ def get_command(self, ctx, cmd_name): callback=command_callback, ) - cmd = configuration_option(provider=TomlProvider(section="parameters"))(debug_option(cmd)) + cmd = configuration_option(provider=ConfigProvider(section="parameters"))(debug_option(cmd)) return cmd def list_commands(self, ctx): diff --git a/samcli/commands/local/invoke/cli.py b/samcli/commands/local/invoke/cli.py index a9a3fc9571..0442e7b7ed 100644 --- a/samcli/commands/local/invoke/cli.py +++ b/samcli/commands/local/invoke/cli.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.experimental import ExperimentalFlag, is_experimental_enabled @@ -43,7 +43,7 @@ short_help=HELP_TEXT, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @hook_name_click_option( force_prepare=False, invalid_coexist_options=["t", "template-file", "template", "parameter-overrides"] ) diff --git a/samcli/commands/local/start_api/cli.py b/samcli/commands/local/start_api/cli.py index 9de4d7982c..3e2f02b5f3 100644 --- a/samcli/commands/local/start_api/cli.py +++ b/samcli/commands/local/start_api/cli.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.experimental import ExperimentalFlag, is_experimental_enabled @@ -58,7 +58,7 @@ requires_credentials=False, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @hook_name_click_option( force_prepare=False, invalid_coexist_options=["t", "template-file", "template", "parameter-overrides"] ) diff --git a/samcli/commands/local/start_lambda/cli.py b/samcli/commands/local/start_lambda/cli.py index 9aaec50976..ded8b786fc 100644 --- a/samcli/commands/local/start_lambda/cli.py +++ b/samcli/commands/local/start_lambda/cli.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.experimental import ExperimentalFlag, is_experimental_enabled @@ -52,7 +52,7 @@ requires_credentials=False, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @hook_name_click_option( force_prepare=False, invalid_coexist_options=["t", "template-file", "template", "parameter-overrides"] ) diff --git a/samcli/commands/logs/command.py b/samcli/commands/logs/command.py index 1146767a60..f8df636b59 100644 --- a/samcli/commands/logs/command.py +++ b/samcli/commands/logs/command.py @@ -6,7 +6,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -49,7 +49,7 @@ description=DESCRIPTION, requires_credentials=True, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option( "--name", "-n", diff --git a/samcli/commands/package/command.py b/samcli/commands/package/command.py index 41fb10b133..151661c91e 100644 --- a/samcli/commands/package/command.py +++ b/samcli/commands/package/command.py @@ -3,7 +3,7 @@ """ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.cdk_support_decorators import unsupported_command_cdk from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -67,7 +67,7 @@ def resources_and_properties_help_string(): description=DESCRIPTION, requires_credentials=True, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @template_click_option(include_build=True) @click.option( "--output-template-file", diff --git a/samcli/commands/pipeline/bootstrap/cli.py b/samcli/commands/pipeline/bootstrap/cli.py index 9b7db02578..7ba17e43fe 100644 --- a/samcli/commands/pipeline/bootstrap/cli.py +++ b/samcli/commands/pipeline/bootstrap/cli.py @@ -7,7 +7,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.commands._utils.click_mutex import ClickMutex from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -38,7 +38,7 @@ @click.command("bootstrap", short_help=SHORT_HELP, help=HELP_TEXT, context_settings=dict(max_content_width=120)) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option( "--interactive/--no-interactive", is_flag=True, diff --git a/samcli/commands/pipeline/init/cli.py b/samcli/commands/pipeline/init/cli.py index 9e42f6e74b..ec675fb608 100644 --- a/samcli/commands/pipeline/init/cli.py +++ b/samcli/commands/pipeline/init/cli.py @@ -5,7 +5,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import common_options as cli_framework_options from samcli.cli.main import pass_context from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -24,7 +24,7 @@ @click.command("init", help=HELP_TEXT, short_help=SHORT_HELP) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option( "--bootstrap", is_flag=True, diff --git a/samcli/commands/publish/command.py b/samcli/commands/publish/command.py index 091a044192..4ebfb3e8e5 100644 --- a/samcli/commands/publish/command.py +++ b/samcli/commands/publish/command.py @@ -7,7 +7,7 @@ import click from serverlessrepo.publish import CREATE_APPLICATION -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -44,7 +44,7 @@ @click.command("publish", help=HELP_TEXT, short_help=SHORT_HELP) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @template_common_option @click.option("--semantic-version", help=SEMANTIC_VERSION_HELP) @aws_creds_options diff --git a/samcli/commands/remote/invoke/cli.py b/samcli/commands/remote/invoke/cli.py index 3f3a771ea1..0318566b4a 100644 --- a/samcli/commands/remote/invoke/cli.py +++ b/samcli/commands/remote/invoke/cli.py @@ -4,7 +4,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.context import Context from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.cli.types import RemoteInvokeOutputFormatType @@ -42,7 +42,7 @@ requires_credentials=True, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option("--stack-name", required=False, help="Name of the stack to get the resource information from") @click.argument("resource-id", required=False) @click.option( diff --git a/samcli/commands/sync/command.py b/samcli/commands/sync/command.py index 81f1222207..ddc5e2a165 100644 --- a/samcli/commands/sync/command.py +++ b/samcli/commands/sync/command.py @@ -5,7 +5,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.context import Context from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options @@ -114,7 +114,7 @@ requires_credentials=True, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @template_option_without_build @click.option( "--code", diff --git a/samcli/commands/traces/command.py b/samcli/commands/traces/command.py index 183a2bd156..d82b6871ab 100644 --- a/samcli/commands/traces/command.py +++ b/samcli/commands/traces/command.py @@ -5,7 +5,7 @@ import click -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options from samcli.commands._utils.command_exception_handler import command_exception_handler @@ -28,7 +28,7 @@ @click.command("traces", help=HELP_TEXT, short_help="Fetch AWS X-Ray traces") -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @click.option( "--trace-id", "-ti", diff --git a/samcli/commands/validate/validate.py b/samcli/commands/validate/validate.py index 1c5c2b28b1..221284b1ac 100644 --- a/samcli/commands/validate/validate.py +++ b/samcli/commands/validate/validate.py @@ -8,7 +8,7 @@ from botocore.exceptions import NoCredentialsError from samtranslator.translator.arn_generator import NoRegionFound -from samcli.cli.cli_config_file import TomlProvider, configuration_option +from samcli.cli.cli_config_file import ConfigProvider, configuration_option from samcli.cli.context import Context from samcli.cli.main import aws_creds_options, pass_context, print_cmdline_args from samcli.cli.main import common_options as cli_framework_options @@ -35,7 +35,7 @@ requires_credentials=False, context_settings={"max_content_width": 120}, ) -@configuration_option(provider=TomlProvider(section="parameters")) +@configuration_option(provider=ConfigProvider(section="parameters")) @template_option_without_build @aws_creds_options @cli_framework_options diff --git a/samcli/lib/config/exceptions.py b/samcli/lib/config/exceptions.py index 50297ce722..c179b4a13c 100644 --- a/samcli/lib/config/exceptions.py +++ b/samcli/lib/config/exceptions.py @@ -4,4 +4,12 @@ class SamConfigVersionException(Exception): - pass + """Exception for the `samconfig` file being not present or in unrecognized format""" + + +class FileParseException(Exception): + """Exception when a file is incorrectly parsed by a FileManager object.""" + + +class SamConfigFileReadException(Exception): + """Exception when a `samconfig` file is read incorrectly.""" diff --git a/samcli/lib/config/file_manager.py b/samcli/lib/config/file_manager.py new file mode 100644 index 0000000000..0629ace318 --- /dev/null +++ b/samcli/lib/config/file_manager.py @@ -0,0 +1,342 @@ +""" +Class to represent the parsing of different file types into Python objects. +""" + + +import json +import logging +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any, Dict, Type + +import tomlkit +from ruamel.yaml import YAML, YAMLError +from ruamel.yaml.compat import StringIO + +from samcli.lib.config.exceptions import FileParseException + +LOG = logging.getLogger(__name__) +COMMENT_KEY = "__comment__" + + +class FileManager(ABC): + """ + Abstract class to be overridden by file managers for specific file extensions. + """ + + @staticmethod + @abstractmethod + def read(filepath: Path) -> Any: + """ + Read a file at a given path. + + Parameters + ---------- + filepath: Path + The Path object that points to the file to be read. + + Returns + ------- + Any + A dictionary-like representation of the contents at the filepath location. + """ + raise NotImplementedError("Read method not implemented.") + + @staticmethod + @abstractmethod + def write(document: dict, filepath: Path): + """ + Write a dictionary or dictionary-like object to a given file. + + Parameters + ---------- + document: dict + The object to write. + filepath: Path + The final location for the document to be written. + """ + raise NotImplementedError("Write method not implemented.") + + @staticmethod + @abstractmethod + def put_comment(document: Any, comment: str) -> Any: + """ + Put a comment in a document object. + + Parameters + ---------- + document: Any + The object to write + comment: str + The comment to include in the document. + + Returns + ------- + Any + The new document, with the comment added to it. + """ + raise NotImplementedError("Put comment method not implemented.") + + +class TomlFileManager(FileManager): + """ + Static class to read and write toml files. + """ + + file_format = "TOML" + + @staticmethod + def read(filepath: Path) -> Any: + """ + Read a TOML file at the given path. + + Parameters + ---------- + filepath: Path + The Path object that points to the file to be read. + + Returns + ------- + Any + A dictionary-like tomlkit.TOMLDocument object, which represents the contents of the TOML file at the + provided location. + """ + toml_doc = tomlkit.document() + try: + txt = filepath.read_text() + toml_doc = tomlkit.loads(txt) + except OSError as e: + LOG.debug(f"OSError occurred while reading {TomlFileManager.file_format} file: {str(e)}") + except tomlkit.exceptions.TOMLKitError as e: + raise FileParseException(e) from e + + return toml_doc + + @staticmethod + def write(document: dict, filepath: Path): + """ + Write the contents of a dictionary or tomlkit.TOMLDocument to a TOML file at the provided location. + + Parameters + ---------- + document: dict + The object to write. + filepath: Path + The final location for the TOML file to be written. + """ + if not document: + LOG.debug("Nothing for TomlFileManager to write.") + return + + toml_document = TomlFileManager._to_toml(document) + + if toml_document.get(COMMENT_KEY, None): # Remove dunder comments that may be residue from other formats + toml_document.add(tomlkit.comment(toml_document.get(COMMENT_KEY, ""))) + toml_document.pop(COMMENT_KEY) + + filepath.write_text(tomlkit.dumps(toml_document)) + + @staticmethod + def put_comment(document: dict, comment: str) -> Any: + """ + Put a comment in a document object. + + Parameters + ---------- + document: Any + The tomlkit.TOMLDocument object to write + comment: str + The comment to include in the document. + + Returns + ------- + Any + The new TOMLDocument, with the comment added to it. + """ + document = TomlFileManager._to_toml(document) + document.add(tomlkit.comment(comment)) + return document + + @staticmethod + def _to_toml(document: dict) -> tomlkit.TOMLDocument: + """Ensure that a dictionary-like object is a TOMLDocument.""" + return tomlkit.parse(tomlkit.dumps(document)) + + +class YamlFileManager(FileManager): + """ + Static class to read and write yaml files. + """ + + yaml = YAML() + file_format = "YAML" + + @staticmethod + def read(filepath: Path) -> Any: + """ + Read a YAML file at the given path. + + Parameters + ---------- + filepath: Path + The Path object that points to the file to be read. + + Returns + ------- + Any + A dictionary-like yaml object, which represents the contents of the YAML file at the + provided location. + """ + yaml_doc = {} + try: + yaml_doc = YamlFileManager.yaml.load(filepath.read_text()) + except OSError as e: + LOG.debug(f"OSError occurred while reading {YamlFileManager.file_format} file: {str(e)}") + except YAMLError as e: + raise FileParseException(e) from e + + return yaml_doc + + @staticmethod + def write(document: dict, filepath: Path): + """ + Write the contents of a dictionary to a YAML file at the provided location. + + Parameters + ---------- + document: dict + The object to write. + filepath: Path + The final location for the YAML file to be written. + """ + if not document: + LOG.debug("No document given to YamlFileManager to write.") + return + + yaml_doc = YamlFileManager._to_yaml(document) + + if yaml_doc.get(COMMENT_KEY, None): # Comment appears at the top of doc + yaml_doc.yaml_set_start_comment(document[COMMENT_KEY]) + yaml_doc.pop(COMMENT_KEY) + + YamlFileManager.yaml.dump(yaml_doc, filepath) + + @staticmethod + def put_comment(document: Any, comment: str) -> Any: + """ + Put a comment in a document object. + + Parameters + ---------- + document: Any + The yaml object to write + comment: str + The comment to include in the document. + + Returns + ------- + Any + The new yaml document, with the comment added to it. + """ + document = YamlFileManager._to_yaml(document) + document.yaml_set_start_comment(comment) + return document + + @staticmethod + def _to_yaml(document: dict) -> Any: + """ + Ensure a dictionary-like object is a YAML document. + + Parameters + ---------- + document: dict + A dictionary-like object to parse. + + Returns + ------- + Any + A dictionary-like YAML object, as derived from `yaml.load()`. + """ + with StringIO() as stream: + YamlFileManager.yaml.dump(document, stream) + return YamlFileManager.yaml.load(stream.getvalue()) + + +class JsonFileManager(FileManager): + """ + Static class to read and write json files. + """ + + file_format = "JSON" + INDENT_SIZE = 2 + + @staticmethod + def read(filepath: Path) -> Any: + """ + Read a JSON file at a given path. + + Parameters + ---------- + filepath: Path + The Path object that points to the file to be read. + + Returns + ------- + Any + A dictionary representation of the contents of the JSON document. + """ + json_file = {} + try: + json_file = json.loads(filepath.read_text()) + except OSError as e: + LOG.debug(f"OSError occurred while reading {JsonFileManager.file_format} file: {str(e)}") + except json.JSONDecodeError as e: + raise FileParseException(e) from e + return json_file + + @staticmethod + def write(document: dict, filepath: Path): + """ + Write a dictionary or dictionary-like object to a JSON file. + + Parameters + ---------- + document: dict + The JSON object to write. + filepath: Path + The final location for the document to be written. + """ + if not document: + LOG.debug("No document given to JsonFileManager to write.") + return + + with filepath.open("w") as file: + json.dump(document, file, indent=JsonFileManager.INDENT_SIZE) + + @staticmethod + def put_comment(document: Any, comment: str) -> Any: + """ + Put a comment in a JSON object. + + Parameters + ---------- + document: Any + The JSON object to write + comment: str + The comment to include in the document. + + Returns + ------- + Any + The new JSON dictionary object, with the comment added to it. + """ + document.update({COMMENT_KEY: comment}) + return document + + +FILE_MANAGER_MAPPER: Dict[str, Type[FileManager]] = { # keys ordered by priority + ".toml": TomlFileManager, + ".yaml": YamlFileManager, + ".yml": YamlFileManager, + # ".json": JsonFileManager, # JSON support disabled +} diff --git a/samcli/lib/config/samconfig.py b/samcli/lib/config/samconfig.py index e48e53d625..e9acafd557 100644 --- a/samcli/lib/config/samconfig.py +++ b/samcli/lib/config/samconfig.py @@ -7,26 +7,25 @@ from pathlib import Path from typing import Any, Iterable -import tomlkit - -from samcli.lib.config.exceptions import SamConfigVersionException +from samcli.lib.config.exceptions import FileParseException, SamConfigFileReadException, SamConfigVersionException +from samcli.lib.config.file_manager import FILE_MANAGER_MAPPER from samcli.lib.config.version import SAM_CONFIG_VERSION, VERSION_KEY +from samcli.lib.telemetry.event import EventTracker LOG = logging.getLogger(__name__) -DEFAULT_CONFIG_FILE_EXTENSION = "toml" -DEFAULT_CONFIG_FILE_NAME = f"samconfig.{DEFAULT_CONFIG_FILE_EXTENSION}" +DEFAULT_CONFIG_FILE_EXTENSION = ".toml" +DEFAULT_CONFIG_FILE = "samconfig" +DEFAULT_CONFIG_FILE_NAME = DEFAULT_CONFIG_FILE + DEFAULT_CONFIG_FILE_EXTENSION DEFAULT_ENV = "default" DEFAULT_GLOBAL_CMDNAME = "global" class SamConfig: """ - Class to interface with `samconfig.toml` file. + Class to represent `samconfig` config options. """ - document = None - def __init__(self, config_dir, filename=None): """ Initialize the class @@ -39,11 +38,23 @@ def __init__(self, config_dir, filename=None): Optional. Name of the configuration file. It is recommended to stick with default so in the future we could automatically support auto-resolving multiple config files within same directory. """ - self.filepath = Path(config_dir, filename or DEFAULT_CONFIG_FILE_NAME) + self.document = {} + self.filepath = Path(config_dir, filename or self.get_default_file(config_dir=config_dir)) + file_extension = self.filepath.suffix + self.file_manager = FILE_MANAGER_MAPPER.get(file_extension, None) + if not self.file_manager: + LOG.warning( + f"The config file extension '{file_extension}' is not supported. " + f"Supported formats are: [{'|'.join(FILE_MANAGER_MAPPER.keys())}]" + ) + raise SamConfigFileReadException( + f"The config file {self.filepath} uses an unsupported extension, and cannot be read." + ) + self._read() + EventTracker.track_event("SamConfigFileExtension", file_extension) def get_stage_configuration_names(self): - self._read() - if isinstance(self.document, dict): + if self.document: return [stage for stage, value in self.document.items() if isinstance(value, dict)] return [] @@ -69,23 +80,19 @@ def get_all(self, cmd_names, section, env=DEFAULT_ENV): ------ KeyError If the config file does *not* have the specific section - - tomlkit.exceptions.TOMLKitError - If the configuration file is invalid """ env = env or DEFAULT_ENV - self._read() - if isinstance(self.document, dict): - toml_content = self.document.get(env, {}) - params = toml_content.get(self._to_key(cmd_names), {}).get(section, {}) - if DEFAULT_GLOBAL_CMDNAME in toml_content: - global_params = toml_content.get(DEFAULT_GLOBAL_CMDNAME, {}).get(section, {}) - global_params.update(params.copy()) - params = global_params.copy() - return params - return {} + self.document = self._read() + + config_content = self.document.get(env, {}) + params = config_content.get(self._to_key(cmd_names), {}).get(section, {}) + if DEFAULT_GLOBAL_CMDNAME in config_content: + global_params = config_content.get(DEFAULT_GLOBAL_CMDNAME, {}).get(section, {}) + global_params.update(params.copy()) + params = global_params.copy() + return params def put(self, cmd_names, section, key, value, env=DEFAULT_ENV): """ @@ -102,20 +109,10 @@ def put(self, cmd_names, section, key, value, env=DEFAULT_ENV): key : str Key to write the data under value : Any - Value to write. Could be any of the supported TOML types. + Value to write. Could be any of the supported types. env : str Optional, Name of the environment - - Raises - ------ - tomlkit.exceptions.TOMLKitError - If the data is invalid """ - - if self.document is None: - # Checking for None here since a TOMLDocument can include a - # 'body' property but still be falsy without a 'value' property - self._read() # Empty document prepare the initial structure. # self.document is a nested dict, we need to check each layer and add new tables, otherwise duplicated key # in parent layer will override the whole child layer @@ -144,20 +141,12 @@ def put_comment(self, comment): comment: str A comment to write to the samconfg file """ - if self.document is None: - self._read() - self.document.add(tomlkit.comment(comment)) + self.document = self.file_manager.put_comment(self.document, comment) def flush(self): """ Write the data back to file - - Raises - ------ - tomlkit.exceptions.TOMLKitError - If the data is invalid - """ self._write() @@ -167,7 +156,7 @@ def sanity_check(self): """ try: self._read() - except tomlkit.exceptions.TOMLKitError: + except SamConfigFileReadException: return False else: return True @@ -196,13 +185,10 @@ def config_dir(template_file_path=None): def _read(self): if not self.document: try: - txt = self.filepath.read_text() - self.document = tomlkit.loads(txt) - self._version_sanity_check(self._version()) - except OSError: - self.document = tomlkit.document() - - if self.document.body: + self.document = self.file_manager.read(self.filepath) + except FileParseException as e: + raise SamConfigFileReadException(e) from e + if self.document: self._version_sanity_check(self._version()) return self.document @@ -213,12 +199,9 @@ def _write(self): self._ensure_exists() current_version = self._version() if self._version() else SAM_CONFIG_VERSION - try: - self.document.add(VERSION_KEY, current_version) - except tomlkit.exceptions.KeyAlreadyPresent: - # NOTE(TheSriram): Do not attempt to re-write an existing version - pass - self.filepath.write_text(tomlkit.dumps(self.document)) + self.document.update({VERSION_KEY: current_version}) + + self.file_manager.write(self.document, self.filepath) def _version(self): return self.document.get(VERSION_KEY, None) @@ -261,6 +244,40 @@ def _deduplicate_global_parameters(self, cmd_name_key, section, key, env=DEFAULT # Only keep the global parameter del self.document[env][cmd_name_key][section][key] + @staticmethod + def get_default_file(config_dir: str) -> str: + """Return a defaultly-named config file, if it exists, otherwise the current default. + + Parameters + ---------- + config_dir: str + The name of the directory where the config file is/will be stored. + + Returns + ------- + str + The name of the config file found, if it exists. In the case that it does not exist, the default config + file name is returned instead. + """ + config_files_found = 0 + config_file = DEFAULT_CONFIG_FILE_NAME + + for extension in reversed(list(FILE_MANAGER_MAPPER.keys())): + filename = DEFAULT_CONFIG_FILE + extension + if Path(config_dir, filename).exists(): + config_files_found += 1 + config_file = filename + + if config_files_found == 0: # Config file doesn't exist (yet!) + LOG.debug("No config file found in this directory.") + elif config_files_found > 1: # Multiple config files; let user know which is used + LOG.info( + f"More than one samconfig file found; using {config_file}." + f" To use another config file, please specify it using the '--config-file' flag." + ) + + return config_file + @staticmethod def _version_sanity_check(version: Any) -> None: if not isinstance(version, float): diff --git a/samcli/lib/telemetry/event.py b/samcli/lib/telemetry/event.py index 2d819e37bf..2c35748951 100644 --- a/samcli/lib/telemetry/event.py +++ b/samcli/lib/telemetry/event.py @@ -11,6 +11,7 @@ from samcli.cli.context import Context from samcli.lib.build.workflows import ALL_CONFIGS +from samcli.lib.config.file_manager import FILE_MANAGER_MAPPER from samcli.lib.telemetry.telemetry import Telemetry from samcli.local.common.runtime_template import INIT_RUNTIMES @@ -26,6 +27,7 @@ class EventName(Enum): SYNC_FLOW_START = "SyncFlowStart" SYNC_FLOW_END = "SyncFlowEnd" BUILD_WORKFLOW_USED = "BuildWorkflowUsed" + CONFIG_FILE_EXTENSION = "SamConfigFileExtension" class UsedFeature(Enum): @@ -69,6 +71,7 @@ class EventType: EventName.SYNC_FLOW_START: _SYNC_FLOWS, EventName.SYNC_FLOW_END: _SYNC_FLOWS, EventName.BUILD_WORKFLOW_USED: _WORKFLOWS, + EventName.CONFIG_FILE_EXTENSION: list(FILE_MANAGER_MAPPER.keys()), } @staticmethod diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py index 13ba14f310..2dab556843 100644 --- a/tests/integration/buildcmd/build_integ_base.py +++ b/tests/integration/buildcmd/build_integ_base.py @@ -83,6 +83,7 @@ def get_command_list( beta_features=None, build_in_source=None, mount_with=None, + config_file=None, ): command_list = [self.cmd, "build"] @@ -146,6 +147,9 @@ def get_command_list( if build_in_source is not None: command_list += ["--build-in-source"] if build_in_source else ["--no-build-in-source"] + if config_file is not None: + command_list += ["--config-file", config_file] + return command_list def verify_docker_container_cleanedup(self, runtime): diff --git a/tests/integration/buildcmd/test_build_samconfig.py b/tests/integration/buildcmd/test_build_samconfig.py new file mode 100644 index 0000000000..3df5052599 --- /dev/null +++ b/tests/integration/buildcmd/test_build_samconfig.py @@ -0,0 +1,114 @@ +import os +from pathlib import Path +from parameterized import parameterized, parameterized_class + +from tests.integration.buildcmd.build_integ_base import BuildIntegBase +from tests.testing_utils import run_command + + +configs = { + ".toml": "samconfig/samconfig.toml", + ".yaml": "samconfig/samconfig.yaml", + ".yml": "samconfig/samconfig.yml", + ".json": "samconfig/samconfig.json", +} + + +class TestSamConfigWithBuild(BuildIntegBase): + @parameterized.expand( + [ + (".toml"), + (".yaml"), + # (".json"), + ] + ) + def test_samconfig_works_with_extension(self, extension): + cmdlist = self.get_command_list(config_file=configs[extension]) + + command_result = run_command(cmdlist, cwd=self.working_dir) + stdout = str(command_result[1]) + stderr = str(command_result[2]) + + self.assertEqual(command_result.process.returncode, 0, "Build should succeed") + self.assertIn( + f"Built Artifacts : {extension}", + stdout, + f"Build template should use build_dir from samconfig{extension}", + ) + self.assertIn("Starting Build use cache", stderr, f"'cache'=true should be set in samconfig{extension}") + + @parameterized.expand( + [ + (".toml"), + (".yaml"), + # (".json"), + ] + ) + def test_samconfig_parameters_are_overridden(self, extension): + overrides = {"Runtime": "python3.8"} + overridden_build_dir = f"override_{extension}" + + cmdlist = self.get_command_list( + config_file=configs[extension], parameter_overrides=overrides, build_dir=overridden_build_dir + ) + + command_result = run_command(cmdlist, cwd=self.working_dir) + stdout = str(command_result[1]) + stderr = str(command_result[2]) + + self.assertEqual(command_result.process.returncode, 0, "Build should succeed") + self.assertNotIn( + f"Built Artifacts : {extension}", + stdout, + f"Build template should not use build_dir from samconfig{extension}", + ) + self.assertIn( + f"Built Artifacts : {overridden_build_dir}", stdout, f"Build template should use overridden build_dir" + ) + self.assertIn("Starting Build use cache", stderr, f"'cache'=true should be set in samconfig{extension}") + self.assertNotIn("python3.9", stderr, f"parameter_overrides runtime should not read from samconfig{extension}") + self.assertIn(overrides["Runtime"], stderr, "parameter_overrides should use overridden runtime") + self.assertNotIn("SomeURI", stderr, f"parameter_overrides should not read ANY values from samconfig{extension}") + + +@parameterized_class( + [ # Ordered by expected priority + {"extensions": [".toml", ".yaml", ".yml"]}, + {"extensions": [".yaml", ".yml"]}, + ] +) +class TestSamConfigExtensionHierarchy(BuildIntegBase): + def setUp(self): + super().setUp() + new_template_location = Path(self.working_dir, "template.yaml") + new_template_location.write_text(Path(self.template_path).read_text()) + for extension in self.extensions: + config_contents = Path(self.test_data_path, configs[extension]).read_text() + new_path = Path(self.working_dir, f"samconfig{extension}") + new_path.write_text(config_contents) + self.assertTrue(new_path.exists(), f"File samconfig{extension} should have been created in cwd") + + def tearDown(self): + for extension in self.extensions: + config_path = Path(self.working_dir, f"samconfig{extension}") + os.remove(config_path) + super().tearDown() + + def test_samconfig_pulls_correct_file_if_multiple(self): + self.template_path = str(Path(self.working_dir, "template.yaml")) + cmdlist = self.get_command_list(debug=True) + command_result = run_command(cmdlist, cwd=self.working_dir) + stdout = str(command_result[1]) + + self.assertEqual(command_result.process.returncode, 0, "Build should succeed") + self.assertIn( + f" {self.extensions[0]}", + stdout, + f"samconfig{self.extensions[0]} should take priority in current test group", + ) + for other_extension in self.extensions[1:]: + self.assertNotIn( + f" {other_extension}", + stdout, + f"samconfig{other_extension} should not be read over another, higher priority extension", + ) diff --git a/tests/integration/deploy/deploy_integ_base.py b/tests/integration/deploy/deploy_integ_base.py index f48eae9d61..4db4e5ff6a 100644 --- a/tests/integration/deploy/deploy_integ_base.py +++ b/tests/integration/deploy/deploy_integ_base.py @@ -2,11 +2,13 @@ import tempfile from pathlib import Path from enum import Enum, auto +from typing import List, Optional import boto3 from botocore.config import Config from samcli.lib.bootstrap.bootstrap import SAM_CLI_STACK_NAME +from samcli.lib.config.samconfig import SamConfig from tests.integration.package.package_integ_base import PackageIntegBase from tests.testing_utils import get_sam_command, run_command, run_command_with_input @@ -212,3 +214,26 @@ def get_minimal_build_command_list(template_file=None, build_dir=None): command_list = command_list + ["--build-dir", str(build_dir)] return command_list + + def _assert_deploy_samconfig_parameters( + self, + config: SamConfig, + stack_name: str = SAM_CLI_STACK_NAME, + resolve_s3: bool = True, + region: str = "us-east-1", + capabilities: str = "CAPABILITY_IAM", + confirm_changeset: Optional[bool] = None, + parameter_overrides: Optional[str] = None, + ): + params = config.document["default"]["deploy"]["parameters"] + + self.assertEqual(params["stack_name"], stack_name) + self.assertEqual(params["resolve_s3"], resolve_s3) + self.assertEqual(params["region"], region) + self.assertEqual(params["capabilities"], capabilities) + + if confirm_changeset is not None: + self.assertEqual(params["confirm_changeset"], confirm_changeset) + + if parameter_overrides is not None: + self.assertEqual(params["parameter_overrides"], parameter_overrides) diff --git a/tests/integration/deploy/test_deploy_command.py b/tests/integration/deploy/test_deploy_command.py index 131d3969ab..a7ece04a5c 100644 --- a/tests/integration/deploy/test_deploy_command.py +++ b/tests/integration/deploy/test_deploy_command.py @@ -10,7 +10,7 @@ from parameterized import parameterized from samcli.lib.bootstrap.bootstrap import SAM_CLI_STACK_NAME -from samcli.lib.config.samconfig import DEFAULT_CONFIG_FILE_NAME +from samcli.lib.config.samconfig import DEFAULT_CONFIG_FILE_NAME, SamConfig from tests.integration.deploy.deploy_integ_base import DeployIntegBase from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY, UpdatableSARTemplate @@ -613,6 +613,13 @@ def test_deploy_guided_zip(self, template_file): # Deploy should succeed with a managed stack self.assertEqual(deploy_process_execute.process.returncode, 0) self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + deploy_config_params = config.document["default"]["deploy"]["parameters"] + self.assertEqual(deploy_config_params["stack_name"], stack_name) + self.assertTrue(deploy_config_params["resolve_s3"]) + self.assertEqual(deploy_config_params["region"], "us-east-1") + self.assertEqual(deploy_config_params["capabilities"], "CAPABILITY_IAM") # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -627,7 +634,7 @@ def test_deploy_guided_image_auto(self, template_file): deploy_command_list = self.get_deploy_command_list(template_file=template_path, guided=True) deploy_process_execute = self.run_command_with_input( - deploy_command_list, f"{stack_name}\n\n\n\n\ny\n\n\ny\n\n\n\n".encode() + deploy_command_list, f"{stack_name}\n\n\n\n\ny\n\n\n\n\n\n\n".encode() ) # Deploy should succeed with a managed stack @@ -638,6 +645,10 @@ def test_deploy_guided_image_auto(self, template_file): self._assert_companion_stack(self.cfn_client, companion_stack_name) self._assert_companion_stack_content(self.ecr_client, companion_stack_name) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters(config, stack_name=stack_name) + # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -669,6 +680,9 @@ def test_deploy_guided_image_specify(self, template_file, does_ask_for_authoriza self.fail("Companion stack was created. This should not happen with specifying image repos.") self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters(config, stack_name=stack_name) # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -690,6 +704,11 @@ def test_deploy_guided_set_parameter(self, template_file): # Deploy should succeed with a managed stack self.assertEqual(deploy_process_execute.process.returncode, 0) self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters( + config, stack_name=stack_name, parameter_overrides='Parameter="SuppliedParameter"' + ) # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -710,6 +729,14 @@ def test_deploy_guided_set_capabilities(self, template_file): # Deploy should succeed with a managed stack self.assertEqual(deploy_process_execute.process.returncode, 0) self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters( + config, + stack_name=stack_name, + capabilities="CAPABILITY_IAM CAPABILITY_NAMED_IAM", + parameter_overrides='Parameter="SuppliedParameter"', + ) # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -731,6 +758,11 @@ def test_deploy_guided_capabilities_default(self, template_file): # Deploy should succeed with a managed stack self.assertEqual(deploy_process_execute.process.returncode, 0) self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters( + config, stack_name=stack_name, parameter_overrides='Parameter="SuppliedParameter"' + ) # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -752,6 +784,11 @@ def test_deploy_guided_set_confirm_changeset(self, template_file): # Deploy should succeed with a managed stack self.assertEqual(deploy_process_execute.process.returncode, 0) self.stacks.append({"name": SAM_CLI_STACK_NAME}) + # Verify the contents in samconfig + config = SamConfig(self.test_data_path) + self._assert_deploy_samconfig_parameters( + config, stack_name=stack_name, confirm_changeset=True, parameter_overrides='Parameter="SuppliedParameter"' + ) # Remove samconfig.toml os.remove(self.test_data_path.joinpath(DEFAULT_CONFIG_FILE_NAME)) @@ -789,7 +826,7 @@ def test_deploy_with_invalid_config(self, template_file, config_file): deploy_process_execute = self.run_command(deploy_command_list) self.assertEqual(deploy_process_execute.process.returncode, 1) - self.assertIn("Error reading configuration: Unexpected character", str(deploy_process_execute.stderr)) + self.assertIn("SamConfigFileReadException: Unexpected character", str(deploy_process_execute.stderr)) @parameterized.expand([("aws-serverless-function.yaml", "samconfig-tags-list.toml")]) def test_deploy_with_valid_config_tags_list(self, template_file, config_file): diff --git a/tests/integration/telemetry/test_experimental_metric.py b/tests/integration/telemetry/test_experimental_metric.py index 977e65053f..702ceb4a5f 100644 --- a/tests/integration/telemetry/test_experimental_metric.py +++ b/tests/integration/telemetry/test_experimental_metric.py @@ -211,8 +211,11 @@ def test_must_send_not_experimental_metrics_if_not_experimental(self): self.assertEqual(process.returncode, 2, "Command should fail") all_requests = server.get_all_requests() - self.assertEqual(1, len(all_requests), "Command run metric must be sent") - request = all_requests[0] + self.assertEqual(2, len(all_requests), "Command run and event metrics must be sent") + # NOTE: Since requests happen asynchronously, we cannot guarantee whether the + # commandRun metric will be first or second, so we sort for consistency. + all_requests.sort(key=lambda x: list(x["data"]["metrics"][0].keys())[0]) + request = all_requests[0] # "commandRun" comes before "events" self.assertIn("Content-Type", request["headers"]) self.assertEqual(request["headers"]["Content-Type"], "application/json") diff --git a/tests/integration/telemetry/test_installed_metric.py b/tests/integration/telemetry/test_installed_metric.py index e17459828c..725a2d0ff5 100644 --- a/tests/integration/telemetry/test_installed_metric.py +++ b/tests/integration/telemetry/test_installed_metric.py @@ -24,7 +24,9 @@ def test_send_installed_metric_on_first_run(self): self.assertIn(EXPECTED_TELEMETRY_PROMPT, stderrdata.decode()) all_requests = server.get_all_requests() - self.assertEqual(2, len(all_requests), "There should be exactly two metrics request") + self.assertEqual( + 3, len(all_requests), "There should be exactly three metrics request" + ) # 3 = 2 expected + events # First one is usually the installed metric requests = filter_installed_metric_requests(all_requests) diff --git a/tests/integration/telemetry/test_telemetry_contract.py b/tests/integration/telemetry/test_telemetry_contract.py index 08b3585b99..a3e383bb5d 100644 --- a/tests/integration/telemetry/test_telemetry_contract.py +++ b/tests/integration/telemetry/test_telemetry_contract.py @@ -28,7 +28,9 @@ def test_must_not_send_metrics_if_disabled_using_envvar(self): self.assertEqual(process.returncode, 0, "Command should successfully complete") all_requests = server.get_all_requests() - self.assertEqual(1, len(all_requests), "Command run metric should be sent") + self.assertEqual( + 2, len(all_requests), "Command run and event metrics should be sent" + ) # 2 = cmd_run + events def test_must_send_metrics_if_enabled_via_envvar(self): """ @@ -52,7 +54,7 @@ def test_must_send_metrics_if_enabled_via_envvar(self): self.assertEqual(process.returncode, 0, "Command should successfully complete") all_requests = server.get_all_requests() - self.assertEqual(1, len(all_requests), "Command run metric must be sent") + self.assertEqual(2, len(all_requests), "Command run and event metrics must be sent") # cmd_run + events def test_must_not_crash_when_offline(self): """ diff --git a/tests/integration/testdata/buildcmd/samconfig/samconfig.json b/tests/integration/testdata/buildcmd/samconfig/samconfig.json new file mode 100644 index 0000000000..6fa18e1c11 --- /dev/null +++ b/tests/integration/testdata/buildcmd/samconfig/samconfig.json @@ -0,0 +1,12 @@ +{ + "version": 0.1, + "default": { + "build": { + "parameters": { + "build_dir": ".json", + "cached": true, + "parameter_overrides": "Runtime=python3.9 CodeUri=SomeURI Handler=SomeHandler" + } + } + } +} \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/samconfig/samconfig.toml b/tests/integration/testdata/buildcmd/samconfig/samconfig.toml new file mode 100644 index 0000000000..23a769ff5e --- /dev/null +++ b/tests/integration/testdata/buildcmd/samconfig/samconfig.toml @@ -0,0 +1,5 @@ +version = 0.1 +[default.build.parameters] +build_dir = ".toml" +cached = true +parameter_overrides = "Runtime=python3.9 CodeUri=SomeURI Handler=SomeHandler" \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/samconfig/samconfig.yaml b/tests/integration/testdata/buildcmd/samconfig/samconfig.yaml new file mode 100644 index 0000000000..63af206238 --- /dev/null +++ b/tests/integration/testdata/buildcmd/samconfig/samconfig.yaml @@ -0,0 +1,7 @@ +version: 0.1 +default: + build: + parameters: + build_dir: .yaml + cached: true + parameter_overrides: Runtime=python3.9 CodeUri=SomeURI Handler=SomeHandler \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/samconfig/samconfig.yml b/tests/integration/testdata/buildcmd/samconfig/samconfig.yml new file mode 100644 index 0000000000..4af8baa434 --- /dev/null +++ b/tests/integration/testdata/buildcmd/samconfig/samconfig.yml @@ -0,0 +1,7 @@ +version: 0.1 +default: + build: + parameters: + build_dir: .yml + cached: true + parameter_overrides: Runtime=python3.9 CodeUri=SomeURI Handler=SomeHandler \ No newline at end of file diff --git a/tests/integration/testdata/buildcmd/samconfig/template.yaml b/tests/integration/testdata/buildcmd/samconfig/template.yaml new file mode 100644 index 0000000000..6944799912 --- /dev/null +++ b/tests/integration/testdata/buildcmd/samconfig/template.yaml @@ -0,0 +1,37 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Parameters: + Runtime: + Type: String + CodeUri: + Type: String + Handler: + Type: String + +Resources: + + Function: + Type: AWS::Serverless::Function + Properties: + Handler: !Ref Handler + Runtime: !Ref Runtime + CodeUri: !Ref CodeUri + Timeout: 600 + + + OtherRelativePathResource: + Type: AWS::ApiGateway::RestApi + Properties: + BodyS3Location: SomeRelativePath + + GlueResource: + Type: AWS::Glue::Job + Properties: + Command: + ScriptLocation: SomeRelativePath + + ExampleNestedStack: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: https://s3.amazonaws.com/examplebucket/exampletemplate.yml diff --git a/tests/unit/cli/test_cli_config_file.py b/tests/unit/cli/test_cli_config_file.py index 606e0a004e..19bfcfa011 100644 --- a/tests/unit/cli/test_cli_config_file.py +++ b/tests/unit/cli/test_cli_config_file.py @@ -5,8 +5,11 @@ from unittest import TestCase, skipIf from unittest.mock import MagicMock, patch +import tomlkit + from samcli.commands.exceptions import ConfigException -from samcli.cli.cli_config_file import TomlProvider, configuration_option, configuration_callback, get_ctx_defaults +from samcli.cli.cli_config_file import ConfigProvider, configuration_option, configuration_callback, get_ctx_defaults +from samcli.lib.config.exceptions import SamConfigFileReadException, SamConfigVersionException from samcli.lib.config.samconfig import DEFAULT_ENV from tests.testing_utils import IS_WINDOWS @@ -21,9 +24,9 @@ def __init__(self, info_name, parent, params=None, command=None, default_map=Non self.default_map = default_map -class TestTomlProvider(TestCase): +class TestConfigProvider(TestCase): def setUp(self): - self.toml_provider = TomlProvider() + self.config_provider = ConfigProvider() self.config_env = "config_env" self.parameters = "parameters" self.cmd_name = "topic" @@ -33,29 +36,30 @@ def test_toml_valid_with_section(self): config_path = Path(config_dir, "samconfig.toml") config_path.write_text("version=0.1\n[config_env.topic.parameters]\nword='clarity'\n") self.assertEqual( - TomlProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]), {"word": "clarity"} + ConfigProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]), {"word": "clarity"} ) def test_toml_valid_with_no_version(self): config_dir = tempfile.gettempdir() config_path = Path(config_dir, "samconfig.toml") config_path.write_text("[config_env.topic.parameters]\nword='clarity'\n") - with self.assertRaises(ConfigException): - TomlProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]) + with self.assertRaises(SamConfigVersionException): + ConfigProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]) def test_toml_valid_with_invalid_version(self): config_dir = tempfile.gettempdir() config_path = Path(config_dir, "samconfig.toml") config_path.write_text("version='abc'\n[config_env.topic.parameters]\nword='clarity'\n") - with self.assertRaises(ConfigException): - TomlProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]) + with self.assertRaises(SamConfigVersionException): + ConfigProvider(section=self.parameters)(config_path, self.config_env, [self.cmd_name]) def test_toml_invalid_empty_dict(self): config_dir = tempfile.gettempdir() config_path = Path(config_dir, "samconfig.toml") config_path.write_text("[topic]\nword=clarity\n") - self.assertEqual(self.toml_provider(config_dir, self.config_env, [self.cmd_name]), {}) + with self.assertRaises(SamConfigFileReadException): + self.config_provider(config_path, self.config_env, [self.cmd_name]) def test_toml_invalid_file_name(self): config_dir = tempfile.gettempdir() @@ -63,16 +67,16 @@ def test_toml_invalid_file_name(self): config_path.write_text("version=0.1\n[config_env.topic.parameters]\nword='clarity'\n") config_path_invalid = Path(config_dir, "samconfig.toml") - with self.assertRaises(ConfigException): - self.toml_provider(config_path_invalid, self.config_env, [self.cmd_name]) + with self.assertRaises(SamConfigFileReadException): + self.config_provider(config_path_invalid, self.config_env, [self.cmd_name]) def test_toml_invalid_syntax(self): config_dir = tempfile.gettempdir() config_path = Path(config_dir, "samconfig.toml") config_path.write_text("version=0.1\n[config_env.topic.parameters]\nword=_clarity'\n") - with self.assertRaises(ConfigException): - self.toml_provider(config_path, self.config_env, [self.cmd_name]) + with self.assertRaises(SamConfigFileReadException): + self.config_provider(config_path, self.config_env, [self.cmd_name]) class TestCliConfiguration(TestCase): @@ -121,6 +125,7 @@ def test_callback_with_invalid_config_file(self): self.ctx.parent = mock_context3 self.ctx.info_name = "test_info" self.ctx.params = {"config_file": "invalid_config_file"} + self.ctx._parameter_source.__get__ = "COMMANDLINE" setattr(self.ctx, "samconfig_dir", None) with self.assertRaises(ConfigException): configuration_callback( @@ -197,8 +202,8 @@ def test_callback_with_config_file_from_pipe(self): self.assertNotIn(self.value, self.saved_callback.call_args[0]) def test_configuration_option(self): - toml_provider = TomlProvider() - click_option = configuration_option(provider=toml_provider) + config_provider = ConfigProvider() + click_option = configuration_option(provider=config_provider) clc = click_option(self.Dummy()) self.assertEqual(clc.__click_params__[0].is_eager, True) self.assertEqual( @@ -207,7 +212,7 @@ def test_configuration_option(self): ) self.assertEqual(clc.__click_params__[0].hidden, True) self.assertEqual(clc.__click_params__[0].expose_value, False) - self.assertEqual(clc.__click_params__[0].callback.args, (None, None, None, toml_provider)) + self.assertEqual(clc.__click_params__[0].callback.args, (None, None, None, config_provider)) def test_get_ctx_defaults_non_nested(self): provider = MagicMock() diff --git a/tests/unit/commands/delete/test_delete_context.py b/tests/unit/commands/delete/test_delete_context.py index daa72c187b..92e2aa2c6a 100644 --- a/tests/unit/commands/delete/test_delete_context.py +++ b/tests/unit/commands/delete/test_delete_context.py @@ -7,7 +7,7 @@ from samcli.commands.delete.delete_context import DeleteContext from samcli.lib.package.artifact_exporter import Template -from samcli.cli.cli_config_file import TomlProvider +from samcli.cli.cli_config_file import ConfigProvider from samcli.lib.delete.cfn_utils import CfnUtils from samcli.lib.package.s3_uploader import S3Uploader from samcli.lib.package.ecr_uploader import ECRUploader @@ -58,7 +58,7 @@ def test_delete_context_enter(self, get_boto_client_provider_mock): self.assertEqual(delete_context.init_clients.call_count, 1) @patch.object( - TomlProvider, + ConfigProvider, "__call__", MagicMock( return_value=( @@ -123,7 +123,7 @@ def test_delete_no_user_input( self.assertEqual(expected_prompt_calls, patched_prompt.call_args_list) @patch.object( - TomlProvider, + ConfigProvider, "__call__", MagicMock( return_value=( @@ -506,7 +506,7 @@ def test_s3_option_flag(self): self.assertEqual(delete_context.s3_prefix, "s3_prefix") @patch.object( - TomlProvider, + ConfigProvider, "__call__", MagicMock( return_value=( diff --git a/tests/unit/commands/samconfig/test_samconfig.py b/tests/unit/commands/samconfig/test_samconfig.py index 675f22a4bf..b2e0822c78 100644 --- a/tests/unit/commands/samconfig/test_samconfig.py +++ b/tests/unit/commands/samconfig/test_samconfig.py @@ -8,7 +8,6 @@ import tempfile from pathlib import Path from contextlib import contextmanager -from samcli.lib.config.samconfig import SamConfig, DEFAULT_ENV from click.testing import CliRunner @@ -16,6 +15,7 @@ from unittest.mock import patch, ANY import logging +from samcli.lib.config.samconfig import SamConfig, DEFAULT_ENV from samcli.lib.utils.packagetype import ZIP, IMAGE LOG = logging.getLogger() diff --git a/tests/unit/lib/samconfig/test_file_manager.py b/tests/unit/lib/samconfig/test_file_manager.py new file mode 100644 index 0000000000..18df66474c --- /dev/null +++ b/tests/unit/lib/samconfig/test_file_manager.py @@ -0,0 +1,277 @@ +import json +from pathlib import Path +import tempfile +from unittest import TestCase, skip + +import tomlkit +from ruamel.yaml import YAML + +from samcli.lib.config.exceptions import FileParseException +from samcli.lib.config.file_manager import COMMENT_KEY, JsonFileManager, TomlFileManager, YamlFileManager + + +class TestTomlFileManager(TestCase): + def test_read_toml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + config_path.write_text( + "version=0.1\n[config_env.topic1.parameters]\nword='clarity'\nmultiword=['thing 1', 'thing 2']" + ) + config_doc = TomlFileManager.read(config_path) + self.assertEqual( + config_doc, + { + "version": 0.1, + "config_env": {"topic1": {"parameters": {"word": "clarity", "multiword": ["thing 1", "thing 2"]}}}, + }, + ) + + def test_read_toml_invalid_toml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + config_path.write_text("fake='not real'\nimproper toml file\n") + with self.assertRaises(FileParseException): + TomlFileManager.read(config_path) + + def test_read_toml_file_path_not_valid(self): + config_dir = "path/that/doesnt/exist" + config_path = Path(config_dir, "samconfig.toml") + config_doc = TomlFileManager.read(config_path) + self.assertEqual(config_doc, tomlkit.document()) + + def test_write_toml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + toml = { + "version": 0.1, + "config_env": {"topic2": {"parameters": {"word": "clarity"}}}, + COMMENT_KEY: "This is a comment", + } + + TomlFileManager.write(toml, config_path) + + txt = config_path.read_text() + self.assertIn("version = 0.1", txt) + self.assertIn("[config_env.topic2.parameters]", txt) + self.assertIn('word = "clarity"', txt) + self.assertIn("# This is a comment", txt) + self.assertNotIn(COMMENT_KEY, txt) + + def test_dont_write_toml_if_empty(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + config_path.write_text("nothing to see here\n") + toml = {} + + TomlFileManager.write(toml, config_path) + + self.assertEqual(config_path.read_text(), "nothing to see here\n") + + def test_write_toml_bad_path(self): + config_path = Path("path/to/some", "file_that_doesnt_exist.toml") + with self.assertRaises(FileNotFoundError): + TomlFileManager.write({"key": "some value"}, config_path) + + def test_write_toml_file(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + toml = tomlkit.parse('# This is a comment\nversion = 0.1\n[config_env.topic2.parameters]\nword = "clarity"\n') + + TomlFileManager.write(toml, config_path) + + txt = config_path.read_text() + self.assertIn("version = 0.1", txt) + self.assertIn("[config_env.topic2.parameters]", txt) + self.assertIn('word = "clarity"', txt) + self.assertIn("# This is a comment", txt) + + def test_dont_write_toml_file_if_empty(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.toml") + config_path.write_text("nothing to see here\n") + toml = tomlkit.document() + + TomlFileManager.write(toml, config_path) + + self.assertEqual(config_path.read_text(), "nothing to see here\n") + + def test_write_toml_file_bad_path(self): + config_path = Path("path/to/some", "file_that_doesnt_exist.toml") + with self.assertRaises(FileNotFoundError): + TomlFileManager.write(tomlkit.parse('key = "some value"'), config_path) + + def test_toml_put_comment(self): + toml_doc = tomlkit.loads('version = 0.1\n[config_env.topic2.parameters]\nword = "clarity"\n') + + toml_doc = TomlFileManager.put_comment(toml_doc, "This is a comment") + + txt = tomlkit.dumps(toml_doc) + self.assertIn("# This is a comment", txt) + + +class TestYamlFileManager(TestCase): + + yaml = YAML() + + def test_read_yaml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.yaml") + config_path.write_text( + "version: 0.1\nconfig_env:\n topic1:\n parameters:\n word: clarity\n multiword: [thing 1, thing 2]" + ) + + config_doc = YamlFileManager.read(config_path) + + self.assertEqual( + config_doc, + { + "version": 0.1, + "config_env": {"topic1": {"parameters": {"word": "clarity", "multiword": ["thing 1", "thing 2"]}}}, + }, + ) + + def test_read_yaml_invalid_yaml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.yaml") + config_path.write_text("fake: not real\nthisYaml isn't correct") + + with self.assertRaises(FileParseException): + YamlFileManager.read(config_path) + + def test_read_yaml_file_path_not_valid(self): + config_dir = "path/that/doesnt/exist" + config_path = Path(config_dir, "samconfig.yaml") + + config_doc = YamlFileManager.read(config_path) + + self.assertEqual(config_doc, {}) + + def test_write_yaml(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.yaml") + yaml = { + "version": 0.1, + "config_env": {"topic2": {"parameters": {"word": "clarity"}}}, + COMMENT_KEY: "This is a comment", + } + + YamlFileManager.write(yaml, config_path) + + txt = config_path.read_text() + self.assertIn("version: 0.1", txt) + self.assertIn("config_env:\n topic2:\n parameters:\n", txt) + self.assertIn("word: clarity", txt) + self.assertIn("# This is a comment", txt) + self.assertNotIn(COMMENT_KEY, txt) + + def test_dont_write_yaml_if_empty(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.yaml") + config_path.write_text("nothing to see here\n") + yaml = {} + + YamlFileManager.write(yaml, config_path) + + self.assertEqual(config_path.read_text(), "nothing to see here\n") + + def test_write_yaml_file_bad_path(self): + config_path = Path("path/to/some", "file_that_doesnt_exist.yaml") + + with self.assertRaises(FileNotFoundError): + YamlFileManager.write(self.yaml.load("key: some value"), config_path) + + def test_yaml_put_comment(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.yaml") + yaml_doc = self.yaml.load("version: 0.1\nconfig_env:\n topic2:\n parameters:\n word: clarity\n") + + yaml_doc = YamlFileManager.put_comment(yaml_doc, "This is a comment") + + self.yaml.dump(yaml_doc, config_path) + txt = config_path.read_text() + self.assertIn("# This is a comment", txt) + + +@skip("JSON config support disabled") +class TestJsonFileManager(TestCase): + def test_read_json(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.json") + config_path.write_text( + json.dumps( + { + "version": 0.1, + "config_env": {"topic1": {"parameters": {"word": "clarity", "multiword": ["thing 1", "thing 2"]}}}, + }, + indent=JsonFileManager.INDENT_SIZE, + ) + ) + + config_doc = JsonFileManager.read(config_path) + + self.assertEqual( + config_doc, + { + "version": 0.1, + "config_env": {"topic1": {"parameters": {"word": "clarity", "multiword": ["thing 1", "thing 2"]}}}, + }, + ) + + def test_read_json_invalid_json(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.json") + config_path.write_text("{\n" + ' "bad_file": "very bad"\n' + ' "improperly": "formatted"\n' + "}\n") + + with self.assertRaises(FileParseException): + JsonFileManager.read(config_path) + + def test_read_json_file_path_not_valid(self): + config_dir = "path/that/doesnt/exist" + config_path = Path(config_dir, "samconfig.json") + + config_doc = JsonFileManager.read(config_path) + + self.assertEqual(config_doc, {}) + + def test_write_json(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.json") + json_doc = { + "version": 0.1, + "config_env": {"topic2": {"parameters": {"word": "clarity"}}}, + COMMENT_KEY: "This is a comment", + } + + JsonFileManager.write(json_doc, config_path) + + txt = config_path.read_text() + self.assertIn('"version": 0.1', txt) + self.assertIn('"config_env": {', txt) + self.assertIn('"topic2": {', txt) + self.assertIn('"parameters": {', txt) + self.assertIn('"word": "clarity"', txt) + self.assertIn(f'"{COMMENT_KEY}": "This is a comment"', txt) + + def test_dont_write_json_if_empty(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.json") + config_path.write_text("nothing to see here\n") + json_doc = {} + + JsonFileManager.write(json_doc, config_path) + + self.assertEqual(config_path.read_text(), "nothing to see here\n") + + def test_write_json_file_bad_path(self): + config_path = Path("path/to/some", "file_that_doesnt_exist.json") + + with self.assertRaises(FileNotFoundError): + JsonFileManager.write({"key": "value"}, config_path) + + def test_json_put_comment(self): + json_doc = {"version": 0.1, "config_env": {"topic1": {"parameters": {"word": "clarity"}}}} + + json_doc = JsonFileManager.put_comment(json_doc, "This is a comment") + + txt = json.dumps(json_doc) + self.assertIn(f'"{COMMENT_KEY}": "This is a comment"', txt) diff --git a/tests/unit/lib/samconfig/test_samconfig.py b/tests/unit/lib/samconfig/test_samconfig.py index 7a86e6f97d..c58f0709a6 100644 --- a/tests/unit/lib/samconfig/test_samconfig.py +++ b/tests/unit/lib/samconfig/test_samconfig.py @@ -1,10 +1,21 @@ import os from pathlib import Path +from unittest.mock import patch +from parameterized import parameterized +import tempfile from unittest import TestCase -from samcli.lib.config.exceptions import SamConfigVersionException -from samcli.lib.config.samconfig import SamConfig, DEFAULT_CONFIG_FILE_NAME, DEFAULT_GLOBAL_CMDNAME, DEFAULT_ENV +from samcli.lib.config.exceptions import SamConfigFileReadException, SamConfigVersionException +from samcli.lib.config.file_manager import FILE_MANAGER_MAPPER, JsonFileManager, TomlFileManager, YamlFileManager +from samcli.lib.config.samconfig import ( + DEFAULT_CONFIG_FILE, + SamConfig, + DEFAULT_CONFIG_FILE_NAME, + DEFAULT_GLOBAL_CMDNAME, + DEFAULT_ENV, +) from samcli.lib.config.version import VERSION_KEY, SAM_CONFIG_VERSION +from samcli.lib.telemetry.event import Event from samcli.lib.utils import osutils @@ -182,27 +193,27 @@ def test_check_sanity(self): def test_check_version_non_supported_type(self): self._setup_config() - self.samconfig.document.remove(VERSION_KEY) - self.samconfig.document.add(VERSION_KEY, "aadeff") + self.samconfig.document.pop(VERSION_KEY) + self.samconfig.document.update({VERSION_KEY: "aadeff"}) with self.assertRaises(SamConfigVersionException): self.samconfig.sanity_check() def test_check_version_no_version_exists(self): self._setup_config() - self.samconfig.document.remove(VERSION_KEY) + self.samconfig.document.pop(VERSION_KEY) with self.assertRaises(SamConfigVersionException): self.samconfig.sanity_check() def test_check_version_float(self): self._setup_config() - self.samconfig.document.remove(VERSION_KEY) - self.samconfig.document.add(VERSION_KEY, 0.2) + self.samconfig.document.pop(VERSION_KEY) + self.samconfig.document.update({VERSION_KEY: 0.2}) self.samconfig.sanity_check() def test_write_config_file_non_standard_version(self): self._setup_config() - self.samconfig.document.remove(VERSION_KEY) - self.samconfig.document.add(VERSION_KEY, 0.2) + self.samconfig.document.pop(VERSION_KEY) + self.samconfig.document.update({VERSION_KEY: 0.2}) self.samconfig.put(cmd_names=["local", "start", "api"], section="parameters", key="skip_pull_image", value=True) self.samconfig.sanity_check() self.assertEqual(self.samconfig.document.get(VERSION_KEY), 0.2) @@ -210,7 +221,7 @@ def test_write_config_file_non_standard_version(self): def test_write_config_file_will_create_the_file_if_not_exist(self): with osutils.mkdir_temp(ignore_errors=True) as tempdir: non_existing_dir = os.path.join(tempdir, "non-existing-dir") - non_existing_file = "non-existing-file" + non_existing_file = "non-existing-file.toml" samconfig = SamConfig(config_dir=non_existing_dir, filename=non_existing_file) self.assertFalse(samconfig.exists()) @@ -221,3 +232,84 @@ def test_write_config_file_will_create_the_file_if_not_exist(self): samconfig.put(cmd_names=["any", "command"], section="any-section", key="any-key", value="any-value") samconfig.flush() self.assertTrue(samconfig.exists()) + + def test_passed_filename_used(self): + config_path = Path(self.config_dir, "myconfigfile.toml") + + self.assertFalse(config_path.exists()) + + self.samconfig = SamConfig(self.config_dir, filename="myconfigfile.toml") + self.samconfig.put( # put some config options so it creates the file + cmd_names=["any", "command"], section="section", key="key", value="value" + ) + self.samconfig.flush() + + self.assertTrue(config_path.exists()) + self.assertFalse(Path(self.config_dir, DEFAULT_CONFIG_FILE_NAME).exists()) + + def test_config_uses_default_if_none_provided(self): + self.samconfig = SamConfig(self.config_dir) + self.samconfig.put( # put some config options so it creates the file + cmd_names=["any", "command"], section="section", key="key", value="value" + ) + self.samconfig.flush() + + self.assertTrue(Path(self.config_dir, DEFAULT_CONFIG_FILE_NAME).exists()) + + def test_config_priority(self): + config_files = [] + extensions_in_priority = list(FILE_MANAGER_MAPPER.keys()) # priority by order in dict + for extension in extensions_in_priority: + filename = DEFAULT_CONFIG_FILE + extension + config = SamConfig(self.config_dir, filename=filename) + config.put( # put some config options so it creates the file + cmd_names=["any", "command"], section="section", key="key", value="value" + ) + config.flush() + config_files.append(config) + + while extensions_in_priority: + config = SamConfig(self.config_dir) + next_priority = extensions_in_priority.pop(0) + self.assertEqual(config.filepath, Path(self.config_dir, DEFAULT_CONFIG_FILE + next_priority)) + os.remove(config.path()) + + +class TestSamConfigFileManager(TestCase): + def test_file_manager_not_declared(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig") + + with self.assertRaises(SamConfigFileReadException): + SamConfig(config_path, filename="samconfig") + + def test_file_manager_unsupported(self): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, "samconfig.jpeg") + + with self.assertRaises(SamConfigFileReadException): + SamConfig(config_path, filename="samconfig.jpeg") + + @parameterized.expand( + [ + ("samconfig.toml", TomlFileManager, ".toml"), + ("samconfig.yaml", YamlFileManager, ".yaml"), + ("samconfig.yml", YamlFileManager, ".yml"), + # ("samconfig.json", JsonFileManager, ".json"), + ] + ) + @patch("samcli.lib.telemetry.event.EventTracker.track_event") + def test_file_manager(self, filename, expected_file_manager, expected_extension, track_mock): + config_dir = tempfile.gettempdir() + config_path = Path(config_dir, filename) + tracked_events = [] + + def mock_tracker(name, value): # when track_event is called, just append the Event to our list + tracked_events.append(Event(name, value)) + + track_mock.side_effect = mock_tracker + + samconfig = SamConfig(config_path, filename=filename) + + self.assertIs(samconfig.file_manager, expected_file_manager) + self.assertIn(Event("SamConfigFileExtension", expected_extension), tracked_events)