From 4232a83629b0f6fd5755e72f66543261a585f65c Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Tue, 4 Jul 2023 16:23:28 -0700 Subject: [PATCH 1/8] Retrieve SAM CLI command parameters --- schema/schema.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 schema/schema.py diff --git a/schema/schema.py b/schema/schema.py new file mode 100644 index 0000000000..41adc14325 --- /dev/null +++ b/schema/schema.py @@ -0,0 +1,84 @@ +"""Handles JSON schema generation logic""" + + +import click +import importlib + +from samcli.cli.command import _SAM_CLI_COMMAND_PACKAGES +from samcli.commands._utils.click_mutex import ClickMutex +from samcli.commands.build.click_container import ContainerOptions +from samcli.lib.config.samconfig import SamConfig + + +def retrieve_command_structure(package_name: str) -> dict: + """Given a SAM CLI package name, retrieve its structure. + + Returns + ------- + dict + A dictionary that maps the name of the command to its relevant click options. + """ + module = importlib.import_module(package_name) + command_name = package_name.split(".")[-1] # command name is the last folder + command = {} + + def format_param(param: click.core.Option): + """Format a click Option parameter to a dictionary object.""" + formatted_param = {"name": param.name, "help": param.help} + + if param.type.name in ["text", "path", "choice"]: + formatted_param["type"] = "string" + else: + formatted_param["type"] = param.type.name + + if param.default: + formatted_param["default"] = param.default + + if param.type.name == "choice": + formatted_param["choices"] = param.type.choices + + return formatted_param + + if isinstance(module.cli, click.core.Group): # command has subcommands (e.g. local invoke) + for subcommand in module.cli.commands.values(): + params = [ + param + for param in subcommand.params + if param.name != None and isinstance(param, click.core.Option) # exclude None and non-Options + ] + formatted_params = {param.name: format_param(param) for param in params} + command.update({SamConfig._to_key([command_name, subcommand.name]): formatted_params}) + else: + params = [ + param + for param in module.cli.params + if param.name != None and isinstance(param, click.core.Option) # exclude None and non-Options + ] + formatted_params = {param.name: format_param(param) for param in params} + command.update({command_name: formatted_params}) + return command + + +def generate_schema() -> dict: + """Generate a JSON schema for all SAM CLI commands. + + Returns + ------- + dict + A dictionary representation of the JSON schema. + """ + schema = {} + commands = {} + params = set() # NOTE(leogama): Currently unused due to some params having different help values + # TODO: Populate schema with relevant attributes + for package_name in _SAM_CLI_COMMAND_PACKAGES: + new_command = retrieve_command_structure(package_name) + commands.update(new_command) + command_params = [param for param_list in new_command.values() for param in param_list] + params.update(command_params) + print(commands) # DEBUG: note that some params appear + return schema + + +if __name__ == "__main__": + generate_schema() From 1a6ebd11e9f1ab346acfafef2386f789af49a6bc Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Wed, 5 Jul 2023 11:21:50 -0700 Subject: [PATCH 2/8] Implement requested changes --- samcli/lib/config/samconfig.py | 6 ++-- schema/schema.py | 64 +++++++++++++++++----------------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/samcli/lib/config/samconfig.py b/samcli/lib/config/samconfig.py index e48e53d625..2c3ba303a6 100644 --- a/samcli/lib/config/samconfig.py +++ b/samcli/lib/config/samconfig.py @@ -79,7 +79,7 @@ def get_all(self, cmd_names, section, env=DEFAULT_ENV): self._read() if isinstance(self.document, dict): toml_content = self.document.get(env, {}) - params = toml_content.get(self._to_key(cmd_names), {}).get(section, {}) + params = toml_content.get(self.to_key(cmd_names), {}).get(section, {}) if DEFAULT_GLOBAL_CMDNAME in toml_content: global_params = toml_content.get(DEFAULT_GLOBAL_CMDNAME, {}).get(section, {}) global_params.update(params.copy()) @@ -119,7 +119,7 @@ def put(self, cmd_names, section, key, value, env=DEFAULT_ENV): # Empty document prepare the initial structure. # self.document is a nested dict, we need to check each layer and add new tables, otherwise duplicated key # in parent layer will override the whole child layer - cmd_name_key = self._to_key(cmd_names) + cmd_name_key = self.to_key(cmd_names) env_content = self.document.get(env, {}) cmd_content = env_content.get(cmd_name_key, {}) param_content = cmd_content.get(section, {}) @@ -267,6 +267,6 @@ def _version_sanity_check(version: Any) -> None: raise SamConfigVersionException(f"'{VERSION_KEY}' key is not present or is in unrecognized format. ") @staticmethod - def _to_key(cmd_names: Iterable[str]) -> str: + def to_key(cmd_names: Iterable[str]) -> str: # construct a parsed name that is of the format: a_b_c_d return "_".join([cmd.replace("-", "_").replace(" ", "_") for cmd in cmd_names]) diff --git a/schema/schema.py b/schema/schema.py index 41adc14325..2af3d315b1 100644 --- a/schema/schema.py +++ b/schema/schema.py @@ -10,6 +10,24 @@ from samcli.lib.config.samconfig import SamConfig +def format_param(param: click.core.Option): + """Format a click Option parameter to a dictionary object.""" + formatted_param = {"name": param.name, "help": param.help} + + if param.type.name in ["text", "path", "choice"]: + formatted_param["type"] = "string" + else: + formatted_param["type"] = param.type.name + + if param.default: + formatted_param["default"] = param.default + + if param.type.name == "choice": + formatted_param["choices"] = param.type.choices + + return formatted_param + + def retrieve_command_structure(package_name: str) -> dict: """Given a SAM CLI package name, retrieve its structure. @@ -22,40 +40,21 @@ def retrieve_command_structure(package_name: str) -> dict: command_name = package_name.split(".")[-1] # command name is the last folder command = {} - def format_param(param: click.core.Option): - """Format a click Option parameter to a dictionary object.""" - formatted_param = {"name": param.name, "help": param.help} - - if param.type.name in ["text", "path", "choice"]: - formatted_param["type"] = "string" - else: - formatted_param["type"] = param.type.name - - if param.default: - formatted_param["default"] = param.default - - if param.type.name == "choice": - formatted_param["choices"] = param.type.choices - - return formatted_param - - if isinstance(module.cli, click.core.Group): # command has subcommands (e.g. local invoke) - for subcommand in module.cli.commands.values(): - params = [ - param - for param in subcommand.params - if param.name != None and isinstance(param, click.core.Option) # exclude None and non-Options - ] - formatted_params = {param.name: format_param(param) for param in params} - command.update({SamConfig._to_key([command_name, subcommand.name]): formatted_params}) - else: + def get_params_from_command(cli, main_command_name: str = "") -> dict: + """Given a command CLI, return its parameters.""" params = [ param - for param in module.cli.params + for param in cli.params if param.name != None and isinstance(param, click.core.Option) # exclude None and non-Options ] - formatted_params = {param.name: format_param(param) for param in params} - command.update({command_name: formatted_params}) + cmd_name = SamConfig.to_key([main_command_name, cli.name]) if main_command_name else cli.name + return {cmd_name: {param.name: format_param(param) for param in params}} + + if isinstance(module.cli, click.core.Group): # command has subcommands (e.g. local invoke) + for subcommand in module.cli.commands.values(): + command.update(get_params_from_command(subcommand)) + else: + command.update(get_params_from_command(module.cli)) return command @@ -74,9 +73,10 @@ def generate_schema() -> dict: for package_name in _SAM_CLI_COMMAND_PACKAGES: new_command = retrieve_command_structure(package_name) commands.update(new_command) - command_params = [param for param_list in new_command.values() for param in param_list] + for param_list in new_command.values(): + command_params = [param for param in param_list] params.update(command_params) - print(commands) # DEBUG: note that some params appear + print(commands) # DEBUG: note that some params appear multiple times due to slight differences return schema From a6c1e15277570f932df7ec8b269c70e806c207bc Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Wed, 5 Jul 2023 14:39:35 -0700 Subject: [PATCH 3/8] Formatting --- schema/schema.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/schema/schema.py b/schema/schema.py index 2af3d315b1..d4a988ec3e 100644 --- a/schema/schema.py +++ b/schema/schema.py @@ -1,12 +1,11 @@ """Handles JSON schema generation logic""" -import click import importlib +import click + from samcli.cli.command import _SAM_CLI_COMMAND_PACKAGES -from samcli.commands._utils.click_mutex import ClickMutex -from samcli.commands.build.click_container import ContainerOptions from samcli.lib.config.samconfig import SamConfig @@ -37,7 +36,6 @@ def retrieve_command_structure(package_name: str) -> dict: A dictionary that maps the name of the command to its relevant click options. """ module = importlib.import_module(package_name) - command_name = package_name.split(".")[-1] # command name is the last folder command = {} def get_params_from_command(cli, main_command_name: str = "") -> dict: @@ -45,7 +43,7 @@ def get_params_from_command(cli, main_command_name: str = "") -> dict: params = [ param for param in cli.params - if param.name != None and isinstance(param, click.core.Option) # exclude None and non-Options + if param.name is not None and isinstance(param, click.core.Option) # exclude None and non-Options ] cmd_name = SamConfig.to_key([main_command_name, cli.name]) if main_command_name else cli.name return {cmd_name: {param.name: format_param(param) for param in params}} From 7a6841edb241617f6bbf36b9cdb932b0a23b010c Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Thu, 6 Jul 2023 11:17:28 -0700 Subject: [PATCH 4/8] Implement requested changes --- schema/schema.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/schema/schema.py b/schema/schema.py index d4a988ec3e..1777693e3f 100644 --- a/schema/schema.py +++ b/schema/schema.py @@ -43,14 +43,14 @@ def get_params_from_command(cli, main_command_name: str = "") -> dict: params = [ param for param in cli.params - if param.name is not None and isinstance(param, click.core.Option) # exclude None and non-Options + if param.name and isinstance(param, click.core.Option) # exclude None and non-Options ] cmd_name = SamConfig.to_key([main_command_name, cli.name]) if main_command_name else cli.name return {cmd_name: {param.name: format_param(param) for param in params}} if isinstance(module.cli, click.core.Group): # command has subcommands (e.g. local invoke) for subcommand in module.cli.commands.values(): - command.update(get_params_from_command(subcommand)) + command.update(get_params_from_command(subcommand, module.__name__.split(".")[-1])) else: command.update(get_params_from_command(module.cli)) return command @@ -74,7 +74,6 @@ def generate_schema() -> dict: for param_list in new_command.values(): command_params = [param for param in param_list] params.update(command_params) - print(commands) # DEBUG: note that some params appear multiple times due to slight differences return schema From 6f3dbceefea5b759c9b092dda03221fd4f78572a Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Thu, 6 Jul 2023 13:33:46 -0700 Subject: [PATCH 5/8] Include comment about module name parsing --- schema/schema.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/schema/schema.py b/schema/schema.py index 1777693e3f..37b201a873 100644 --- a/schema/schema.py +++ b/schema/schema.py @@ -50,7 +50,12 @@ def get_params_from_command(cli, main_command_name: str = "") -> dict: if isinstance(module.cli, click.core.Group): # command has subcommands (e.g. local invoke) for subcommand in module.cli.commands.values(): - command.update(get_params_from_command(subcommand, module.__name__.split(".")[-1])) + command.update( + get_params_from_command( + subcommand, + module.__name__.split(".")[-1] # if Group CLI, get last section of module name for cmd name + ) + ) else: command.update(get_params_from_command(module.cli)) return command From 2415566615b6e878278e4bd42a6f34fcabe6934d Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Thu, 6 Jul 2023 14:01:03 -0700 Subject: [PATCH 6/8] Address comments and update makefile --- Makefile | 4 ++-- schema/schema.py | 47 ++++++++++++++++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 8876d482b2..497f598eef 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ smoke-test: lint: # Linter performs static analysis to catch latent bugs - ruff samcli + ruff samcli schema # mypy performs type check mypy --exclude /testdata/ --exclude /init/templates/ --no-incremental setup.py samcli tests @@ -42,7 +42,7 @@ lint: dev: lint test black: - black setup.py samcli tests + black setup.py samcli tests schema black-check: black --check setup.py samcli tests diff --git a/schema/schema.py b/schema/schema.py index 37b201a873..5a5e9237f8 100644 --- a/schema/schema.py +++ b/schema/schema.py @@ -2,6 +2,7 @@ import importlib +from typing import Dict, List, Union import click @@ -9,8 +10,18 @@ from samcli.lib.config.samconfig import SamConfig -def format_param(param: click.core.Option): - """Format a click Option parameter to a dictionary object.""" +def format_param(param: click.core.Option) -> Dict[str, Union[str, List[str]]]: + """Format a click Option parameter to a dictionary object. + + A parameter object should contain the following information that will be + necessary for including in the JSON schema: + * name - The name of the parameter + * help - The parameter's description (may vary between commands) + * type - The data type accepted by the parameter + * type.choices - If there are only a certain number of options allowed, + a list of those allowed options + * default - The default option for that parameter + """ formatted_param = {"name": param.name, "help": param.help} if param.type.name in ["text", "path", "choice"]: @@ -27,9 +38,25 @@ def format_param(param: click.core.Option): return formatted_param -def retrieve_command_structure(package_name: str) -> dict: +def get_params_from_command(cli, main_command_name: str = "") -> Dict[str, dict]: + """Given a command CLI, return it in a dictionary, pointing to its parameters as dictionary objects.""" + params = [ + param + for param in cli.params + if param.name and isinstance(param, click.core.Option) # exclude None and non-Options + ] + cmd_name = SamConfig.to_key([main_command_name, cli.name]) if main_command_name else cli.name + return {cmd_name: {param.name: format_param(param) for param in params}} + + +def retrieve_command_structure(package_name: str) -> Dict[str, dict]: """Given a SAM CLI package name, retrieve its structure. + Parameters + ---------- + package_name: str + The name of the command package to retrieve. + Returns ------- dict @@ -38,22 +65,12 @@ def retrieve_command_structure(package_name: str) -> dict: module = importlib.import_module(package_name) command = {} - def get_params_from_command(cli, main_command_name: str = "") -> dict: - """Given a command CLI, return its parameters.""" - params = [ - param - for param in cli.params - if param.name and isinstance(param, click.core.Option) # exclude None and non-Options - ] - cmd_name = SamConfig.to_key([main_command_name, cli.name]) if main_command_name else cli.name - return {cmd_name: {param.name: format_param(param) for param in params}} - if isinstance(module.cli, click.core.Group): # command has subcommands (e.g. local invoke) for subcommand in module.cli.commands.values(): command.update( get_params_from_command( - subcommand, - module.__name__.split(".")[-1] # if Group CLI, get last section of module name for cmd name + subcommand, + module.__name__.split(".")[-1], # if Group CLI, get last section of module name for cmd name ) ) else: From 0534d703d65f1ce0d84aef1e7914c5989824b18d Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Thu, 6 Jul 2023 14:30:16 -0700 Subject: [PATCH 7/8] Address final comments --- Makefile | 4 ++-- schema/schema.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 497f598eef..03443f7868 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ lint: # Linter performs static analysis to catch latent bugs ruff samcli schema # mypy performs type check - mypy --exclude /testdata/ --exclude /init/templates/ --no-incremental setup.py samcli tests + mypy --exclude /testdata/ --exclude /init/templates/ --no-incremental setup.py samcli tests schema # Command to run everytime you make changes to verify everything works dev: lint test @@ -45,7 +45,7 @@ black: black setup.py samcli tests schema black-check: - black --check setup.py samcli tests + black --check setup.py samcli tests schema format: black ruff samcli --fix diff --git a/schema/schema.py b/schema/schema.py index 5a5e9237f8..e895cc7985 100644 --- a/schema/schema.py +++ b/schema/schema.py @@ -24,6 +24,8 @@ def format_param(param: click.core.Option) -> Dict[str, Union[str, List[str]]]: """ formatted_param = {"name": param.name, "help": param.help} + # NOTE: Params do not have explicit "string" type; either "text" or "path". + # All choice options are from a set of strings. if param.type.name in ["text", "path", "choice"]: formatted_param["type"] = "string" else: From 05fdd5644e980cc596d79b373bc9498b62c1344b Mon Sep 17 00:00:00 2001 From: Leonardo Gama Date: Fri, 7 Jul 2023 09:50:46 -0700 Subject: [PATCH 8/8] Fix linting issues --- schema/schema.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/schema/schema.py b/schema/schema.py index e895cc7985..a2925dc2f7 100644 --- a/schema/schema.py +++ b/schema/schema.py @@ -2,7 +2,7 @@ import importlib -from typing import Dict, List, Union +from typing import Dict, List, Optional, Union import click @@ -10,7 +10,7 @@ from samcli.lib.config.samconfig import SamConfig -def format_param(param: click.core.Option) -> Dict[str, Union[str, List[str]]]: +def format_param(param: click.core.Option) -> Dict[str, Union[Optional[str], List[str]]]: """Format a click Option parameter to a dictionary object. A parameter object should contain the following information that will be @@ -22,7 +22,7 @@ def format_param(param: click.core.Option) -> Dict[str, Union[str, List[str]]]: a list of those allowed options * default - The default option for that parameter """ - formatted_param = {"name": param.name, "help": param.help} + formatted_param: Dict[str, Union[Optional[str], List[str]]] = {"name": param.name, "help": param.help} # NOTE: Params do not have explicit "string" type; either "text" or "path". # All choice options are from a set of strings. @@ -32,10 +32,10 @@ def format_param(param: click.core.Option) -> Dict[str, Union[str, List[str]]]: formatted_param["type"] = param.type.name if param.default: - formatted_param["default"] = param.default + formatted_param["default"] = str(param.default) - if param.type.name == "choice": - formatted_param["choices"] = param.type.choices + if param.type.name == "choice" and isinstance(param.type, click.Choice): + formatted_param["choices"] = list(param.type.choices) return formatted_param @@ -88,7 +88,7 @@ def generate_schema() -> dict: dict A dictionary representation of the JSON schema. """ - schema = {} + schema: dict = {} commands = {} params = set() # NOTE(leogama): Currently unused due to some params having different help values # TODO: Populate schema with relevant attributes