From 23c1262eb4c8d7ef4db6a28ff43c1e1c3a34e28e Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Mon, 26 Apr 2021 21:57:43 -0700 Subject: [PATCH 1/5] sam pipeline init command --- samcli/commands/exceptions.py | 13 + samcli/commands/pipeline/init/__init__.py | 0 samcli/commands/pipeline/init/cli.py | 43 +++ .../pipeline/init/interactive_init_flow.py | 244 ++++++++++++++++++ .../init/pipeline_templates_manifest.py | 50 ++++ samcli/commands/pipeline/pipeline.py | 4 +- .../cookiecutter/interactive_flow_creator.py | 4 +- samcli/yamlhelper.py | 10 +- tests/unit/commands/pipeline/init/__init__.py | 0 tests/unit/commands/pipeline/init/test_cli.py | 22 ++ .../init/test_initeractive_init_flow.py | 235 +++++++++++++++++ .../init/test_pipeline_templates_manifest.py | 76 ++++++ 12 files changed, 693 insertions(+), 8 deletions(-) create mode 100644 samcli/commands/pipeline/init/__init__.py create mode 100644 samcli/commands/pipeline/init/cli.py create mode 100644 samcli/commands/pipeline/init/interactive_init_flow.py create mode 100644 samcli/commands/pipeline/init/pipeline_templates_manifest.py create mode 100644 tests/unit/commands/pipeline/init/__init__.py create mode 100644 tests/unit/commands/pipeline/init/test_cli.py create mode 100644 tests/unit/commands/pipeline/init/test_initeractive_init_flow.py create mode 100644 tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py diff --git a/samcli/commands/exceptions.py b/samcli/commands/exceptions.py index 7b8f253609..d560dd9fad 100644 --- a/samcli/commands/exceptions.py +++ b/samcli/commands/exceptions.py @@ -59,3 +59,16 @@ class ContainersInitializationException(UserException): """ Exception class when SAM is not able to initialize any of the lambda functions containers """ + + +class PipelineTemplateCloneException(UserException): + """ + Exception class when unable to download pipeline templates from a Git repository during `sam pipeline init` + """ + + +class AppPipelineTemplateManifestException(UserException): + """ + Exception class when SAM is not able to parse the "manifest.yaml" file located in the SAM pipeline templates + Git repo: "github.com/aws/aws-sam-cli-pipeline-init-templates.git + """ diff --git a/samcli/commands/pipeline/init/__init__.py b/samcli/commands/pipeline/init/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samcli/commands/pipeline/init/cli.py b/samcli/commands/pipeline/init/cli.py new file mode 100644 index 0000000000..a52afef56b --- /dev/null +++ b/samcli/commands/pipeline/init/cli.py @@ -0,0 +1,43 @@ +""" +CLI command for "pipeline init" command +""" +from typing import Any, Optional + +import click + +from samcli.cli.cli_config_file import configuration_option, TomlProvider +from samcli.cli.main import pass_context, common_options as cli_framework_options +from samcli.commands.pipeline.init.interactive_init_flow import do_interactive +from samcli.lib.telemetry.metric import track_command + +SHORT_HELP = "Generates a CI/CD pipeline configuration file." +HELP_TEXT = """ +Generates a CI/CD pipeline configuration file for a chosen CI/CD provider like Jenkins, Gitlab and Github Actions +""" + + +@click.command("init", help=HELP_TEXT, short_help=SHORT_HELP) +@configuration_option(provider=TomlProvider(section="parameters")) +@cli_framework_options +@pass_context +@track_command # pylint: disable=R0914 +def cli( + ctx: Any, + config_env: Optional[str], + config_file: Optional[str], +) -> None: + """ + `sam pipeline init` command entry point + """ + + # Currently we support interactive mode only, i.e. the user doesn't provide the required arguments during the call + # so we call do_cli without any arguments. This will change after supporting the non interactive mode. + do_cli() + + +def do_cli() -> None: + """ + implementation of `sam pipeline init` command + """ + # TODO non-interactive mode + do_interactive() diff --git a/samcli/commands/pipeline/init/interactive_init_flow.py b/samcli/commands/pipeline/init/interactive_init_flow.py new file mode 100644 index 0000000000..a5e72284ed --- /dev/null +++ b/samcli/commands/pipeline/init/interactive_init_flow.py @@ -0,0 +1,244 @@ +""" +Interactive flow that prompts that users for pipeline template (cookiecutter template) and used it to generate +pipeline configuration file +""" +import logging +import os +import shutil +from pathlib import Path +from typing import Dict, List + +import click + +from samcli.cli.main import global_cfg +from samcli.commands.exceptions import PipelineTemplateCloneException +from samcli.lib.cookiecutter.interactive_flow import InteractiveFlow +from samcli.lib.cookiecutter.interactive_flow_creator import InteractiveFlowCreator +from samcli.lib.cookiecutter.template import Template +from samcli.lib.utils.git_repo import GitRepo, CloneRepoException +from samcli.lib.utils.osutils import rmtree_callback +from .pipeline_templates_manifest import PipelineTemplateManifest, PipelineTemplatesManifest + +LOG = logging.getLogger(__name__) +shared_path: Path = global_cfg.config_dir +APP_PIPELINE_TEMPLATES_REPO_URL = "https://github.com/aws/aws-sam-cli-pipeline-init-templates.git" +APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME = "aws-sam-cli-app-pipeline-templates" +CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME = "custom-pipeline-template" + + +def do_interactive() -> None: + """ + An interactive flow that prompts the user for pipeline template (cookiecutter template) location, downloads it, + runs its specific questionnaire then generates the pipeline config file based on the template and user's responses + """ + click.echo("Which pipeline template source would you like to use?") + click.echo("\t1 - AWS Quick Start Pipeline Templates\n\t2 - Custom Pipeline Template Location") + location_choice = click.prompt("Choice", type=click.Choice(["1", "2"]), show_choices=False) + if location_choice == "2": + _generate_from_custom_location() + else: + _generate_from_app_pipeline_templates() + + +def _generate_from_app_pipeline_templates() -> None: + """ + Prompts the user to choose a pipeline template from SAM predefined set of pipeline templates hosted in the git + repository: aws/aws-sam-cli-pipeline-init-templates.git + downloads locally, then generates the pipeline config file from the selected pipeline template. + """ + pipeline_templates_local_dir: Path = _clone_app_pipeline_templates() + + pipeline_templates_manifest: PipelineTemplatesManifest = _read_app_pipeline_templates_manifest( + pipeline_templates_dir=pipeline_templates_local_dir + ) + # The manifest contains multiple pipeline-templates so select one + selected_pipeline_template_manifest: PipelineTemplateManifest = _select_pipeline_template( + pipeline_templates_manifest=pipeline_templates_manifest + ) + selected_pipeline_template_dir: Path = pipeline_templates_local_dir.joinpath( + selected_pipeline_template_manifest.location + ) + _generate_from_pipeline_template(pipeline_template_dir=selected_pipeline_template_dir) + + +def _generate_from_custom_location() -> None: + """ + Prompts the user for a custom pipeline template location, downloads locally, then generates the pipeline config file + """ + pipeline_template_repo_url: str = click.prompt(text="template Git location") + pipeline_template_local_dir: Path = _clone_custom_pipeline_template(repo_url=pipeline_template_repo_url) + _generate_from_pipeline_template(pipeline_template_dir=pipeline_template_local_dir) + # Unlike app pipeline templates, custom pipeline templates are not shared between different SAM applications + # and should be cleaned up from users' machines after generating the pipeline config files. + shutil.rmtree(pipeline_template_local_dir, onerror=rmtree_callback) + + +def _generate_from_pipeline_template(pipeline_template_dir: Path) -> None: + """ + Generates a pipeline config file from a given pipeline template local location + """ + pipeline_template: Template = _initialize_pipeline_template(pipeline_template_dir=pipeline_template_dir) + context: Dict = pipeline_template.run_interactive_flows() + pipeline_template.generate_project(context) + + +def _clone_app_pipeline_templates() -> Path: + """ + clone aws/aws-sam-cli-pipeline-init-templates.git Git repo to the local machine in SAM shared directory. + Returns: + the local directory path where the repo is cloned. + """ + try: + return _clone_pipeline_templates( + repo_url=APP_PIPELINE_TEMPLATES_REPO_URL, clone_name=APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME + ) + except PipelineTemplateCloneException: + # If can't clone app pipeline templates, try using an old clone from a previous run if already exist + expected_previous_clone_local_path: Path = shared_path.joinpath(APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME) + if expected_previous_clone_local_path.exists(): + click.echo("Unable to download updated app pipeline templates, using existing ones") + return expected_previous_clone_local_path + raise + + +def _clone_custom_pipeline_template(repo_url: str) -> Path: + """ + clone a given Git pipeline template's Git repo to the user machine in SAM shared directory. + Returns: the local directory path where the repo is cloned to. + + Parameters: + repo_url: the URL of the Git repo to clone + + Returns: + the local directory path where the repo is cloned. + """ + return _clone_pipeline_templates(repo_url=repo_url, clone_name=CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME) + + +def _clone_pipeline_templates(repo_url: str, clone_name: str) -> Path: + """ + clone a given pipeline templates' Git repo to the user machine inside the SAM shared directory(default: ~/.aws-sam) + under the given clone name. For example, if clone_name is "custom-pipeline-template" then the location to clone + to is "~/.aws-sam/custom-pipeline-template/" + + Parameters: + repo_url: the URL of the Git repo to clone + clone_name: The folder name to give to the created clone + + Returns: + Path to the local clone + """ + try: + repo: GitRepo = GitRepo(url=repo_url) + clone_path: Path = repo.clone(clone_dir=shared_path, clone_name=clone_name, replace_existing=True) + return clone_path + except (OSError, CloneRepoException) as ex: + raise PipelineTemplateCloneException(str(ex)) from ex + + +def _read_app_pipeline_templates_manifest(pipeline_templates_dir: Path) -> PipelineTemplatesManifest: + """ + parse and return the manifest yaml file located in the root directory of the SAM pipeline templates folder: + + Parameters: + pipeline_templates_dir: local directory of SAM pipeline templates + + Raises: + AppPipelineTemplateManifestException if the manifest is not found, ill-formatted or missing required keys + + Returns: + The manifest of the pipeline templates + """ + manifest_path: Path = Path(os.path.normpath(pipeline_templates_dir.joinpath("manifest.yaml"))) + return PipelineTemplatesManifest(manifest_path=manifest_path) + + +def _select_pipeline_template(pipeline_templates_manifest: PipelineTemplatesManifest) -> PipelineTemplateManifest: + """ + Prompts the user a list of the available CI/CD providers along with associated app pipeline templates to choose + one of them + + Parameters: + pipeline_templates_manifest: A manifest file lists the available providers and the associated pipeline templates + + Returns: + The manifest (A section in the pipeline_templates_manifest) of the chosen pipeline template; + """ + provider = _prompt_for_cicd_provider(pipeline_templates_manifest.providers) + provider_pipeline_templates: List[PipelineTemplateManifest] = list( + filter(lambda t: t.provider == provider, pipeline_templates_manifest.templates) + ) + selected_template_manifest: PipelineTemplateManifest = _prompt_for_pipeline_template(provider_pipeline_templates) + return selected_template_manifest + + +def _prompt_for_cicd_provider(available_providers: List[str]) -> str: + """ + Prompts the user a list of the available CI/CD providers to choose from + + Parameters: + available_providers: List of available CI/CD providers like Jenkins, Gitlab and CircleCI + + Returns: + The chosen provider + """ + choices = list(map(str, range(1, len(available_providers) + 1))) + click.echo("CICD provider") + for index, provider in enumerate(available_providers): + click.echo(message=f"\t{index + 1} - {provider}") + choice = click.prompt(text="Choice", show_choices=False, type=click.Choice(choices)) + return available_providers[int(choice) - 1] + + +def _prompt_for_pipeline_template( + available_pipeline_templates_manifests: List[PipelineTemplateManifest], +) -> PipelineTemplateManifest: + """ + Prompts the user a list of the available pipeline templates to choose from + + Parameters: + available_pipeline_templates_manifests: List of available pipeline templates manifests + + Returns: + The chosen pipeline template manifest + """ + choices = list(map(str, range(1, len(available_pipeline_templates_manifests) + 1))) + click.echo("Which pipeline template would you like to use?") + for index, template in enumerate(available_pipeline_templates_manifests): + click.echo(f"\t{index + 1} - {template.name}") + choice = click.prompt(text="Choice", show_choices=False, type=click.Choice(choices)) + return available_pipeline_templates_manifests[int(choice) - 1] + + +def _initialize_pipeline_template(pipeline_template_dir: Path) -> Template: + """ + Initialize a pipeline template from a given pipeline template (cookiecutter template) location + + Parameters: + pipeline_template_dir: The local location of the pipeline cookiecutter template + + Returns: + The initialized pipeline's cookiecutter template + """ + interactive_flow = _get_pipeline_template_interactive_flow(pipeline_template_dir=pipeline_template_dir) + return Template(location=str(pipeline_template_dir), interactive_flows=[interactive_flow]) + + +def _get_pipeline_template_interactive_flow(pipeline_template_dir: Path) -> InteractiveFlow: + """ + A pipeline template defines its own interactive flow (questionnaire) in a JSON file named questions.json located + in the root directory of the template. This questionnaire defines a set of questions to prompt to the user and + use the responses as the cookiecutter context + + Parameters: + pipeline_template_dir: The local location of the pipeline cookiecutter template + + Raises: + QuestionsNotFoundException: if the pipeline template is missing questions.json file. + QuestionsFailedParsingException: if questions.json file is ill-formatted or missing required keys. + + Returns: + The interactive flow + """ + flow_definition_path: str = os.path.normpath(pipeline_template_dir.joinpath("questions.json")) + return InteractiveFlowCreator.create_flow(flow_definition_path=flow_definition_path) diff --git a/samcli/commands/pipeline/init/pipeline_templates_manifest.py b/samcli/commands/pipeline/init/pipeline_templates_manifest.py new file mode 100644 index 0000000000..571e285646 --- /dev/null +++ b/samcli/commands/pipeline/init/pipeline_templates_manifest.py @@ -0,0 +1,50 @@ +""" +Represents a manifest that lists the available SAM pipeline templates. +Example: + providers: + - Jenkins + - Gitlab + - Github Actions + templates: + - name: jenkins-two-stages-pipeline + provider: Jenkins + location: templates/cookiecutter-jenkins-two-stages-pipeline + - name: gitlab-two-stages-pipeline + provider: Gitlab + location: templates/cookiecutter-gitlab-two-stages-pipeline + - name: Github-Actions-two-stages-pipeline + provider: Github Actions + location: templates/cookiecutter-github-actions-two-stages-pipeline +""" +from pathlib import Path +from typing import Dict, List + +import yaml + +from samcli.commands.exceptions import AppPipelineTemplateManifestException +from samcli.yamlhelper import parse_yaml_file + + +class PipelineTemplateManifest: + """ The metadata of a Given pipeline template""" + + def __init__(self, manifest: Dict) -> None: + self.name: str = manifest["name"] + self.provider: str = manifest["provider"] + self.location: str = manifest["location"] + + +class PipelineTemplatesManifest: + """ The metadata of the available CI/CD providers and the pipeline templates""" + + def __init__(self, manifest_path: Path) -> None: + try: + manifest: Dict = parse_yaml_file(file_path=str(manifest_path)) + self.providers: List[str] = manifest["providers"] + self.templates: List[PipelineTemplateManifest] = list(map(PipelineTemplateManifest, manifest["templates"])) + except (FileNotFoundError, KeyError, yaml.YAMLError) as ex: + raise AppPipelineTemplateManifestException( + "SAM pipeline templates manifest file is not found or ill-formatted. This could happen if the file " + f"{manifest_path} got deleted or manipulated." + "If you believe this is not the case, please file an issue at https://github.com/aws/aws-sam-cli/issues" + ) from ex diff --git a/samcli/commands/pipeline/pipeline.py b/samcli/commands/pipeline/pipeline.py index bb1247d770..2d8df4463e 100644 --- a/samcli/commands/pipeline/pipeline.py +++ b/samcli/commands/pipeline/pipeline.py @@ -1,11 +1,12 @@ """ -Command group for "pipeline" suite for commands. It provides common CLI arguments, template parsing capabilities, +Command group for "pipeline" suite commands. It provides common CLI arguments, template parsing capabilities, setting up stdin/stdout etc """ import click from .bootstrap.cli import cli as bootstrap_cli +from .init.cli import cli as init_cli @click.group() @@ -17,3 +18,4 @@ def cli() -> None: # Add individual commands under this group cli.add_command(bootstrap_cli) +cli.add_command(init_cli) diff --git a/samcli/lib/cookiecutter/interactive_flow_creator.py b/samcli/lib/cookiecutter/interactive_flow_creator.py index 9ec0dbfa79..e1429abc99 100644 --- a/samcli/lib/cookiecutter/interactive_flow_creator.py +++ b/samcli/lib/cookiecutter/interactive_flow_creator.py @@ -1,5 +1,5 @@ """ This module parses a json/yaml file that defines a flow of questions to fulfill the cookiecutter context""" -from typing import cast, Dict, Optional, Tuple +from typing import Dict, Optional, Tuple import yaml from samcli.commands.exceptions import UserException from samcli.yamlhelper import parse_yaml_file @@ -96,7 +96,7 @@ def _parse_questions_definition(file_path: str, extra_context: Optional[Dict] = """ try: - return cast(Dict, parse_yaml_file(file_path=file_path, extra_context=extra_context)) + return parse_yaml_file(file_path=file_path, extra_context=extra_context) except FileNotFoundError as ex: raise QuestionsNotFoundException(f"questions definition file not found at {file_path}") from ex except (KeyError, ValueError, yaml.YAMLError) as ex: diff --git a/samcli/yamlhelper.py b/samcli/yamlhelper.py index ca091e61cb..222c7b717e 100644 --- a/samcli/yamlhelper.py +++ b/samcli/yamlhelper.py @@ -18,7 +18,7 @@ # pylint: disable=too-many-ancestors import json -from typing import Dict, Optional +from typing import cast, Dict, Optional from botocore.compat import OrderedDict import yaml @@ -109,20 +109,20 @@ def _dict_constructor(loader, node): return OrderedDict(loader.construct_pairs(node)) -def yaml_parse(yamlstr): +def yaml_parse(yamlstr) -> Dict: """Parse a yaml string""" try: # PyYAML doesn't support json as well as it should, so if the input # is actually just json it is better to parse it with the standard # json parser. - return json.loads(yamlstr, object_pairs_hook=OrderedDict) + return cast(Dict, json.loads(yamlstr, object_pairs_hook=OrderedDict)) except ValueError: yaml.SafeLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _dict_constructor) yaml.SafeLoader.add_multi_constructor("!", intrinsics_multi_constructor) - return yaml.safe_load(yamlstr) + return cast(Dict, yaml.safe_load(yamlstr)) -def parse_yaml_file(file_path, extra_context: Optional[Dict] = None): +def parse_yaml_file(file_path, extra_context: Optional[Dict] = None) -> Dict: """ Read the file, do variable substitution, parse it as JSON/YAML diff --git a/tests/unit/commands/pipeline/init/__init__.py b/tests/unit/commands/pipeline/init/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/commands/pipeline/init/test_cli.py b/tests/unit/commands/pipeline/init/test_cli.py new file mode 100644 index 0000000000..4b9f4b6349 --- /dev/null +++ b/tests/unit/commands/pipeline/init/test_cli.py @@ -0,0 +1,22 @@ +from unittest import TestCase +from unittest.mock import patch + +from click.testing import CliRunner + +from samcli.commands.pipeline.init.cli import cli as init_cmd +from samcli.commands.pipeline.init.cli import do_cli as init_cli + + +class TestCli(TestCase): + @patch("samcli.commands.pipeline.init.cli.do_cli") + def test_cli_default_flow(self, do_cli_mock): + runner: CliRunner = CliRunner() + runner.invoke(init_cmd) + # Currently we support the interactive mode only, i.e. we don't accept any command arguments, + # instead we ask the user about the required arguments in an interactive way + do_cli_mock.assert_called_once_with() # Called without arguments + + @patch("samcli.commands.pipeline.init.cli.do_interactive") + def test_do_cli(self, do_interactive_mock): + init_cli() + do_interactive_mock.assert_called_once_with() # Called without arguments diff --git a/tests/unit/commands/pipeline/init/test_initeractive_init_flow.py b/tests/unit/commands/pipeline/init/test_initeractive_init_flow.py new file mode 100644 index 0000000000..f38222742c --- /dev/null +++ b/tests/unit/commands/pipeline/init/test_initeractive_init_flow.py @@ -0,0 +1,235 @@ +from unittest import TestCase +from unittest.mock import patch, Mock, ANY +import os +from pathlib import Path +from samcli.commands.pipeline.init.interactive_init_flow import ( + do_interactive, + PipelineTemplateCloneException, + APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME, + shared_path, + CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME, +) +from samcli.commands.pipeline.init.pipeline_templates_manifest import AppPipelineTemplateManifestException +from samcli.lib.utils.git_repo import CloneRepoException +from samcli.lib.cookiecutter.interactive_flow_creator import QuestionsNotFoundException + + +class TestInteractiveInitFlow(TestCase): + @patch("samcli.commands.pipeline.init.interactive_init_flow._read_app_pipeline_templates_manifest") + @patch("samcli.commands.pipeline.init.interactive_init_flow._select_pipeline_template") + @patch("samcli.commands.pipeline.init.interactive_init_flow._generate_from_pipeline_template") + @patch("samcli.commands.pipeline.init.interactive_init_flow.shared_path") + @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") + @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + def test_app_pipeline_templates_clone_fail_when_an_old_clone_exists( + self, + click_mock, + clone_mock, + shared_path_mock, + generate_from_pipeline_template_mock, + select_pipeline_template_mock, + read_app_pipeline_templates_manifest_mock, + ): + # setup + clone_mock.side_effect = CloneRepoException # clone fail + app_pipeline_templates_path_mock = Mock() + selected_pipeline_template_path_mock = Mock() + pipeline_templates_manifest_mock = Mock() + shared_path_mock.joinpath.return_value = app_pipeline_templates_path_mock + app_pipeline_templates_path_mock.exists.return_value = True # An old clone exists + app_pipeline_templates_path_mock.joinpath.return_value = selected_pipeline_template_path_mock + read_app_pipeline_templates_manifest_mock.return_value = pipeline_templates_manifest_mock + click_mock.prompt.return_value = "1" # App pipeline templates + + # trigger + do_interactive() + + # verify + clone_mock.assert_called_once_with( + clone_dir=shared_path_mock, clone_name=APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME, replace_existing=True + ) + app_pipeline_templates_path_mock.exists.assert_called_once() + read_app_pipeline_templates_manifest_mock.assert_called_once_with( + pipeline_templates_dir=app_pipeline_templates_path_mock + ) + select_pipeline_template_mock.assert_called_once_with( + pipeline_templates_manifest=pipeline_templates_manifest_mock + ) + generate_from_pipeline_template_mock.assert_called_once_with( + pipeline_template_dir=selected_pipeline_template_path_mock + ) + + @patch("samcli.commands.pipeline.init.interactive_init_flow.shared_path") + @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") + @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + def test_app_pipeline_templates_clone_fail_when_no_old_clone_exist(self, click_mock, clone_mock, shared_path_mock): + # setup + clone_mock.side_effect = CloneRepoException # clone fail + app_pipeline_templates_path_mock = Mock() + shared_path_mock.joinpath.return_value = app_pipeline_templates_path_mock + app_pipeline_templates_path_mock.exists.return_value = False # No old clone exists + click_mock.prompt.return_value = "1" # App pipeline templates + + # trigger + with self.assertRaises(PipelineTemplateCloneException): + do_interactive() + + @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") + @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + def test_custom_pipeline_template_clone_fail(self, click_mock, clone_mock): + # setup + clone_mock.side_effect = CloneRepoException # clone fail + click_mock.prompt.side_effect = [ + "2", # Custom pipeline templates + "https://github.com/any-custom-pipeline-template-repo.git", # Custom pipeline template repo URL + ] + + # trigger + with self.assertRaises(PipelineTemplateCloneException): + do_interactive() + + @patch("samcli.commands.pipeline.init.interactive_init_flow._read_app_pipeline_templates_manifest") + @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") + @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + def test_app_pipeline_templates_with_invalid_manifest( + self, click_mock, clone_mock, read_app_pipeline_templates_manifest_mock + ): + # setup + app_pipeline_templates_path_mock = Mock() + clone_mock.return_value = app_pipeline_templates_path_mock + read_app_pipeline_templates_manifest_mock.side_effect = AppPipelineTemplateManifestException("") + click_mock.prompt.return_value = "1" # App pipeline templates + + # trigger + with self.assertRaises(AppPipelineTemplateManifestException): + do_interactive() + + @patch("samcli.commands.pipeline.init.interactive_init_flow.shutil") + @patch("samcli.lib.cookiecutter.template.cookiecutter") + @patch("samcli.commands.pipeline.init.interactive_init_flow.InteractiveFlowCreator.create_flow") + @patch("samcli.commands.pipeline.init.interactive_init_flow._read_app_pipeline_templates_manifest") + @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") + @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_case( + self, + click_mock, + clone_mock, + read_app_pipeline_templates_manifest_mock, + create_interactive_flow_mock, + cookiecutter_mock, + shutil_mock, + ): + # setup + any_app_pipeline_templates_path = Path( + os.path.normpath(shared_path.joinpath(APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME)) + ) + clone_mock.return_value = any_app_pipeline_templates_path + jenkins_template_location = "some/location" + jenkins_template_mock = Mock( + name="Jenkins pipeline template", location=jenkins_template_location, provider="Jenkins" + ) + pipeline_templates_manifest_mock = Mock(providers=["Gitlab", "Jenkins"], templates=[jenkins_template_mock]) + read_app_pipeline_templates_manifest_mock.return_value = pipeline_templates_manifest_mock + interactive_flow_mock = Mock() + create_interactive_flow_mock.return_value = interactive_flow_mock + cookiecutter_context_mock = Mock() + interactive_flow_mock.run.return_value = cookiecutter_context_mock + + click_mock.prompt.side_effect = [ + "1", # App pipeline templates + "2", # choose "Jenkins" when prompt for CI/CD provider. (See pipeline_templates_manifest_mock, Jenkins is the 2nd provider) + "1", # choose "Jenkins pipeline template" when prompt for pipeline template + ] + + # trigger + do_interactive() + + # verify + expected_cookicutter_template_location = any_app_pipeline_templates_path.joinpath(jenkins_template_location) + clone_mock.assert_called_once_with( + clone_dir=shared_path, clone_name=APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME, replace_existing=True + ) + read_app_pipeline_templates_manifest_mock.assert_called_once_with( + pipeline_templates_dir=any_app_pipeline_templates_path + ) + create_interactive_flow_mock.assert_called_once_with( + flow_definition_path=str(expected_cookicutter_template_location.joinpath("questions.json")) + ) + interactive_flow_mock.run.assert_called_once() + cookiecutter_mock.assert_called_once_with( + template=str(expected_cookicutter_template_location), + output_dir=".", + no_input=True, + extra_context=cookiecutter_context_mock, + ) + shutil_mock.rm_tree.assert_not_called() + + @patch("samcli.commands.pipeline.init.interactive_init_flow._read_app_pipeline_templates_manifest") + @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") + @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + def test_generate_pipeline_configuration_file_when_pipeline_template_missing_questions_file( + self, click_mock, clone_mock, read_app_pipeline_templates_manifest_mock + ): + # setup + any_app_pipeline_templates_path = Path( + os.path.normpath(shared_path.joinpath(APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME)) + ) + clone_mock.return_value = any_app_pipeline_templates_path + jenkins_template_location = "some/location" + jenkins_template_mock = Mock( + name="Jenkins pipeline template", location=jenkins_template_location, provider="Jenkins" + ) + pipeline_templates_manifest_mock = Mock(providers=["Gitlab", "Jenkins"], templates=[jenkins_template_mock]) + read_app_pipeline_templates_manifest_mock.return_value = pipeline_templates_manifest_mock + + click_mock.prompt.side_effect = [ + "1", # App pipeline templates + "2", # choose "Jenkins" when prompt for CI/CD provider. (See pipeline_templates_manifest_mock, Jenkins is the 2nd provider) + "1", # choose "Jenkins pipeline template" when prompt for pipeline template + ] + + # trigger + with self.assertRaises(QuestionsNotFoundException): + do_interactive() + + @patch("samcli.commands.pipeline.init.interactive_init_flow.shutil") + @patch("samcli.lib.cookiecutter.template.cookiecutter") + @patch("samcli.commands.pipeline.init.interactive_init_flow.InteractiveFlowCreator.create_flow") + @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") + @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + def test_generate_pipeline_configuration_file_from_custom_pipeline_template_happy_case( + self, click_mock, clone_mock, create_interactive_flow_mock, cookiecutter_mock, shutil_mock + ): + # setup + any_custom_pipeline_templates_path = Path( + os.path.normpath(shared_path.joinpath(CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME)) + ) + clone_mock.return_value = any_custom_pipeline_templates_path + interactive_flow_mock = Mock() + create_interactive_flow_mock.return_value = interactive_flow_mock + cookiecutter_context_mock = Mock() + interactive_flow_mock.run.return_value = cookiecutter_context_mock + + click_mock.prompt.side_effect = [ + "2", # Custom pipeline templates + "https://github.com/any-custom-pipeline-template-repo.git", # Custom pipeline template repo URL + ] + + # trigger + do_interactive() + + # verify + clone_mock.assert_called_once_with( + clone_dir=shared_path, clone_name=CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME, replace_existing=True + ) + create_interactive_flow_mock.assert_called_once_with( + flow_definition_path=str(any_custom_pipeline_templates_path.joinpath("questions.json")) + ) + interactive_flow_mock.run.assert_called_once() + cookiecutter_mock.assert_called_once_with( + template=str(any_custom_pipeline_templates_path), + output_dir=".", + no_input=True, + extra_context=cookiecutter_context_mock, + ) + shutil_mock.rmtree.assert_called_once_with(any_custom_pipeline_templates_path, onerror=ANY) diff --git a/tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py b/tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py new file mode 100644 index 0000000000..5d37ad9764 --- /dev/null +++ b/tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py @@ -0,0 +1,76 @@ +from unittest import TestCase +import os +from pathlib import Path +from samcli.commands.pipeline.init.pipeline_templates_manifest import ( + PipelineTemplatesManifest, + PipelineTemplateManifest, + AppPipelineTemplateManifestException, +) +from samcli.lib.utils import osutils + +INVALID_YAML_MANIFEST = """ +providers: +- Jenkins with wrong identation +""" + +MISSING_KEYS_MANIFEST = """ +NotProviders: + - Jenkins +Templates: + - NotName: jenkins-two-stages-pipeline + provider: Jenkins + location: templates/cookiecutter-jenkins-two-stages-pipeline +""" + +VALID_MANIFEST = """ +providers: + - Jenkins + - Gitlab + - Github Actions +templates: + - name: jenkins-two-stages-pipeline + provider: Jenkins + location: templates/cookiecutter-jenkins-two-stages-pipeline + - name: gitlab-two-stages-pipeline + provider: Gitlab + location: templates/cookiecutter-gitlab-two-stages-pipeline + - name: Github-Actions-two-stages-pipeline + provider: Github Actions + location: templates/cookiecutter-github-actions-two-stages-pipeline +""" + + +class TestCli(TestCase): + def test_manifest_file_not_found(self): + non_existing_path = Path(os.path.normpath("/any/non/existing/manifest.yaml")) + with self.assertRaises(AppPipelineTemplateManifestException): + PipelineTemplatesManifest(manifest_path=non_existing_path) + + def test_invalid_yaml_manifest_file(self): + with osutils.mkdir_temp(ignore_errors=True) as tempdir: + manifest_path = os.path.normpath(os.path.join(tempdir, "manifest.yaml")) + with open(manifest_path, "w", encoding="utf-8") as fp: + fp.write(INVALID_YAML_MANIFEST) + with self.assertRaises(AppPipelineTemplateManifestException): + PipelineTemplatesManifest(manifest_path=Path(manifest_path)) + + def test_manifest_missing_required_keys(self): + with osutils.mkdir_temp(ignore_errors=True) as tempdir: + manifest_path = os.path.normpath(os.path.join(tempdir, "manifest.yaml")) + with open(manifest_path, "w", encoding="utf-8") as fp: + fp.write(MISSING_KEYS_MANIFEST) + with self.assertRaises(AppPipelineTemplateManifestException): + PipelineTemplatesManifest(manifest_path=Path(manifest_path)) + + def test_manifest_happy_case(self): + with osutils.mkdir_temp(ignore_errors=True) as tempdir: + manifest_path = os.path.normpath(os.path.join(tempdir, "manifest.yaml")) + with open(manifest_path, "w", encoding="utf-8") as fp: + fp.write(VALID_MANIFEST) + manifest = PipelineTemplatesManifest(manifest_path=Path(manifest_path)) + self.assertEquals(manifest.providers, ["Jenkins", "Gitlab", "Github Actions"]) + self.assertEquals(len(manifest.templates), 3) + gitlab_template: PipelineTemplateManifest = next(t for t in manifest.templates if t.provider == "Gitlab") + self.assertEquals(gitlab_template.name, "gitlab-two-stages-pipeline") + self.assertEquals(gitlab_template.provider, "Gitlab") + self.assertEquals(gitlab_template.location, "templates/cookiecutter-gitlab-two-stages-pipeline") From b7dd26a3aa474b6440618e2ac936018d34316e63 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Wed, 28 Apr 2021 18:11:14 -0700 Subject: [PATCH 2/5] apply review comments --- samcli/commands/pipeline/init/cli.py | 5 +- .../pipeline/init/interactive_init_flow.py | 127 +++++++++--------- .../init/pipeline_templates_manifest.py | 6 +- .../init/test_initeractive_init_flow.py | 84 ++++++------ .../init/test_pipeline_templates_manifest.py | 4 +- 5 files changed, 108 insertions(+), 118 deletions(-) diff --git a/samcli/commands/pipeline/init/cli.py b/samcli/commands/pipeline/init/cli.py index a52afef56b..b90308bb7d 100644 --- a/samcli/commands/pipeline/init/cli.py +++ b/samcli/commands/pipeline/init/cli.py @@ -10,9 +10,10 @@ from samcli.commands.pipeline.init.interactive_init_flow import do_interactive from samcli.lib.telemetry.metric import track_command -SHORT_HELP = "Generates a CI/CD pipeline configuration file." +SHORT_HELP = "Generates CI/CD pipeline configuration files." HELP_TEXT = """ -Generates a CI/CD pipeline configuration file for a chosen CI/CD provider like Jenkins, Gitlab and Github Actions +Generates CI/CD pipeline configuration files for a chosen CI/CD provider such as Jenkins, +GitLab CI/CD or GitHub Actions """ diff --git a/samcli/commands/pipeline/init/interactive_init_flow.py b/samcli/commands/pipeline/init/interactive_init_flow.py index a5e72284ed..5b0d75d48c 100644 --- a/samcli/commands/pipeline/init/interactive_init_flow.py +++ b/samcli/commands/pipeline/init/interactive_init_flow.py @@ -3,8 +3,6 @@ pipeline configuration file """ import logging -import os -import shutil from pathlib import Path from typing import Dict, List @@ -14,16 +12,20 @@ from samcli.commands.exceptions import PipelineTemplateCloneException from samcli.lib.cookiecutter.interactive_flow import InteractiveFlow from samcli.lib.cookiecutter.interactive_flow_creator import InteractiveFlowCreator +from samcli.lib.cookiecutter.question import Choice from samcli.lib.cookiecutter.template import Template from samcli.lib.utils.git_repo import GitRepo, CloneRepoException -from samcli.lib.utils.osutils import rmtree_callback -from .pipeline_templates_manifest import PipelineTemplateManifest, PipelineTemplatesManifest +from samcli.lib.utils import osutils +from .pipeline_templates_manifest import PipelineTemplateMetadata, PipelineTemplatesManifest LOG = logging.getLogger(__name__) shared_path: Path = global_cfg.config_dir APP_PIPELINE_TEMPLATES_REPO_URL = "https://github.com/aws/aws-sam-cli-pipeline-init-templates.git" +APP_PIPELINE_TEMPLATES_REPO_URL = "https://github.com/elbayaaa/pipeline-templates.git" APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME = "aws-sam-cli-app-pipeline-templates" CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME = "custom-pipeline-template" +SAM_PIPELINE_TEMPLATE_SOURCE = "AWS Quick Start Pipeline Templates" +CUSTOM_PIPELINE_TEMPLATE_SOURCE = "Custom Pipeline Template Location" def do_interactive() -> None: @@ -31,10 +33,13 @@ def do_interactive() -> None: An interactive flow that prompts the user for pipeline template (cookiecutter template) location, downloads it, runs its specific questionnaire then generates the pipeline config file based on the template and user's responses """ - click.echo("Which pipeline template source would you like to use?") - click.echo("\t1 - AWS Quick Start Pipeline Templates\n\t2 - Custom Pipeline Template Location") - location_choice = click.prompt("Choice", type=click.Choice(["1", "2"]), show_choices=False) - if location_choice == "2": + pipeline_template_source_question = Choice( + key="pipeline-template-source", + text="Which pipeline template source would you like to use?", + options=[SAM_PIPELINE_TEMPLATE_SOURCE, CUSTOM_PIPELINE_TEMPLATE_SOURCE], + ) + source = pipeline_template_source_question.ask() + if source == CUSTOM_PIPELINE_TEMPLATE_SOURCE: _generate_from_custom_location() else: _generate_from_app_pipeline_templates() @@ -47,37 +52,40 @@ def _generate_from_app_pipeline_templates() -> None: downloads locally, then generates the pipeline config file from the selected pipeline template. """ pipeline_templates_local_dir: Path = _clone_app_pipeline_templates() - pipeline_templates_manifest: PipelineTemplatesManifest = _read_app_pipeline_templates_manifest( - pipeline_templates_dir=pipeline_templates_local_dir + pipeline_templates_local_dir ) # The manifest contains multiple pipeline-templates so select one - selected_pipeline_template_manifest: PipelineTemplateManifest = _select_pipeline_template( - pipeline_templates_manifest=pipeline_templates_manifest + selected_pipeline_template_metadata: PipelineTemplateMetadata = _select_pipeline_template( + pipeline_templates_manifest ) selected_pipeline_template_dir: Path = pipeline_templates_local_dir.joinpath( - selected_pipeline_template_manifest.location + selected_pipeline_template_metadata.location ) - _generate_from_pipeline_template(pipeline_template_dir=selected_pipeline_template_dir) + _generate_from_pipeline_template(selected_pipeline_template_dir) def _generate_from_custom_location() -> None: """ Prompts the user for a custom pipeline template location, downloads locally, then generates the pipeline config file """ - pipeline_template_repo_url: str = click.prompt(text="template Git location") - pipeline_template_local_dir: Path = _clone_custom_pipeline_template(repo_url=pipeline_template_repo_url) - _generate_from_pipeline_template(pipeline_template_dir=pipeline_template_local_dir) # Unlike app pipeline templates, custom pipeline templates are not shared between different SAM applications - # and should be cleaned up from users' machines after generating the pipeline config files. - shutil.rmtree(pipeline_template_local_dir, onerror=rmtree_callback) + # and should be cleaned up from users' machines after generating the pipeline config files, so, we are creating + # inside a tem directory + with osutils.mkdir_temp(ignore_errors=True) as tempdir: + tempdir_path = Path(tempdir) + pipeline_template_repo_url: str = click.prompt("Template Git location") + pipeline_template_local_dir: Path = _clone_pipeline_templates( + pipeline_template_repo_url, tempdir_path, CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME + ) + _generate_from_pipeline_template(pipeline_template_local_dir) def _generate_from_pipeline_template(pipeline_template_dir: Path) -> None: """ Generates a pipeline config file from a given pipeline template local location """ - pipeline_template: Template = _initialize_pipeline_template(pipeline_template_dir=pipeline_template_dir) + pipeline_template: Template = _initialize_pipeline_template(pipeline_template_dir) context: Dict = pipeline_template.run_interactive_flows() pipeline_template.generate_project(context) @@ -90,7 +98,9 @@ def _clone_app_pipeline_templates() -> Path: """ try: return _clone_pipeline_templates( - repo_url=APP_PIPELINE_TEMPLATES_REPO_URL, clone_name=APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME + repo_url=APP_PIPELINE_TEMPLATES_REPO_URL, + clone_dir=shared_path, + clone_name=APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME, ) except PipelineTemplateCloneException: # If can't clone app pipeline templates, try using an old clone from a previous run if already exist @@ -101,36 +111,23 @@ def _clone_app_pipeline_templates() -> Path: raise -def _clone_custom_pipeline_template(repo_url: str) -> Path: +def _clone_pipeline_templates(repo_url: str, clone_dir: Path, clone_name: str) -> Path: """ - clone a given Git pipeline template's Git repo to the user machine in SAM shared directory. - Returns: the local directory path where the repo is cloned to. - - Parameters: - repo_url: the URL of the Git repo to clone - - Returns: - the local directory path where the repo is cloned. - """ - return _clone_pipeline_templates(repo_url=repo_url, clone_name=CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME) - - -def _clone_pipeline_templates(repo_url: str, clone_name: str) -> Path: - """ - clone a given pipeline templates' Git repo to the user machine inside the SAM shared directory(default: ~/.aws-sam) + clone a given pipeline templates' Git repo to the user machine inside the given clone_dir directory under the given clone name. For example, if clone_name is "custom-pipeline-template" then the location to clone - to is "~/.aws-sam/custom-pipeline-template/" + to is "/clone/dir/path/custom-pipeline-template/" Parameters: repo_url: the URL of the Git repo to clone - clone_name: The folder name to give to the created clone + clone_dir: the local parent directory to clone to + clone_name: The folder name to give to the created clone inside clone_dir Returns: Path to the local clone """ try: - repo: GitRepo = GitRepo(url=repo_url) - clone_path: Path = repo.clone(clone_dir=shared_path, clone_name=clone_name, replace_existing=True) + repo: GitRepo = GitRepo(repo_url) + clone_path: Path = repo.clone(clone_dir, clone_name, replace_existing=True) return clone_path except (OSError, CloneRepoException) as ex: raise PipelineTemplateCloneException(str(ex)) from ex @@ -149,11 +146,11 @@ def _read_app_pipeline_templates_manifest(pipeline_templates_dir: Path) -> Pipel Returns: The manifest of the pipeline templates """ - manifest_path: Path = Path(os.path.normpath(pipeline_templates_dir.joinpath("manifest.yaml"))) - return PipelineTemplatesManifest(manifest_path=manifest_path) + manifest_path: Path = Path(pipeline_templates_dir.joinpath("manifest.yaml")) + return PipelineTemplatesManifest(manifest_path) -def _select_pipeline_template(pipeline_templates_manifest: PipelineTemplatesManifest) -> PipelineTemplateManifest: +def _select_pipeline_template(pipeline_templates_manifest: PipelineTemplatesManifest) -> PipelineTemplateMetadata: """ Prompts the user a list of the available CI/CD providers along with associated app pipeline templates to choose one of them @@ -165,10 +162,10 @@ def _select_pipeline_template(pipeline_templates_manifest: PipelineTemplatesMani The manifest (A section in the pipeline_templates_manifest) of the chosen pipeline template; """ provider = _prompt_for_cicd_provider(pipeline_templates_manifest.providers) - provider_pipeline_templates: List[PipelineTemplateManifest] = list( - filter(lambda t: t.provider == provider, pipeline_templates_manifest.templates) - ) - selected_template_manifest: PipelineTemplateManifest = _prompt_for_pipeline_template(provider_pipeline_templates) + provider_pipeline_templates: List[PipelineTemplateMetadata] = [ + t for t in pipeline_templates_manifest.templates if t.provider == provider + ] + selected_template_manifest: PipelineTemplateMetadata = _prompt_for_pipeline_template(provider_pipeline_templates) return selected_template_manifest @@ -182,32 +179,30 @@ def _prompt_for_cicd_provider(available_providers: List[str]) -> str: Returns: The chosen provider """ - choices = list(map(str, range(1, len(available_providers) + 1))) - click.echo("CICD provider") - for index, provider in enumerate(available_providers): - click.echo(message=f"\t{index + 1} - {provider}") - choice = click.prompt(text="Choice", show_choices=False, type=click.Choice(choices)) - return available_providers[int(choice) - 1] + question_to_choose_provider = Choice(key="provider", text="CI/CD provider", options=available_providers) + chosen_provider = question_to_choose_provider.ask() + return chosen_provider def _prompt_for_pipeline_template( - available_pipeline_templates_manifests: List[PipelineTemplateManifest], -) -> PipelineTemplateManifest: + available_pipeline_templates_metadata: List[PipelineTemplateMetadata], +) -> PipelineTemplateMetadata: """ Prompts the user a list of the available pipeline templates to choose from Parameters: - available_pipeline_templates_manifests: List of available pipeline templates manifests + available_pipeline_templates_metadata: List of available pipeline templates manifests Returns: The chosen pipeline template manifest """ - choices = list(map(str, range(1, len(available_pipeline_templates_manifests) + 1))) - click.echo("Which pipeline template would you like to use?") - for index, template in enumerate(available_pipeline_templates_manifests): - click.echo(f"\t{index + 1} - {template.name}") - choice = click.prompt(text="Choice", show_choices=False, type=click.Choice(choices)) - return available_pipeline_templates_manifests[int(choice) - 1] + question_to_choose_pipeline_template = Choice( + key="pipeline-template", + text="Which pipeline template would you like to use?", + options=[t.name for t in available_pipeline_templates_metadata], + ) + chosen_pipeline_template_name = question_to_choose_pipeline_template.ask() + return next(t for t in available_pipeline_templates_metadata if t.name == chosen_pipeline_template_name) def _initialize_pipeline_template(pipeline_template_dir: Path) -> Template: @@ -220,7 +215,7 @@ def _initialize_pipeline_template(pipeline_template_dir: Path) -> Template: Returns: The initialized pipeline's cookiecutter template """ - interactive_flow = _get_pipeline_template_interactive_flow(pipeline_template_dir=pipeline_template_dir) + interactive_flow = _get_pipeline_template_interactive_flow(pipeline_template_dir) return Template(location=str(pipeline_template_dir), interactive_flows=[interactive_flow]) @@ -240,5 +235,5 @@ def _get_pipeline_template_interactive_flow(pipeline_template_dir: Path) -> Inte Returns: The interactive flow """ - flow_definition_path: str = os.path.normpath(pipeline_template_dir.joinpath("questions.json")) - return InteractiveFlowCreator.create_flow(flow_definition_path=flow_definition_path) + flow_definition_path: Path = pipeline_template_dir.joinpath("questions.json") + return InteractiveFlowCreator.create_flow(str(flow_definition_path)) diff --git a/samcli/commands/pipeline/init/pipeline_templates_manifest.py b/samcli/commands/pipeline/init/pipeline_templates_manifest.py index 571e285646..9362b67ae3 100644 --- a/samcli/commands/pipeline/init/pipeline_templates_manifest.py +++ b/samcli/commands/pipeline/init/pipeline_templates_manifest.py @@ -25,7 +25,7 @@ from samcli.yamlhelper import parse_yaml_file -class PipelineTemplateManifest: +class PipelineTemplateMetadata: """ The metadata of a Given pipeline template""" def __init__(self, manifest: Dict) -> None: @@ -41,10 +41,10 @@ def __init__(self, manifest_path: Path) -> None: try: manifest: Dict = parse_yaml_file(file_path=str(manifest_path)) self.providers: List[str] = manifest["providers"] - self.templates: List[PipelineTemplateManifest] = list(map(PipelineTemplateManifest, manifest["templates"])) + self.templates: List[PipelineTemplateMetadata] = list(map(PipelineTemplateMetadata, manifest["templates"])) except (FileNotFoundError, KeyError, yaml.YAMLError) as ex: raise AppPipelineTemplateManifestException( "SAM pipeline templates manifest file is not found or ill-formatted. This could happen if the file " - f"{manifest_path} got deleted or manipulated." + f"{manifest_path} got deleted or modified." "If you believe this is not the case, please file an issue at https://github.com/aws/aws-sam-cli/issues" ) from ex diff --git a/tests/unit/commands/pipeline/init/test_initeractive_init_flow.py b/tests/unit/commands/pipeline/init/test_initeractive_init_flow.py index f38222742c..1284d7e3d5 100644 --- a/tests/unit/commands/pipeline/init/test_initeractive_init_flow.py +++ b/tests/unit/commands/pipeline/init/test_initeractive_init_flow.py @@ -20,7 +20,7 @@ class TestInteractiveInitFlow(TestCase): @patch("samcli.commands.pipeline.init.interactive_init_flow._generate_from_pipeline_template") @patch("samcli.commands.pipeline.init.interactive_init_flow.shared_path") @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") - @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + @patch("samcli.lib.cookiecutter.question.click") def test_app_pipeline_templates_clone_fail_when_an_old_clone_exists( self, click_mock, @@ -46,22 +46,16 @@ def test_app_pipeline_templates_clone_fail_when_an_old_clone_exists( # verify clone_mock.assert_called_once_with( - clone_dir=shared_path_mock, clone_name=APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME, replace_existing=True + shared_path_mock, APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME, replace_existing=True ) app_pipeline_templates_path_mock.exists.assert_called_once() - read_app_pipeline_templates_manifest_mock.assert_called_once_with( - pipeline_templates_dir=app_pipeline_templates_path_mock - ) - select_pipeline_template_mock.assert_called_once_with( - pipeline_templates_manifest=pipeline_templates_manifest_mock - ) - generate_from_pipeline_template_mock.assert_called_once_with( - pipeline_template_dir=selected_pipeline_template_path_mock - ) + read_app_pipeline_templates_manifest_mock.assert_called_once_with(app_pipeline_templates_path_mock) + select_pipeline_template_mock.assert_called_once_with(pipeline_templates_manifest_mock) + generate_from_pipeline_template_mock.assert_called_once_with(selected_pipeline_template_path_mock) @patch("samcli.commands.pipeline.init.interactive_init_flow.shared_path") @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") - @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + @patch("samcli.lib.cookiecutter.question.click") def test_app_pipeline_templates_clone_fail_when_no_old_clone_exist(self, click_mock, clone_mock, shared_path_mock): # setup clone_mock.side_effect = CloneRepoException # clone fail @@ -76,13 +70,14 @@ def test_app_pipeline_templates_clone_fail_when_no_old_clone_exist(self, click_m @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") @patch("samcli.commands.pipeline.init.interactive_init_flow.click") - def test_custom_pipeline_template_clone_fail(self, click_mock, clone_mock): + @patch("samcli.lib.cookiecutter.question.click") + def test_custom_pipeline_template_clone_fail(self, question_click_mock, init_click_mock, clone_mock): # setup clone_mock.side_effect = CloneRepoException # clone fail - click_mock.prompt.side_effect = [ - "2", # Custom pipeline templates - "https://github.com/any-custom-pipeline-template-repo.git", # Custom pipeline template repo URL - ] + question_click_mock.prompt.return_value = "2" # Custom pipeline templates + init_click_mock.prompt.return_value = ( + "https://github.com/any-custom-pipeline-template-repo.git" # Custom pipeline template repo URL + ) # trigger with self.assertRaises(PipelineTemplateCloneException): @@ -90,7 +85,7 @@ def test_custom_pipeline_template_clone_fail(self, click_mock, clone_mock): @patch("samcli.commands.pipeline.init.interactive_init_flow._read_app_pipeline_templates_manifest") @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") - @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + @patch("samcli.lib.cookiecutter.question.click") def test_app_pipeline_templates_with_invalid_manifest( self, click_mock, clone_mock, read_app_pipeline_templates_manifest_mock ): @@ -104,12 +99,11 @@ def test_app_pipeline_templates_with_invalid_manifest( with self.assertRaises(AppPipelineTemplateManifestException): do_interactive() - @patch("samcli.commands.pipeline.init.interactive_init_flow.shutil") @patch("samcli.lib.cookiecutter.template.cookiecutter") @patch("samcli.commands.pipeline.init.interactive_init_flow.InteractiveFlowCreator.create_flow") @patch("samcli.commands.pipeline.init.interactive_init_flow._read_app_pipeline_templates_manifest") @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") - @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + @patch("samcli.lib.cookiecutter.question.click") def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_case( self, click_mock, @@ -117,7 +111,6 @@ def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_c read_app_pipeline_templates_manifest_mock, create_interactive_flow_mock, cookiecutter_mock, - shutil_mock, ): # setup any_app_pipeline_templates_path = Path( @@ -146,14 +139,10 @@ def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_c # verify expected_cookicutter_template_location = any_app_pipeline_templates_path.joinpath(jenkins_template_location) - clone_mock.assert_called_once_with( - clone_dir=shared_path, clone_name=APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME, replace_existing=True - ) - read_app_pipeline_templates_manifest_mock.assert_called_once_with( - pipeline_templates_dir=any_app_pipeline_templates_path - ) + clone_mock.assert_called_once_with(shared_path, APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME, replace_existing=True) + read_app_pipeline_templates_manifest_mock.assert_called_once_with(any_app_pipeline_templates_path) create_interactive_flow_mock.assert_called_once_with( - flow_definition_path=str(expected_cookicutter_template_location.joinpath("questions.json")) + str(expected_cookicutter_template_location.joinpath("questions.json")) ) interactive_flow_mock.run.assert_called_once() cookiecutter_mock.assert_called_once_with( @@ -162,18 +151,15 @@ def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_c no_input=True, extra_context=cookiecutter_context_mock, ) - shutil_mock.rm_tree.assert_not_called() @patch("samcli.commands.pipeline.init.interactive_init_flow._read_app_pipeline_templates_manifest") @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") - @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + @patch("samcli.lib.cookiecutter.question.click") def test_generate_pipeline_configuration_file_when_pipeline_template_missing_questions_file( self, click_mock, clone_mock, read_app_pipeline_templates_manifest_mock ): # setup - any_app_pipeline_templates_path = Path( - os.path.normpath(shared_path.joinpath(APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME)) - ) + any_app_pipeline_templates_path = shared_path.joinpath(APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME) clone_mock.return_value = any_app_pipeline_templates_path jenkins_template_location = "some/location" jenkins_template_mock = Mock( @@ -192,38 +178,47 @@ def test_generate_pipeline_configuration_file_when_pipeline_template_missing_que with self.assertRaises(QuestionsNotFoundException): do_interactive() - @patch("samcli.commands.pipeline.init.interactive_init_flow.shutil") + @patch("samcli.commands.pipeline.init.interactive_init_flow.osutils") @patch("samcli.lib.cookiecutter.template.cookiecutter") @patch("samcli.commands.pipeline.init.interactive_init_flow.InteractiveFlowCreator.create_flow") @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + @patch("samcli.lib.cookiecutter.question.click") def test_generate_pipeline_configuration_file_from_custom_pipeline_template_happy_case( - self, click_mock, clone_mock, create_interactive_flow_mock, cookiecutter_mock, shutil_mock + self, + questions_click_mock, + init_click_mock, + clone_mock, + create_interactive_flow_mock, + cookiecutter_mock, + osutils_mock, ): # setup - any_custom_pipeline_templates_path = Path( - os.path.normpath(shared_path.joinpath(CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME)) - ) + any_temp_dir = "/tmp/any/dir" + osutils_mock.mkdir_temp.return_value.__enter__ = Mock(return_value=any_temp_dir) + osutils_mock.mkdir_temp.return_value.__exit__ = Mock() + any_custom_pipeline_templates_path = Path(os.path.join(any_temp_dir, CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME)) clone_mock.return_value = any_custom_pipeline_templates_path interactive_flow_mock = Mock() create_interactive_flow_mock.return_value = interactive_flow_mock cookiecutter_context_mock = Mock() interactive_flow_mock.run.return_value = cookiecutter_context_mock - click_mock.prompt.side_effect = [ - "2", # Custom pipeline templates - "https://github.com/any-custom-pipeline-template-repo.git", # Custom pipeline template repo URL - ] + questions_click_mock.prompt.return_value = "2" # Custom pipeline templates + init_click_mock.prompt.return_value = ( + "https://github.com/any-custom-pipeline-template-repo.git", + ) # Custom pipeline template repo URL # trigger do_interactive() # verify + osutils_mock.mkdir_temp.assert_called_once() # Custom templates are cloned to temp clone_mock.assert_called_once_with( - clone_dir=shared_path, clone_name=CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME, replace_existing=True + Path(any_temp_dir), CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME, replace_existing=True ) create_interactive_flow_mock.assert_called_once_with( - flow_definition_path=str(any_custom_pipeline_templates_path.joinpath("questions.json")) + str(any_custom_pipeline_templates_path.joinpath("questions.json")) ) interactive_flow_mock.run.assert_called_once() cookiecutter_mock.assert_called_once_with( @@ -232,4 +227,3 @@ def test_generate_pipeline_configuration_file_from_custom_pipeline_template_happ no_input=True, extra_context=cookiecutter_context_mock, ) - shutil_mock.rmtree.assert_called_once_with(any_custom_pipeline_templates_path, onerror=ANY) diff --git a/tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py b/tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py index 5d37ad9764..acfb590cf2 100644 --- a/tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py +++ b/tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py @@ -3,7 +3,7 @@ from pathlib import Path from samcli.commands.pipeline.init.pipeline_templates_manifest import ( PipelineTemplatesManifest, - PipelineTemplateManifest, + PipelineTemplateMetadata, AppPipelineTemplateManifestException, ) from samcli.lib.utils import osutils @@ -70,7 +70,7 @@ def test_manifest_happy_case(self): manifest = PipelineTemplatesManifest(manifest_path=Path(manifest_path)) self.assertEquals(manifest.providers, ["Jenkins", "Gitlab", "Github Actions"]) self.assertEquals(len(manifest.templates), 3) - gitlab_template: PipelineTemplateManifest = next(t for t in manifest.templates if t.provider == "Gitlab") + gitlab_template: PipelineTemplateMetadata = next(t for t in manifest.templates if t.provider == "Gitlab") self.assertEquals(gitlab_template.name, "gitlab-two-stages-pipeline") self.assertEquals(gitlab_template.provider, "Gitlab") self.assertEquals(gitlab_template.location, "templates/cookiecutter-gitlab-two-stages-pipeline") From c14cb51836b093c0638264a43be11706d2ac27a0 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Thu, 29 Apr 2021 14:10:33 -0700 Subject: [PATCH 3/5] apply review comments --- .../pipeline/init/interactive_init_flow.py | 70 +++++++++++-------- .../init/pipeline_templates_manifest.py | 14 +++- samcli/lib/utils/git_repo.py | 2 +- .../init/test_initeractive_init_flow.py | 56 ++++++++++++--- .../init/test_pipeline_templates_manifest.py | 32 +++++---- 5 files changed, 118 insertions(+), 56 deletions(-) diff --git a/samcli/commands/pipeline/init/interactive_init_flow.py b/samcli/commands/pipeline/init/interactive_init_flow.py index 5b0d75d48c..88c64e81b6 100644 --- a/samcli/commands/pipeline/init/interactive_init_flow.py +++ b/samcli/commands/pipeline/init/interactive_init_flow.py @@ -3,6 +3,7 @@ pipeline configuration file """ import logging +import os from pathlib import Path from typing import Dict, List @@ -14,14 +15,13 @@ from samcli.lib.cookiecutter.interactive_flow_creator import InteractiveFlowCreator from samcli.lib.cookiecutter.question import Choice from samcli.lib.cookiecutter.template import Template -from samcli.lib.utils.git_repo import GitRepo, CloneRepoException from samcli.lib.utils import osutils -from .pipeline_templates_manifest import PipelineTemplateMetadata, PipelineTemplatesManifest +from samcli.lib.utils.git_repo import GitRepo, CloneRepoException +from .pipeline_templates_manifest import Provider, PipelineTemplateMetadata, PipelineTemplatesManifest LOG = logging.getLogger(__name__) shared_path: Path = global_cfg.config_dir APP_PIPELINE_TEMPLATES_REPO_URL = "https://github.com/aws/aws-sam-cli-pipeline-init-templates.git" -APP_PIPELINE_TEMPLATES_REPO_URL = "https://github.com/elbayaaa/pipeline-templates.git" APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME = "aws-sam-cli-app-pipeline-templates" CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME = "custom-pipeline-template" SAM_PIPELINE_TEMPLATE_SOURCE = "AWS Quick Start Pipeline Templates" @@ -56,7 +56,7 @@ def _generate_from_app_pipeline_templates() -> None: pipeline_templates_local_dir ) # The manifest contains multiple pipeline-templates so select one - selected_pipeline_template_metadata: PipelineTemplateMetadata = _select_pipeline_template( + selected_pipeline_template_metadata: PipelineTemplateMetadata = _prompt_pipeline_template( pipeline_templates_manifest ) selected_pipeline_template_dir: Path = pipeline_templates_local_dir.joinpath( @@ -69,16 +69,16 @@ def _generate_from_custom_location() -> None: """ Prompts the user for a custom pipeline template location, downloads locally, then generates the pipeline config file """ - # Unlike app pipeline templates, custom pipeline templates are not shared between different SAM applications - # and should be cleaned up from users' machines after generating the pipeline config files, so, we are creating - # inside a tem directory - with osutils.mkdir_temp(ignore_errors=True) as tempdir: - tempdir_path = Path(tempdir) - pipeline_template_repo_url: str = click.prompt("Template Git location") - pipeline_template_local_dir: Path = _clone_pipeline_templates( - pipeline_template_repo_url, tempdir_path, CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME - ) - _generate_from_pipeline_template(pipeline_template_local_dir) + pipeline_template_git_location: str = click.prompt("Template Git location") + if os.path.exists(pipeline_template_git_location): + _generate_from_pipeline_template(Path(pipeline_template_git_location)) + else: + with osutils.mkdir_temp(ignore_errors=True) as tempdir: + tempdir_path = Path(tempdir) + pipeline_template_local_dir: Path = _clone_pipeline_templates( + pipeline_template_git_location, tempdir_path, CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME + ) + _generate_from_pipeline_template(pipeline_template_local_dir) def _generate_from_pipeline_template(pipeline_template_dir: Path) -> None: @@ -146,11 +146,11 @@ def _read_app_pipeline_templates_manifest(pipeline_templates_dir: Path) -> Pipel Returns: The manifest of the pipeline templates """ - manifest_path: Path = Path(pipeline_templates_dir.joinpath("manifest.yaml")) + manifest_path: Path = pipeline_templates_dir.joinpath("manifest.yaml") return PipelineTemplatesManifest(manifest_path) -def _select_pipeline_template(pipeline_templates_manifest: PipelineTemplatesManifest) -> PipelineTemplateMetadata: +def _prompt_pipeline_template(pipeline_templates_manifest: PipelineTemplatesManifest) -> PipelineTemplateMetadata: """ Prompts the user a list of the available CI/CD providers along with associated app pipeline templates to choose one of them @@ -161,37 +161,43 @@ def _select_pipeline_template(pipeline_templates_manifest: PipelineTemplatesMani Returns: The manifest (A section in the pipeline_templates_manifest) of the chosen pipeline template; """ - provider = _prompt_for_cicd_provider(pipeline_templates_manifest.providers) + provider = _prompt_cicd_provider(pipeline_templates_manifest.providers) provider_pipeline_templates: List[PipelineTemplateMetadata] = [ - t for t in pipeline_templates_manifest.templates if t.provider == provider + t for t in pipeline_templates_manifest.templates if t.provider == provider.id ] - selected_template_manifest: PipelineTemplateMetadata = _prompt_for_pipeline_template(provider_pipeline_templates) + selected_template_manifest: PipelineTemplateMetadata = _prompt_provider_pipeline_template( + provider_pipeline_templates + ) return selected_template_manifest -def _prompt_for_cicd_provider(available_providers: List[str]) -> str: +def _prompt_cicd_provider(available_providers: List[Provider]) -> Provider: """ Prompts the user a list of the available CI/CD providers to choose from Parameters: - available_providers: List of available CI/CD providers like Jenkins, Gitlab and CircleCI + available_providers: List of available CI/CD providers such as Jenkins, Gitlab and CircleCI Returns: The chosen provider """ - question_to_choose_provider = Choice(key="provider", text="CI/CD provider", options=available_providers) - chosen_provider = question_to_choose_provider.ask() - return chosen_provider + question_to_choose_provider = Choice( + key="provider", + text="CI/CD provider", + options=[p.display_name for p in available_providers], + ) + chosen_provider_display_name = question_to_choose_provider.ask() + return next(p for p in available_providers if p.display_name == chosen_provider_display_name) -def _prompt_for_pipeline_template( - available_pipeline_templates_metadata: List[PipelineTemplateMetadata], +def _prompt_provider_pipeline_template( + provider_available_pipeline_templates_metadata: List[PipelineTemplateMetadata], ) -> PipelineTemplateMetadata: """ Prompts the user a list of the available pipeline templates to choose from Parameters: - available_pipeline_templates_metadata: List of available pipeline templates manifests + provider_available_pipeline_templates_metadata: List of available pipeline templates manifests Returns: The chosen pipeline template manifest @@ -199,10 +205,14 @@ def _prompt_for_pipeline_template( question_to_choose_pipeline_template = Choice( key="pipeline-template", text="Which pipeline template would you like to use?", - options=[t.name for t in available_pipeline_templates_metadata], + options=[t.display_name for t in provider_available_pipeline_templates_metadata], + ) + chosen_pipeline_template_display_name = question_to_choose_pipeline_template.ask() + return next( + t + for t in provider_available_pipeline_templates_metadata + if t.display_name == chosen_pipeline_template_display_name ) - chosen_pipeline_template_name = question_to_choose_pipeline_template.ask() - return next(t for t in available_pipeline_templates_metadata if t.name == chosen_pipeline_template_name) def _initialize_pipeline_template(pipeline_template_dir: Path) -> Template: diff --git a/samcli/commands/pipeline/init/pipeline_templates_manifest.py b/samcli/commands/pipeline/init/pipeline_templates_manifest.py index 9362b67ae3..815263f5f7 100644 --- a/samcli/commands/pipeline/init/pipeline_templates_manifest.py +++ b/samcli/commands/pipeline/init/pipeline_templates_manifest.py @@ -25,11 +25,19 @@ from samcli.yamlhelper import parse_yaml_file +class Provider: + """ CI/CD provider such as Jenkins, Gitlab and GitHub-Actions""" + + def __init__(self, manifest: Dict) -> None: + self.id: str = manifest["id"] + self.display_name: str = manifest["displayName"] + + class PipelineTemplateMetadata: """ The metadata of a Given pipeline template""" def __init__(self, manifest: Dict) -> None: - self.name: str = manifest["name"] + self.display_name: str = manifest["displayName"] self.provider: str = manifest["provider"] self.location: str = manifest["location"] @@ -40,9 +48,9 @@ class PipelineTemplatesManifest: def __init__(self, manifest_path: Path) -> None: try: manifest: Dict = parse_yaml_file(file_path=str(manifest_path)) - self.providers: List[str] = manifest["providers"] + self.providers: List[Provider] = list(map(Provider, manifest["providers"])) self.templates: List[PipelineTemplateMetadata] = list(map(PipelineTemplateMetadata, manifest["templates"])) - except (FileNotFoundError, KeyError, yaml.YAMLError) as ex: + except (FileNotFoundError, KeyError, TypeError, yaml.YAMLError) as ex: raise AppPipelineTemplateManifestException( "SAM pipeline templates manifest file is not found or ill-formatted. This could happen if the file " f"{manifest_path} got deleted or modified." diff --git a/samcli/lib/utils/git_repo.py b/samcli/lib/utils/git_repo.py index 33e4597726..ddc7fba52f 100644 --- a/samcli/lib/utils/git_repo.py +++ b/samcli/lib/utils/git_repo.py @@ -132,7 +132,7 @@ def clone(self, clone_dir: Path, clone_name: str, replace_existing: bool = False output = clone_error.output.decode("utf-8") if "not found" in output.lower(): LOG.warning("WARN: Could not clone repo %s", self.url, exc_info=clone_error) - raise CloneRepoException from clone_error + raise CloneRepoException(output) from clone_error finally: self.clone_attempted = True diff --git a/tests/unit/commands/pipeline/init/test_initeractive_init_flow.py b/tests/unit/commands/pipeline/init/test_initeractive_init_flow.py index 1284d7e3d5..10668acd51 100644 --- a/tests/unit/commands/pipeline/init/test_initeractive_init_flow.py +++ b/tests/unit/commands/pipeline/init/test_initeractive_init_flow.py @@ -16,7 +16,7 @@ class TestInteractiveInitFlow(TestCase): @patch("samcli.commands.pipeline.init.interactive_init_flow._read_app_pipeline_templates_manifest") - @patch("samcli.commands.pipeline.init.interactive_init_flow._select_pipeline_template") + @patch("samcli.commands.pipeline.init.interactive_init_flow._prompt_pipeline_template") @patch("samcli.commands.pipeline.init.interactive_init_flow._generate_from_pipeline_template") @patch("samcli.commands.pipeline.init.interactive_init_flow.shared_path") @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") @@ -119,9 +119,15 @@ def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_c clone_mock.return_value = any_app_pipeline_templates_path jenkins_template_location = "some/location" jenkins_template_mock = Mock( - name="Jenkins pipeline template", location=jenkins_template_location, provider="Jenkins" + display_name="Jenkins pipeline template", location=jenkins_template_location, provider="jenkins" + ) + pipeline_templates_manifest_mock = Mock( + providers=[ + Mock(id="gitlab", display_name="Gitlab"), + Mock(id="jenkins", display_name="Jenkins"), + ], + templates=[jenkins_template_mock], ) - pipeline_templates_manifest_mock = Mock(providers=["Gitlab", "Jenkins"], templates=[jenkins_template_mock]) read_app_pipeline_templates_manifest_mock.return_value = pipeline_templates_manifest_mock interactive_flow_mock = Mock() create_interactive_flow_mock.return_value = interactive_flow_mock @@ -163,9 +169,15 @@ def test_generate_pipeline_configuration_file_when_pipeline_template_missing_que clone_mock.return_value = any_app_pipeline_templates_path jenkins_template_location = "some/location" jenkins_template_mock = Mock( - name="Jenkins pipeline template", location=jenkins_template_location, provider="Jenkins" + display_name="Jenkins pipeline template", location=jenkins_template_location, provider="jenkins" + ) + pipeline_templates_manifest_mock = Mock( + providers=[ + Mock(id="gitlab", display_name="Gitlab"), + Mock(id="jenkins", display_name="Jenkins"), + ], + templates=[jenkins_template_mock], ) - pipeline_templates_manifest_mock = Mock(providers=["Gitlab", "Jenkins"], templates=[jenkins_template_mock]) read_app_pipeline_templates_manifest_mock.return_value = pipeline_templates_manifest_mock click_mock.prompt.side_effect = [ @@ -178,13 +190,41 @@ def test_generate_pipeline_configuration_file_when_pipeline_template_missing_que with self.assertRaises(QuestionsNotFoundException): do_interactive() + @patch("samcli.commands.pipeline.init.interactive_init_flow.os") + @patch("samcli.commands.pipeline.init.interactive_init_flow.osutils") + @patch("samcli.commands.pipeline.init.interactive_init_flow._generate_from_pipeline_template") + @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") + @patch("samcli.commands.pipeline.init.interactive_init_flow.click") + @patch("samcli.lib.cookiecutter.question.click") + def test_generate_pipeline_configuration_file_from_custom_local_existing_path_will_not_do_git_clone( + self, + questions_click_mock, + init_click_mock, + clone_mock, + generate_from_pipeline_template_mock, + osutils_mock, + os_mock, + ): + # setup + local_pipeline_templates_path = "/any/existing/local/path" + os_mock.path.exists.return_value = True + questions_click_mock.prompt.return_value = "2" # Custom pipeline templates + init_click_mock.prompt.return_value = local_pipeline_templates_path # git repo path + # trigger + do_interactive() + + # verify + osutils_mock.mkdir_temp.assert_not_called() + clone_mock.assert_not_called() + generate_from_pipeline_template_mock.assert_called_once_with(Path(local_pipeline_templates_path)) + @patch("samcli.commands.pipeline.init.interactive_init_flow.osutils") @patch("samcli.lib.cookiecutter.template.cookiecutter") @patch("samcli.commands.pipeline.init.interactive_init_flow.InteractiveFlowCreator.create_flow") @patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone") @patch("samcli.commands.pipeline.init.interactive_init_flow.click") @patch("samcli.lib.cookiecutter.question.click") - def test_generate_pipeline_configuration_file_from_custom_pipeline_template_happy_case( + def test_generate_pipeline_configuration_file_from_custom_remote_pipeline_template_happy_case( self, questions_click_mock, init_click_mock, @@ -205,9 +245,7 @@ def test_generate_pipeline_configuration_file_from_custom_pipeline_template_happ interactive_flow_mock.run.return_value = cookiecutter_context_mock questions_click_mock.prompt.return_value = "2" # Custom pipeline templates - init_click_mock.prompt.return_value = ( - "https://github.com/any-custom-pipeline-template-repo.git", - ) # Custom pipeline template repo URL + init_click_mock.prompt.return_value = "https://github.com/any-custom-pipeline-template-repo.git" # trigger do_interactive() diff --git a/tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py b/tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py index acfb590cf2..a1b765bb5c 100644 --- a/tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py +++ b/tests/unit/commands/pipeline/init/test_pipeline_templates_manifest.py @@ -2,6 +2,7 @@ import os from pathlib import Path from samcli.commands.pipeline.init.pipeline_templates_manifest import ( + Provider, PipelineTemplatesManifest, PipelineTemplateMetadata, AppPipelineTemplateManifestException, @@ -24,18 +25,21 @@ VALID_MANIFEST = """ providers: - - Jenkins - - Gitlab - - Github Actions + - displayName: Jenkins + id: jenkins + - displayName: Gitlab CI/CD + id: gitlab + - displayName: Github Actions + id: github-actions templates: - - name: jenkins-two-stages-pipeline - provider: Jenkins + - displayName: jenkins-two-stages-pipeline + provider: jenkins location: templates/cookiecutter-jenkins-two-stages-pipeline - - name: gitlab-two-stages-pipeline - provider: Gitlab + - displayName: gitlab-two-stages-pipeline + provider: gitlab location: templates/cookiecutter-gitlab-two-stages-pipeline - - name: Github-Actions-two-stages-pipeline - provider: Github Actions + - displayName: Github-Actions-two-stages-pipeline + provider: github-actions location: templates/cookiecutter-github-actions-two-stages-pipeline """ @@ -68,9 +72,11 @@ def test_manifest_happy_case(self): with open(manifest_path, "w", encoding="utf-8") as fp: fp.write(VALID_MANIFEST) manifest = PipelineTemplatesManifest(manifest_path=Path(manifest_path)) - self.assertEquals(manifest.providers, ["Jenkins", "Gitlab", "Github Actions"]) + self.assertEquals(len(manifest.providers), 3) + gitlab_provider: Provider = next(p for p in manifest.providers if p.id == "gitlab") + self.assertEquals(gitlab_provider.display_name, "Gitlab CI/CD") self.assertEquals(len(manifest.templates), 3) - gitlab_template: PipelineTemplateMetadata = next(t for t in manifest.templates if t.provider == "Gitlab") - self.assertEquals(gitlab_template.name, "gitlab-two-stages-pipeline") - self.assertEquals(gitlab_template.provider, "Gitlab") + gitlab_template: PipelineTemplateMetadata = next(t for t in manifest.templates if t.provider == "gitlab") + self.assertEquals(gitlab_template.display_name, "gitlab-two-stages-pipeline") + self.assertEquals(gitlab_template.provider, "gitlab") self.assertEquals(gitlab_template.location, "templates/cookiecutter-gitlab-two-stages-pipeline") From d76eba50dfa81522dc1178c268b8f11de5752e12 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Thu, 29 Apr 2021 15:24:22 -0700 Subject: [PATCH 4/5] display a message that we have successfully created the pipeline configuration file(s). --- samcli/commands/pipeline/init/interactive_init_flow.py | 1 + 1 file changed, 1 insertion(+) diff --git a/samcli/commands/pipeline/init/interactive_init_flow.py b/samcli/commands/pipeline/init/interactive_init_flow.py index 88c64e81b6..b4083cffb0 100644 --- a/samcli/commands/pipeline/init/interactive_init_flow.py +++ b/samcli/commands/pipeline/init/interactive_init_flow.py @@ -43,6 +43,7 @@ def do_interactive() -> None: _generate_from_custom_location() else: _generate_from_app_pipeline_templates() + click.echo("Successfully created the pipeline configuration file(s)") def _generate_from_app_pipeline_templates() -> None: From 9045fe01b6b6ac820612db2d2eab2bf2be6126b3 Mon Sep 17 00:00:00 2001 From: Ahmed Elbayaa Date: Thu, 29 Apr 2021 16:06:55 -0700 Subject: [PATCH 5/5] doc typo --- .../pipeline/init/pipeline_templates_manifest.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/samcli/commands/pipeline/init/pipeline_templates_manifest.py b/samcli/commands/pipeline/init/pipeline_templates_manifest.py index 815263f5f7..b24d2f2a7d 100644 --- a/samcli/commands/pipeline/init/pipeline_templates_manifest.py +++ b/samcli/commands/pipeline/init/pipeline_templates_manifest.py @@ -2,17 +2,20 @@ Represents a manifest that lists the available SAM pipeline templates. Example: providers: - - Jenkins - - Gitlab - - Github Actions + - displayName:Jenkins + id: jenkins + - displayName:Gitlab CI/CD + id: gitlab + - displayName:Github Actions + id: github-actions templates: - - name: jenkins-two-stages-pipeline + - displayName: jenkins-two-stages-pipeline provider: Jenkins location: templates/cookiecutter-jenkins-two-stages-pipeline - - name: gitlab-two-stages-pipeline + - displayName: gitlab-two-stages-pipeline provider: Gitlab location: templates/cookiecutter-gitlab-two-stages-pipeline - - name: Github-Actions-two-stages-pipeline + - displayName: Github-Actions-two-stages-pipeline provider: Github Actions location: templates/cookiecutter-github-actions-two-stages-pipeline """