sam pipeline init command #2831
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| """ | ||
| CLI command for "pipeline init" command | ||
| """ | ||
| from typing import Any, Optional | ||
|
|
||
| import click | ||
|
|
||
| from samcli.cli.cli_config_file import configuration_option, TomlProvider | ||
| from samcli.cli.main import pass_context, common_options as cli_framework_options | ||
| from samcli.commands.pipeline.init.interactive_init_flow import do_interactive | ||
| from samcli.lib.telemetry.metric import track_command | ||
|
|
||
| SHORT_HELP = "Generates CI/CD pipeline configuration files." | ||
| HELP_TEXT = """ | ||
| Generates CI/CD pipeline configuration files for a chosen CI/CD provider such as Jenkins, | ||
| GitLab CI/CD or GitHub Actions | ||
| """ | ||
|
|
||
|
|
||
| @click.command("init", help=HELP_TEXT, short_help=SHORT_HELP) | ||
| @configuration_option(provider=TomlProvider(section="parameters")) | ||
| @cli_framework_options | ||
| @pass_context | ||
| @track_command # pylint: disable=R0914 | ||
| def cli( | ||
| ctx: Any, | ||
| config_env: Optional[str], | ||
| config_file: Optional[str], | ||
| ) -> None: | ||
| """ | ||
| `sam pipeline init` command entry point | ||
| """ | ||
|
|
||
| # Currently we support interactive mode only, i.e. the user doesn't provide the required arguments during the call | ||
| # so we call do_cli without any arguments. This will change after supporting the non interactive mode. | ||
| do_cli() | ||
|
|
||
|
|
||
| def do_cli() -> None: | ||
| """ | ||
| implementation of `sam pipeline init` command | ||
| """ | ||
| # TODO non-interactive mode | ||
| do_interactive() | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,250 @@ | ||||||
| """ | ||||||
| Interactive flow that prompts that users for pipeline template (cookiecutter template) and used it to generate | ||||||
| pipeline configuration file | ||||||
| """ | ||||||
| import logging | ||||||
| import os | ||||||
| from pathlib import Path | ||||||
| from typing import Dict, List | ||||||
|
|
||||||
| import click | ||||||
|
|
||||||
| from samcli.cli.main import global_cfg | ||||||
| from samcli.commands.exceptions import PipelineTemplateCloneException | ||||||
| from samcli.lib.cookiecutter.interactive_flow import InteractiveFlow | ||||||
| from samcli.lib.cookiecutter.interactive_flow_creator import InteractiveFlowCreator | ||||||
| from samcli.lib.cookiecutter.question import Choice | ||||||
| from samcli.lib.cookiecutter.template import Template | ||||||
| from samcli.lib.utils import osutils | ||||||
| from samcli.lib.utils.git_repo import GitRepo, CloneRepoException | ||||||
| from .pipeline_templates_manifest import Provider, PipelineTemplateMetadata, PipelineTemplatesManifest | ||||||
|
|
||||||
| LOG = logging.getLogger(__name__) | ||||||
| shared_path: Path = global_cfg.config_dir | ||||||
| APP_PIPELINE_TEMPLATES_REPO_URL = "https://github.com/aws/aws-sam-cli-pipeline-init-templates.git" | ||||||
| APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME = "aws-sam-cli-app-pipeline-templates" | ||||||
| CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME = "custom-pipeline-template" | ||||||
| SAM_PIPELINE_TEMPLATE_SOURCE = "AWS Quick Start Pipeline Templates" | ||||||
| CUSTOM_PIPELINE_TEMPLATE_SOURCE = "Custom Pipeline Template Location" | ||||||
|
|
||||||
|
|
||||||
| def do_interactive() -> None: | ||||||
| """ | ||||||
| An interactive flow that prompts the user for pipeline template (cookiecutter template) location, downloads it, | ||||||
| runs its specific questionnaire then generates the pipeline config file based on the template and user's responses | ||||||
| """ | ||||||
| pipeline_template_source_question = Choice( | ||||||
| key="pipeline-template-source", | ||||||
| text="Which pipeline template source would you like to use?", | ||||||
| options=[SAM_PIPELINE_TEMPLATE_SOURCE, CUSTOM_PIPELINE_TEMPLATE_SOURCE], | ||||||
| ) | ||||||
| source = pipeline_template_source_question.ask() | ||||||
| if source == CUSTOM_PIPELINE_TEMPLATE_SOURCE: | ||||||
| _generate_from_custom_location() | ||||||
| else: | ||||||
| _generate_from_app_pipeline_templates() | ||||||
|
Comment on lines
+43
to
+45
Contributor
There was a problem hiding this comment. Why are they treated differently? They both clone from an URI. Edit: in reference to their handling overall (not these lines specifically), e.g. storing in separate directories. |
||||||
| click.echo("Successfully created the pipeline configuration file(s)") | ||||||
|
Contributor
There was a problem hiding this comment.
Suggested change
|
||||||
|
|
||||||
|
|
||||||
| def _generate_from_app_pipeline_templates() -> None: | ||||||
| """ | ||||||
| Prompts the user to choose a pipeline template from SAM predefined set of pipeline templates hosted in the git | ||||||
| repository: aws/aws-sam-cli-pipeline-init-templates.git | ||||||
| downloads locally, then generates the pipeline config file from the selected pipeline template. | ||||||
| """ | ||||||
| pipeline_templates_local_dir: Path = _clone_app_pipeline_templates() | ||||||
|
Contributor
There was a problem hiding this comment. nit: no need for type hints if they're in the function definition (in fact might be better, to avoid accidental casting) |
||||||
| pipeline_templates_manifest: PipelineTemplatesManifest = _read_app_pipeline_templates_manifest( | ||||||
| pipeline_templates_local_dir | ||||||
| ) | ||||||
| # The manifest contains multiple pipeline-templates so select one | ||||||
|
Contributor
There was a problem hiding this comment.
Suggested change
|
||||||
| selected_pipeline_template_metadata: PipelineTemplateMetadata = _prompt_pipeline_template( | ||||||
| pipeline_templates_manifest | ||||||
| ) | ||||||
| selected_pipeline_template_dir: Path = pipeline_templates_local_dir.joinpath( | ||||||
| selected_pipeline_template_metadata.location | ||||||
| ) | ||||||
| _generate_from_pipeline_template(selected_pipeline_template_dir) | ||||||
|
|
||||||
|
|
||||||
| def _generate_from_custom_location() -> None: | ||||||
| """ | ||||||
| Prompts the user for a custom pipeline template location, downloads locally, then generates the pipeline config file | ||||||
| """ | ||||||
| pipeline_template_git_location: str = click.prompt("Template Git location") | ||||||
| if os.path.exists(pipeline_template_git_location): | ||||||
| _generate_from_pipeline_template(Path(pipeline_template_git_location)) | ||||||
|
Comment on lines
+74
to
+75
Contributor
There was a problem hiding this comment. A comment here can help understand why we check a git location exists or not, is to check whether it is a local path right? |
||||||
| else: | ||||||
| with osutils.mkdir_temp(ignore_errors=True) as tempdir: | ||||||
| tempdir_path = Path(tempdir) | ||||||
| pipeline_template_local_dir: Path = _clone_pipeline_templates( | ||||||
| pipeline_template_git_location, tempdir_path, CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME | ||||||
| ) | ||||||
| _generate_from_pipeline_template(pipeline_template_local_dir) | ||||||
|
|
||||||
|
|
||||||
| def _generate_from_pipeline_template(pipeline_template_dir: Path) -> None: | ||||||
| """ | ||||||
| Generates a pipeline config file from a given pipeline template local location | ||||||
| """ | ||||||
| pipeline_template: Template = _initialize_pipeline_template(pipeline_template_dir) | ||||||
| context: Dict = pipeline_template.run_interactive_flows() | ||||||
| pipeline_template.generate_project(context) | ||||||
|
|
||||||
|
|
||||||
| def _clone_app_pipeline_templates() -> Path: | ||||||
| """ | ||||||
| clone aws/aws-sam-cli-pipeline-init-templates.git Git repo to the local machine in SAM shared directory. | ||||||
| Returns: | ||||||
| the local directory path where the repo is cloned. | ||||||
| """ | ||||||
| try: | ||||||
| return _clone_pipeline_templates( | ||||||
| repo_url=APP_PIPELINE_TEMPLATES_REPO_URL, | ||||||
| clone_dir=shared_path, | ||||||
| clone_name=APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME, | ||||||
| ) | ||||||
| except PipelineTemplateCloneException: | ||||||
| # If can't clone app pipeline templates, try using an old clone from a previous run if already exist | ||||||
hoffa marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
| expected_previous_clone_local_path: Path = shared_path.joinpath(APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME) | ||||||
| if expected_previous_clone_local_path.exists(): | ||||||
| click.echo("Unable to download updated app pipeline templates, using existing ones") | ||||||
| return expected_previous_clone_local_path | ||||||
| raise | ||||||
|
|
||||||
|
|
||||||
| def _clone_pipeline_templates(repo_url: str, clone_dir: Path, clone_name: str) -> Path: | ||||||
| """ | ||||||
| clone a given pipeline templates' Git repo to the user machine inside the given clone_dir directory | ||||||
| under the given clone name. For example, if clone_name is "custom-pipeline-template" then the location to clone | ||||||
| to is "/clone/dir/path/custom-pipeline-template/" | ||||||
|
|
||||||
| Parameters: | ||||||
| repo_url: the URL of the Git repo to clone | ||||||
| clone_dir: the local parent directory to clone to | ||||||
| clone_name: The folder name to give to the created clone inside clone_dir | ||||||
|
|
||||||
| Returns: | ||||||
| Path to the local clone | ||||||
| """ | ||||||
| try: | ||||||
| repo: GitRepo = GitRepo(repo_url) | ||||||
| clone_path: Path = repo.clone(clone_dir, clone_name, replace_existing=True) | ||||||
| return clone_path | ||||||
| except (OSError, CloneRepoException) as ex: | ||||||
| raise PipelineTemplateCloneException(str(ex)) from ex | ||||||
|
|
||||||
|
|
||||||
| def _read_app_pipeline_templates_manifest(pipeline_templates_dir: Path) -> PipelineTemplatesManifest: | ||||||
| """ | ||||||
| parse and return the manifest yaml file located in the root directory of the SAM pipeline templates folder: | ||||||
|
|
||||||
| Parameters: | ||||||
| pipeline_templates_dir: local directory of SAM pipeline templates | ||||||
|
|
||||||
| Raises: | ||||||
| AppPipelineTemplateManifestException if the manifest is not found, ill-formatted or missing required keys | ||||||
|
|
||||||
| Returns: | ||||||
| The manifest of the pipeline templates | ||||||
| """ | ||||||
| manifest_path: Path = pipeline_templates_dir.joinpath("manifest.yaml") | ||||||
| return PipelineTemplatesManifest(manifest_path) | ||||||
|
|
||||||
|
|
||||||
| def _prompt_pipeline_template(pipeline_templates_manifest: PipelineTemplatesManifest) -> PipelineTemplateMetadata: | ||||||
| """ | ||||||
| Prompts the user a list of the available CI/CD providers along with associated app pipeline templates to choose | ||||||
| one of them | ||||||
|
|
||||||
| Parameters: | ||||||
| pipeline_templates_manifest: A manifest file lists the available providers and the associated pipeline templates | ||||||
|
|
||||||
| Returns: | ||||||
| The manifest (A section in the pipeline_templates_manifest) of the chosen pipeline template; | ||||||
| """ | ||||||
| provider = _prompt_cicd_provider(pipeline_templates_manifest.providers) | ||||||
| provider_pipeline_templates: List[PipelineTemplateMetadata] = [ | ||||||
| t for t in pipeline_templates_manifest.templates if t.provider == provider.id | ||||||
| ] | ||||||
| selected_template_manifest: PipelineTemplateMetadata = _prompt_provider_pipeline_template( | ||||||
| provider_pipeline_templates | ||||||
| ) | ||||||
| return selected_template_manifest | ||||||
|
|
||||||
|
|
||||||
| def _prompt_cicd_provider(available_providers: List[Provider]) -> Provider: | ||||||
| """ | ||||||
| Prompts the user a list of the available CI/CD providers to choose from | ||||||
|
|
||||||
| Parameters: | ||||||
| available_providers: List of available CI/CD providers such as Jenkins, Gitlab and CircleCI | ||||||
|
|
||||||
| Returns: | ||||||
| The chosen provider | ||||||
| """ | ||||||
| question_to_choose_provider = Choice( | ||||||
| key="provider", | ||||||
| text="CI/CD provider", | ||||||
| options=[p.display_name for p in available_providers], | ||||||
| ) | ||||||
| chosen_provider_display_name = question_to_choose_provider.ask() | ||||||
| return next(p for p in available_providers if p.display_name == chosen_provider_display_name) | ||||||
|
|
||||||
|
|
||||||
| def _prompt_provider_pipeline_template( | ||||||
| provider_available_pipeline_templates_metadata: List[PipelineTemplateMetadata], | ||||||
| ) -> PipelineTemplateMetadata: | ||||||
| """ | ||||||
| Prompts the user a list of the available pipeline templates to choose from | ||||||
|
|
||||||
| Parameters: | ||||||
| provider_available_pipeline_templates_metadata: List of available pipeline templates manifests | ||||||
|
|
||||||
| Returns: | ||||||
| The chosen pipeline template manifest | ||||||
| """ | ||||||
| question_to_choose_pipeline_template = Choice( | ||||||
| key="pipeline-template", | ||||||
| text="Which pipeline template would you like to use?", | ||||||
| options=[t.display_name for t in provider_available_pipeline_templates_metadata], | ||||||
| ) | ||||||
| chosen_pipeline_template_display_name = question_to_choose_pipeline_template.ask() | ||||||
| return next( | ||||||
| t | ||||||
| for t in provider_available_pipeline_templates_metadata | ||||||
| if t.display_name == chosen_pipeline_template_display_name | ||||||
| ) | ||||||
|
|
||||||
|
|
||||||
| def _initialize_pipeline_template(pipeline_template_dir: Path) -> Template: | ||||||
| """ | ||||||
| Initialize a pipeline template from a given pipeline template (cookiecutter template) location | ||||||
|
|
||||||
| Parameters: | ||||||
| pipeline_template_dir: The local location of the pipeline cookiecutter template | ||||||
|
|
||||||
| Returns: | ||||||
| The initialized pipeline's cookiecutter template | ||||||
| """ | ||||||
| interactive_flow = _get_pipeline_template_interactive_flow(pipeline_template_dir) | ||||||
| return Template(location=str(pipeline_template_dir), interactive_flows=[interactive_flow]) | ||||||
|
|
||||||
|
|
||||||
| def _get_pipeline_template_interactive_flow(pipeline_template_dir: Path) -> InteractiveFlow: | ||||||
| """ | ||||||
| A pipeline template defines its own interactive flow (questionnaire) in a JSON file named questions.json located | ||||||
| in the root directory of the template. This questionnaire defines a set of questions to prompt to the user and | ||||||
| use the responses as the cookiecutter context | ||||||
|
|
||||||
| Parameters: | ||||||
| pipeline_template_dir: The local location of the pipeline cookiecutter template | ||||||
|
|
||||||
| Raises: | ||||||
| QuestionsNotFoundException: if the pipeline template is missing questions.json file. | ||||||
| QuestionsFailedParsingException: if questions.json file is ill-formatted or missing required keys. | ||||||
|
|
||||||
| Returns: | ||||||
| The interactive flow | ||||||
| """ | ||||||
| flow_definition_path: Path = pipeline_template_dir.joinpath("questions.json") | ||||||
| return InteractiveFlowCreator.create_flow(str(flow_definition_path)) | ||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| """ | ||
| Represents a manifest that lists the available SAM pipeline templates. | ||
| Example: | ||
| providers: | ||
|
Contributor
There was a problem hiding this comment.
Contributor
Author
There was a problem hiding this comment. We can extract the unique providers from the templates metadata, but defining them separately comes with two benefits:
Contributor
There was a problem hiding this comment. For (2),
Contributor
Author
There was a problem hiding this comment. it depends on how we order the templates. Let's say we order them first by number of stages (single-stage-pipelines, two-stages-pipelines...) then by provider: Jenkins, Gitlab, GithubActions For example, Initially we may have: which gave
Contributor
There was a problem hiding this comment. Okay I see, so it's essentially to ensure prompt ordering. Is |
||
| - displayName:Jenkins | ||
| id: jenkins | ||
| - displayName:Gitlab CI/CD | ||
| id: gitlab | ||
| - displayName:Github Actions | ||
| id: github-actions | ||
| templates: | ||
| - displayName: jenkins-two-stages-pipeline | ||
| provider: Jenkins | ||
| location: templates/cookiecutter-jenkins-two-stages-pipeline | ||
| - displayName: gitlab-two-stages-pipeline | ||
| provider: Gitlab | ||
| location: templates/cookiecutter-gitlab-two-stages-pipeline | ||
| - displayName: Github-Actions-two-stages-pipeline | ||
| provider: Github Actions | ||
| location: templates/cookiecutter-github-actions-two-stages-pipeline | ||
| """ | ||
| from pathlib import Path | ||
| from typing import Dict, List | ||
|
|
||
| import yaml | ||
|
|
||
| from samcli.commands.exceptions import AppPipelineTemplateManifestException | ||
| from samcli.yamlhelper import parse_yaml_file | ||
|
|
||
|
|
||
| class Provider: | ||
| """ CI/CD provider such as Jenkins, Gitlab and GitHub-Actions""" | ||
|
|
||
| def __init__(self, manifest: Dict) -> None: | ||
| self.id: str = manifest["id"] | ||
| self.display_name: str = manifest["displayName"] | ||
|
|
||
|
|
||
| class PipelineTemplateMetadata: | ||
| """ The metadata of a Given pipeline template""" | ||
elbayaaa marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| def __init__(self, manifest: Dict) -> None: | ||
| self.display_name: str = manifest["displayName"] | ||
| self.provider: str = manifest["provider"] | ||
| self.location: str = manifest["location"] | ||
|
|
||
|
|
||
| class PipelineTemplatesManifest: | ||
| """ The metadata of the available CI/CD providers and the pipeline templates""" | ||
|
|
||
| def __init__(self, manifest_path: Path) -> None: | ||
| try: | ||
| manifest: Dict = parse_yaml_file(file_path=str(manifest_path)) | ||
| self.providers: List[Provider] = list(map(Provider, manifest["providers"])) | ||
| self.templates: List[PipelineTemplateMetadata] = list(map(PipelineTemplateMetadata, manifest["templates"])) | ||
| except (FileNotFoundError, KeyError, TypeError, yaml.YAMLError) as ex: | ||
| raise AppPipelineTemplateManifestException( | ||
| "SAM pipeline templates manifest file is not found or ill-formatted. This could happen if the file " | ||
| f"{manifest_path} got deleted or modified." | ||
| "If you believe this is not the case, please file an issue at https://github.com/aws/aws-sam-cli/issues" | ||
| ) from ex | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.