Skip to content

feat: Support creating pipeline files directly into . without hooks #2911

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aahung merged 9 commits into from
May 28, 2021
Merged

feat: Support creating pipeline files directly into . without hooks #2911

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bd2393b
feat: Support creating pipeline files directly into . without hooks
aahung May 28, 2021
d5c1f20
Use osutils to make temp dir
aahung May 28, 2021
7317ca6
Rename _copy_dir_contents_fail_on_exist to _copy_dir_contents_to_cwd_…
aahung May 28, 2021
5363b31
Print out the list of generated files
aahung May 28, 2021
bf916f6
Fix tests
aahung May 28, 2021
d19f753
Print success message and file list in green
aahung May 28, 2021
4494b2e
Use tab to indentation
aahung May 28, 2021
931bd41
Fix a pylint issue
aahung May 28, 2021
d6b56ab
Remove duplicate imports of osutils
aahung May 28, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 48 additions & 16 deletions samcli/commands/pipeline/init/interactive_init_flow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from samcli.lib.cookiecutter.question import Choice
from samcli.lib.cookiecutter.template import Template
from samcli.lib.utils import osutils
from samcli.lib.utils.colors import Colored
from samcli.lib.utils.git_repo import GitRepo, CloneRepoException
from .pipeline_templates_manifest import Provider, PipelineTemplateMetadata, PipelineTemplatesManifest
from ..bootstrap.cli import PIPELINE_CONFIG_DIR, PIPELINE_CONFIG_FILENAME
Expand All @@ -42,17 +43,20 @@ def do_interactive() -> None:
)
source = pipeline_template_source_question.ask()
if source == CUSTOM_PIPELINE_TEMPLATE_SOURCE:
_generate_from_custom_location()
generated_files = _generate_from_custom_location()
else:
_generate_from_app_pipeline_templates()
click.echo("Successfully created the pipeline configuration file(s)")
generated_files = _generate_from_app_pipeline_templates()
click.secho(Colored().green("Successfully created the pipeline configuration file(s):"))
for file in generated_files:
click.secho(Colored().green(f"\t- {file}"))


def _generate_from_app_pipeline_templates() -> None:
def _generate_from_app_pipeline_templates() -> List[str]:
"""
Prompts the user to choose a pipeline template from SAM predefined set of pipeline templates hosted in the git
repository: aws/aws-sam-cli-pipeline-init-templates.git
downloads locally, then generates the pipeline config file from the selected pipeline template.
Finally, return the list of generated files.
"""
pipeline_templates_local_dir: Path = _clone_app_pipeline_templates()
pipeline_templates_manifest: PipelineTemplatesManifest = _read_app_pipeline_templates_manifest(
Expand All @@ -65,23 +69,24 @@ def _generate_from_app_pipeline_templates() -> None:
selected_pipeline_template_dir: Path = pipeline_templates_local_dir.joinpath(
selected_pipeline_template_metadata.location
)
_generate_from_pipeline_template(selected_pipeline_template_dir)
return _generate_from_pipeline_template(selected_pipeline_template_dir)


def _generate_from_custom_location() -> None:
def _generate_from_custom_location() -> List[str]:
"""
Prompts the user for a custom pipeline template location, downloads locally, then generates the pipeline config file
and return the list of generated files
"""
pipeline_template_git_location: str = click.prompt("Template Git location")
if os.path.exists(pipeline_template_git_location):
_generate_from_pipeline_template(Path(pipeline_template_git_location))
else:
with osutils.mkdir_temp(ignore_errors=True) as tempdir:
tempdir_path = Path(tempdir)
pipeline_template_local_dir: Path = _clone_pipeline_templates(
pipeline_template_git_location, tempdir_path, CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME
)
_generate_from_pipeline_template(pipeline_template_local_dir)
return _generate_from_pipeline_template(Path(pipeline_template_git_location))

with osutils.mkdir_temp(ignore_errors=True) as tempdir:
tempdir_path = Path(tempdir)
pipeline_template_local_dir: Path = _clone_pipeline_templates(
pipeline_template_git_location, tempdir_path, CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME
)
return _generate_from_pipeline_template(pipeline_template_local_dir)


def _load_pipeline_bootstrap_context() -> Dict:
Expand All @@ -100,14 +105,34 @@ def _load_pipeline_bootstrap_context() -> Dict:
return context


def _generate_from_pipeline_template(pipeline_template_dir: Path) -> None:
def _generate_from_pipeline_template(pipeline_template_dir: Path) -> List[str]:
"""
Generates a pipeline config file from a given pipeline template local location
and return the list of generated files.
"""
pipeline_template: Template = _initialize_pipeline_template(pipeline_template_dir)
bootstrap_context: Dict = _load_pipeline_bootstrap_context()
context: Dict = pipeline_template.run_interactive_flows(bootstrap_context)
pipeline_template.generate_project(context)
with osutils.mkdir_temp() as generate_dir:
LOG.debug("Generating pipeline files into %s", generate_dir)
context["outputDir"] = "." # prevent cookiecutter from generating a sub-folder
pipeline_template.generate_project(context, generate_dir)
return _copy_dir_contents_to_cwd_fail_on_exist(generate_dir)


def _copy_dir_contents_to_cwd_fail_on_exist(source_dir: str) -> List[str]:
copied_file_paths: List[str] = []
for root, _, files in os.walk(source_dir):
for filename in files:
file_path = Path(root, filename)
target_file_path = Path(".").joinpath(file_path.relative_to(source_dir))
LOG.debug("Verify %s does not exist", target_file_path)
if target_file_path.exists():
raise PipelineFileAlreadyExistsError(target_file_path)
copied_file_paths.append(str(target_file_path))
LOG.debug("Copy contents of %s to cwd", source_dir)
osutils.copytree(source_dir, ".")
return copied_file_paths


def _clone_app_pipeline_templates() -> Path:
Expand Down Expand Up @@ -267,3 +292,10 @@ def _get_pipeline_template_interactive_flow(pipeline_template_dir: Path) -> Inte
"""
flow_definition_path: Path = pipeline_template_dir.joinpath("questions.json")
return InteractiveFlowCreator.create_flow(str(flow_definition_path))


class PipelineFileAlreadyExistsError(Exception):
def __init__(self, file_path: os.PathLike) -> None:
Exception.__init__(
self, f'Pipeline file "{file_path}" already exists in project root directory, please remove it first.'
)
12 changes: 10 additions & 2 deletions samcli/lib/cookiecutter/template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def run_interactive_flows(self, context: Optional[Dict] = None) -> Dict:
except Exception as e:
raise UserException(str(e), wrapped_from=e.__class__.__name__) from e

def generate_project(self, context: Dict) -> None:
def generate_project(self, context: Dict, output_dir: str) -> None:
"""
Generates a project based on this cookiecutter template and the given context. The context is first
processed and manipulated by series of preprocessors(if any) then the project is generated and finally
Expand All @@ -131,6 +131,8 @@ def generate_project(self, context: Dict) -> None:
----------
context: Dict
the cookiecutter context to fulfill the values of cookiecutter.json keys
output_dir: str
the directory where project will be generated in

Raise:
------
Expand All @@ -146,7 +148,13 @@ def generate_project(self, context: Dict) -> None:

try:
LOG.debug("Baking a new template with cookiecutter with all parameters")
cookiecutter(template=self._location, output_dir=".", no_input=True, extra_context=context)
cookiecutter(
template=self._location,
output_dir=output_dir,
no_input=True,
extra_context=context,
overwrite_if_exists=True,
)
except RepositoryNotFound as e:
# cookiecutter.json is not found in the template. Let's just clone it directly without
# using cookiecutter and call it done.
Expand Down
26 changes: 20 additions & 6 deletions tests/unit/commands/pipeline/init/test_initeractive_init_flow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,18 +100,22 @@ def test_app_pipeline_templates_with_invalid_manifest(
do_interactive()

@patch("samcli.commands.pipeline.init.interactive_init_flow.SamConfig")
@patch("samcli.commands.pipeline.init.interactive_init_flow.osutils")
@patch("samcli.lib.cookiecutter.template.cookiecutter")
@patch("samcli.commands.pipeline.init.interactive_init_flow.InteractiveFlowCreator.create_flow")
@patch("samcli.commands.pipeline.init.interactive_init_flow.PipelineTemplatesManifest")
@patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone")
@patch("samcli.commands.pipeline.init.interactive_init_flow._copy_dir_contents_to_cwd_fail_on_exist")
@patch("samcli.lib.cookiecutter.question.click")
def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_case(
self,
click_mock,
_copy_dir_contents_to_cwd_fail_on_exist_mock,
clone_mock,
PipelineTemplatesManifest_mock,
create_interactive_flow_mock,
cookiecutter_mock,
osutils_mock,
samconfig_mock,
):
# setup
Expand All @@ -131,9 +135,11 @@ def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_c
templates=[jenkins_template_mock],
)
PipelineTemplatesManifest_mock.return_value = pipeline_templates_manifest_mock
cookiecutter_output_dir_mock = "/tmp/any/dir2"
osutils_mock.mkdir_temp.return_value.__enter__ = Mock(return_value=cookiecutter_output_dir_mock)
interactive_flow_mock = Mock()
create_interactive_flow_mock.return_value = interactive_flow_mock
cookiecutter_context_mock = Mock()
cookiecutter_context_mock = {"key": "value"}
interactive_flow_mock.run.return_value = cookiecutter_context_mock
config_file = Mock()
samconfig_mock.return_value = config_file
Expand All @@ -152,6 +158,7 @@ def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_c
do_interactive()

# verify
osutils_mock.mkdir_temp.assert_called() # cookiecutter project is generated to temp
expected_cookicutter_template_location = any_app_pipeline_templates_path.joinpath(jenkins_template_location)
clone_mock.assert_called_once_with(shared_path, APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME, replace_existing=True)
PipelineTemplatesManifest_mock.assert_called_once()
Expand All @@ -166,9 +173,10 @@ def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_c
)
cookiecutter_mock.assert_called_once_with(
template=str(expected_cookicutter_template_location),
output_dir=".",
output_dir=cookiecutter_output_dir_mock,
no_input=True,
extra_context=cookiecutter_context_mock,
overwrite_if_exists=True,
)

@patch("samcli.commands.pipeline.init.interactive_init_flow._read_app_pipeline_templates_manifest")
Expand Down Expand Up @@ -236,10 +244,12 @@ def test_generate_pipeline_configuration_file_from_custom_local_existing_path_wi
@patch("samcli.commands.pipeline.init.interactive_init_flow.InteractiveFlowCreator.create_flow")
@patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone")
@patch("samcli.commands.pipeline.init.interactive_init_flow.click")
@patch("samcli.commands.pipeline.init.interactive_init_flow._copy_dir_contents_to_cwd_fail_on_exist")
@patch("samcli.lib.cookiecutter.question.click")
def test_generate_pipeline_configuration_file_from_custom_remote_pipeline_template_happy_case(
self,
questions_click_mock,
_copy_dir_contents_to_cwd_fail_on_exist_mock,
init_click_mock,
clone_mock,
create_interactive_flow_mock,
Expand All @@ -248,14 +258,16 @@ def test_generate_pipeline_configuration_file_from_custom_remote_pipeline_templa
):
# setup
any_temp_dir = "/tmp/any/dir"
osutils_mock.mkdir_temp.return_value.__enter__ = Mock(return_value=any_temp_dir)
cookiecutter_output_dir_mock = "/tmp/any/dir2"
osutils_mock.mkdir_temp.return_value.__enter__ = Mock(side_effect=[any_temp_dir, cookiecutter_output_dir_mock])
osutils_mock.mkdir_temp.return_value.__exit__ = Mock()
any_custom_pipeline_templates_path = Path(os.path.join(any_temp_dir, CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME))
clone_mock.return_value = any_custom_pipeline_templates_path
interactive_flow_mock = Mock()
create_interactive_flow_mock.return_value = interactive_flow_mock
cookiecutter_context_mock = Mock()
cookiecutter_context_mock = {"key": "value"}
interactive_flow_mock.run.return_value = cookiecutter_context_mock
_copy_dir_contents_to_cwd_fail_on_exist_mock.return_value = ["file1"]

questions_click_mock.prompt.return_value = "2" # Custom pipeline templates
init_click_mock.prompt.return_value = "https://github.com/any-custom-pipeline-template-repo.git"
Expand All @@ -264,7 +276,8 @@ def test_generate_pipeline_configuration_file_from_custom_remote_pipeline_templa
do_interactive()

# verify
osutils_mock.mkdir_temp.assert_called_once() # Custom templates are cloned to temp
# Custom templates are cloned to temp; cookiecutter project is generated to temp
osutils_mock.mkdir_temp.assert_called()
clone_mock.assert_called_once_with(
Path(any_temp_dir), CUSTOM_PIPELINE_TEMPLATE_REPO_LOCAL_NAME, replace_existing=True
)
Expand All @@ -274,7 +287,8 @@ def test_generate_pipeline_configuration_file_from_custom_remote_pipeline_templa
interactive_flow_mock.run.assert_called_once()
cookiecutter_mock.assert_called_once_with(
template=str(any_custom_pipeline_templates_path),
output_dir=".",
output_dir=cookiecutter_output_dir_mock,
no_input=True,
extra_context=cookiecutter_context_mock,
overwrite_if_exists=True,
)
19 changes: 12 additions & 7 deletions tests/unit/lib/cookiecutter/test_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,16 @@ def test_generate_project(self, mock_preprocessor, mock_postprocessor, mock_inte
postprocessors=[mock_postprocessor],
)
mock_preprocessor.run.return_value = self._ANY_PROCESSOR_CONTEXT
t.generate_project(context=self._ANY_INTERACTIVE_FLOW_CONTEXT)
output_dir = Mock()
t.generate_project(context=self._ANY_INTERACTIVE_FLOW_CONTEXT, output_dir=output_dir)
mock_interactive_flow.run.assert_not_called()
mock_preprocessor.run.assert_called_once_with(self._ANY_INTERACTIVE_FLOW_CONTEXT)
mock_cookiecutter.assert_called_with(
template=self._ANY_LOCATION, output_dir=".", no_input=True, extra_context=self._ANY_PROCESSOR_CONTEXT
template=self._ANY_LOCATION,
output_dir=output_dir,
no_input=True,
extra_context=self._ANY_PROCESSOR_CONTEXT,
overwrite_if_exists=True,
)
mock_postprocessor.run.assert_called_once_with(self._ANY_PROCESSOR_CONTEXT)

Expand All @@ -127,29 +132,29 @@ def test_generate_project_preprocessors_exceptions(self, mock_preprocessor):
t = Template(location=self._ANY_LOCATION, preprocessors=[mock_preprocessor])
with self.assertRaises(PreprocessingError):
mock_preprocessor.run.side_effect = Exception("something went wrong")
t.generate_project({})
t.generate_project({}, Mock())

@patch("samcli.lib.cookiecutter.template.cookiecutter")
@patch("samcli.lib.cookiecutter.processor")
def test_generate_project_postprocessors_exceptions(self, mock_postprocessor, mock_cookiecutter):
t = Template(location=self._ANY_LOCATION, postprocessors=[mock_postprocessor])
with self.assertRaises(PostprocessingError):
mock_postprocessor.run.side_effect = Exception("something went wrong")
t.generate_project({})
t.generate_project({}, Mock())

@patch("samcli.lib.cookiecutter.template.generate_non_cookiecutter_project")
@patch("samcli.lib.cookiecutter.template.cookiecutter")
def test_generate_project_cookiecutter_exceptions(self, mock_cookiecutter, mock_generate_non_cookiecutter_project):
t = Template(location=self._ANY_LOCATION)
with self.assertRaises(InvalidLocationError):
mock_cookiecutter.side_effect = UnknownRepoType()
t.generate_project({})
t.generate_project({}, Mock())
mock_cookiecutter.reset_mock()
with self.assertRaises(GenerateProjectFailedError):
mock_cookiecutter.side_effect = Exception("something went wrong")
t.generate_project({})
t.generate_project({}, Mock())
mock_cookiecutter.reset_mock()
# if the provided template is not a cookiecutter template, we generate a non cookiecutter template
mock_cookiecutter.side_effect = RepositoryNotFound()
t.generate_project({})
t.generate_project({}, Mock())
mock_generate_non_cookiecutter_project.assert_called_once()