Skip to content

Let 'sam pipeline init' prefills pipeline's infrastructure resources… #2894

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
elbayaaa merged 4 commits into from
May 20, 2021
Merged

Let 'sam pipeline init' prefills pipeline's infrastructure resources… #2894

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e4853b3
Let 'sam pipeline init' prefills pipeline's infrastructure resources…
elbayaaa May 20, 2021
0014b95
save bootstrapped sateg region
elbayaaa May 20, 2021
69b018e
make black happy
elbayaaa May 20, 2021
7b87dea
exclude non-dict keys from samconfig.get_env_names method.
elbayaaa May 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion samcli/commands/pipeline/init/interactive_init_flow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@

from samcli.cli.main import global_cfg
from samcli.commands.exceptions import PipelineTemplateCloneException
from samcli.lib.config.samconfig import SamConfig
from samcli.lib.cookiecutter.interactive_flow import InteractiveFlow
from samcli.lib.cookiecutter.interactive_flow_creator import InteractiveFlowCreator
from samcli.lib.cookiecutter.question import Choice
from samcli.lib.cookiecutter.template import Template
from samcli.lib.utils import osutils
from samcli.lib.utils.git_repo import GitRepo, CloneRepoException
from .pipeline_templates_manifest import Provider, PipelineTemplateMetadata, PipelineTemplatesManifest
from ..bootstrap.cli import PIPELINE_CONFIG_DIR, PIPELINE_CONFIG_FILENAME

LOG = logging.getLogger(__name__)
shared_path: Path = global_cfg.config_dir
Expand Down Expand Up @@ -82,12 +84,29 @@ def _generate_from_custom_location() -> None:
_generate_from_pipeline_template(pipeline_template_local_dir)


def _load_pipeline_bootstrap_context() -> Dict:
bootstrap_command_names = ["pipeline", "bootstrap"]
section = "parameters"
context: Dict = {}

config = SamConfig(PIPELINE_CONFIG_DIR, PIPELINE_CONFIG_FILENAME)
if not config.exists():
return context

for env in config.get_env_names():
for key, value in config.get_all(bootstrap_command_names, section, env).items():
context[str([env, key])] = value

return context


def _generate_from_pipeline_template(pipeline_template_dir: Path) -> None:
"""
Generates a pipeline config file from a given pipeline template local location
"""
pipeline_template: Template = _initialize_pipeline_template(pipeline_template_dir)
context: Dict = pipeline_template.run_interactive_flows()
bootstrap_context: Dict = _load_pipeline_bootstrap_context()
context: Dict = pipeline_template.run_interactive_flows(bootstrap_context)
pipeline_template.generate_project(context)


Expand Down
6 changes: 6 additions & 0 deletions samcli/lib/config/samconfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,12 @@ def __init__(self, config_dir, filename=None):
"""
self.filepath = Path(config_dir, filename or DEFAULT_CONFIG_FILE_NAME)

def get_env_names(self):
self._read()
if isinstance(self.document, dict):
return [env for env, value in self.document.items() if isinstance(value, dict)]
return []

def get_all(self, cmd_names, section, env=DEFAULT_ENV):
"""
Gets a value from the configuration file for the given environment, command and section
Expand Down
16 changes: 9 additions & 7 deletions samcli/lib/cookiecutter/question.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,8 @@ def text(self) -> str:
return self._text

@property
def default_answer(self) -> Optional[str]:
return self._default_answer
def default_answer(self) -> Optional[Any]:
return self._resolve_default_answer()

@property
def required(self) -> Optional[bool]:
Expand All @@ -90,7 +90,7 @@ def next_question_map(self) -> Dict[str, str]:
def default_next_question_key(self) -> Optional[str]:
return self._default_next_question_key

def ask(self, context: Dict) -> Any:
def ask(self, context: Optional[Dict] = None) -> Any:
"""
prompt the user this question

Expand Down Expand Up @@ -150,7 +150,7 @@ def _resolve_key_path(self, key_path: List, context: Dict) -> List[str]:
raise ValueError(f'Invalid value "{unresolved_key}" in key path')
return resolved_key_path

def _resolve_default_answer(self, context: Dict) -> Optional[Any]:
def _resolve_default_answer(self, context: Optional[Dict] = None) -> Optional[Any]:
"""
a question may have a default answer provided directly through the "default_answer" value
or indirectly from cookiecutter context using a key path
Expand All @@ -173,6 +173,8 @@ def _resolve_default_answer(self, context: Dict) -> Optional[Any]:

"""
if isinstance(self._default_answer, dict):
context = context if context else {}

# load value using key path from cookiecutter
if "keyPath" not in self._default_answer:
raise KeyError(f'Missing key "keyPath" in question default "{self._default_answer}".')
Expand All @@ -186,12 +188,12 @@ def _resolve_default_answer(self, context: Dict) -> Optional[Any]:


class Info(Question):
def ask(self, context: Dict) -> None:
def ask(self, context: Optional[Dict] = None) -> None:
return click.echo(message=self._text)


class Confirm(Question):
def ask(self, context: Dict) -> bool:
def ask(self, context: Optional[Dict] = None) -> bool:
return click.confirm(text=self._text)


Expand All @@ -211,7 +213,7 @@ def __init__(
self._options = options
super().__init__(key, text, default, is_required, next_question_map, default_next_question_key)

def ask(self, context: Dict) -> str:
def ask(self, context: Optional[Dict] = None) -> str:
resolved_default_answer = self._resolve_default_answer(context)
click.echo(self._text)
for index, option in enumerate(self._options):
Expand Down
10 changes: 6 additions & 4 deletions samcli/lib/cookiecutter/template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,17 @@
values of the context and how to generate a project from the given template and provided context
"""
import logging
from typing import Any, Dict, List, Optional
from typing import Dict, List, Optional

from cookiecutter.exceptions import RepositoryNotFound, UnknownRepoType
from cookiecutter.main import cookiecutter

from samcli.commands.exceptions import UserException
from samcli.lib.init.arbitrary_project import generate_non_cookiecutter_project
from .exceptions import GenerateProjectFailedError, InvalidLocationError, PreprocessingError, PostprocessingError
from .interactive_flow import InteractiveFlow
from .plugin import Plugin
from .processor import Processor
from .exceptions import GenerateProjectFailedError, InvalidLocationError, PreprocessingError, PostprocessingError

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -98,7 +100,7 @@ def __init__(
if plugin.postprocessor:
self._postprocessors.append(plugin.postprocessor)

def run_interactive_flows(self) -> Dict:
def run_interactive_flows(self, context: Optional[Dict] = None) -> Dict:
"""
prompt the user a series of questions' flows and gather the answers to create the cookiecutter context.
The questions are identified by keys. If multiple questions, whether within the same flow or across
Expand All @@ -112,7 +114,7 @@ def run_interactive_flows(self) -> Dict:
A Dictionary in the form of {question.key: answer} representing user's answers to the flows' questions
"""
try:
context: Dict[str, Any] = {}
context = context if context else {}
for flow in self._interactive_flows:
context = flow.run(context)
return context
Expand Down
2 changes: 2 additions & 0 deletions samcli/lib/pipeline/bootstrap/stage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
CLOUDFORMATION_EXECUTION_ROLE = "cloudformation_execution_role"
ARTIFACTS_BUCKET = "artifacts_bucket"
ECR_REPO = "ecr_repo"
REGION = "region"


class Stage:
Expand Down Expand Up @@ -223,6 +224,7 @@ def save_config(self, config_dir: str, filename: str, cmd_names: List[str]) -> N
CLOUDFORMATION_EXECUTION_ROLE: self.cloudformation_execution_role.arn,
ARTIFACTS_BUCKET: artifacts_bucket_name,
ECR_REPO: ecr_repo_uri,
REGION: self.aws_region,
}

for key, value in stage_specific_configs.items():
Expand Down
23 changes: 18 additions & 5 deletions tests/unit/commands/pipeline/init/test_initeractive_init_flow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,18 +99,20 @@ def test_app_pipeline_templates_with_invalid_manifest(
with self.assertRaises(AppPipelineTemplateManifestException):
do_interactive()

@patch("samcli.commands.pipeline.init.interactive_init_flow.SamConfig")
@patch("samcli.lib.cookiecutter.template.cookiecutter")
@patch("samcli.commands.pipeline.init.interactive_init_flow.InteractiveFlowCreator.create_flow")
@patch("samcli.commands.pipeline.init.interactive_init_flow._read_app_pipeline_templates_manifest")
@patch("samcli.commands.pipeline.init.interactive_init_flow.PipelineTemplatesManifest")
@patch("samcli.commands.pipeline.init.interactive_init_flow.GitRepo.clone")
@patch("samcli.lib.cookiecutter.question.click")
def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_case(
self,
click_mock,
clone_mock,
read_app_pipeline_templates_manifest_mock,
PipelineTemplatesManifest_mock,
create_interactive_flow_mock,
cookiecutter_mock,
samconfig_mock,
):
# setup
any_app_pipeline_templates_path = Path(
Expand All @@ -128,11 +130,17 @@ def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_c
],
templates=[jenkins_template_mock],
)
read_app_pipeline_templates_manifest_mock.return_value = pipeline_templates_manifest_mock
PipelineTemplatesManifest_mock.return_value = pipeline_templates_manifest_mock
interactive_flow_mock = Mock()
create_interactive_flow_mock.return_value = interactive_flow_mock
cookiecutter_context_mock = Mock()
interactive_flow_mock.run.return_value = cookiecutter_context_mock
config_file = Mock()
samconfig_mock.return_value = config_file
config_file.exists.return_value = True
config_file.get_env_names.return_value = ["testing", "prod"]
config_file.get_env_names.return_value = ["testing", "prod"]
config_file.get_all.return_value = {"pipeline_execution_role": "arn:aws:iam::123456789012:role/execution-role"}

click_mock.prompt.side_effect = [
"1", # App pipeline templates
Expand All @@ -146,11 +154,16 @@ def test_generate_pipeline_configuration_file_from_app_pipeline_template_happy_c
# verify
expected_cookicutter_template_location = any_app_pipeline_templates_path.joinpath(jenkins_template_location)
clone_mock.assert_called_once_with(shared_path, APP_PIPELINE_TEMPLATES_REPO_LOCAL_NAME, replace_existing=True)
read_app_pipeline_templates_manifest_mock.assert_called_once_with(any_app_pipeline_templates_path)
PipelineTemplatesManifest_mock.assert_called_once()
create_interactive_flow_mock.assert_called_once_with(
str(expected_cookicutter_template_location.joinpath("questions.json"))
)
interactive_flow_mock.run.assert_called_once()
interactive_flow_mock.run.assert_called_once_with(
{
str(["testing", "pipeline_execution_role"]): "arn:aws:iam::123456789012:role/execution-role",
str(["prod", "pipeline_execution_role"]): "arn:aws:iam::123456789012:role/execution-role",
}
)
cookiecutter_mock.assert_called_once_with(
template=str(expected_cookicutter_template_location),
output_dir=".",
Expand Down
18 changes: 14 additions & 4 deletions tests/unit/lib/samconfig/test_samconfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
import os
import shutil
from pathlib import Path
from unittest import TestCase

from samcli.lib.config.exceptions import SamConfigVersionException
from samcli.lib.config.samconfig import SamConfig, DEFAULT_CONFIG_FILE_NAME, DEFAULT_GLOBAL_CMDNAME
from samcli.lib.config.samconfig import SamConfig, DEFAULT_CONFIG_FILE_NAME, DEFAULT_GLOBAL_CMDNAME, DEFAULT_ENV
from samcli.lib.config.version import VERSION_KEY, SAM_CONFIG_VERSION
from samcli.lib.utils import osutils

Expand All @@ -28,14 +27,25 @@ def _check_config_file(self):
self.assertTrue(self.samconfig.sanity_check())
self.assertEqual(SAM_CONFIG_VERSION, self.samconfig.document.get(VERSION_KEY))

def _update_samconfig(self, cmd_names, section, key, value, env):
self.samconfig.put(cmd_names=cmd_names, section=section, key=key, value=value, env=env)
def _update_samconfig(self, cmd_names, section, key, value, env=None):
if env:
self.samconfig.put(cmd_names=cmd_names, section=section, key=key, value=value, env=env)
else:
self.samconfig.put(cmd_names=cmd_names, section=section, key=key, value=value)
self.samconfig.flush()
self._check_config_file()

def test_init(self):
self.assertEqual(self.samconfig.filepath, Path(self.config_dir, DEFAULT_CONFIG_FILE_NAME))

def test_get_env_names(self):
self.assertEqual(self.samconfig.get_env_names(), [])
self._update_samconfig(cmd_names=["myCommand"], section="mySection", key="port", value=5401, env="env1")
self._update_samconfig(cmd_names=["myCommand"], section="mySection", key="port", value=5401, env="env2")
self.assertEqual(self.samconfig.get_env_names(), ["env1", "env2"])
self._update_samconfig(cmd_names=["myCommand"], section="mySection", key="port", value=5401)
self.assertEqual(self.samconfig.get_env_names(), ["env1", "env2", DEFAULT_ENV])

def test_param_overwrite(self):
self._update_samconfig(cmd_names=["myCommand"], section="mySection", key="port", value=5401, env="myEnv")
self.assertEqual(
Expand Down