Skip to content

Fixing bootstrap integ tests. #3061

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
c2tarun merged 5 commits into from
Jul 16, 2021
Merged

Fixing bootstrap integ tests. #3061

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
50c6f34
Fixing bootstrap integ tests.
Jul 15, 2021
3ca8f11
Cleaning up some integ tests.
Jul 15, 2021
fb08fa3
Using environment variables when running integ test on CI.
Jul 16, 2021
e5f79c2
Using expression instead of full loop.
Jul 16, 2021
d45682f
Adding instruction to use default profile on local.
Jul 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 29 additions & 5 deletions tests/integration/pipeline/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import os
import shutil
import logging
from pathlib import Path
from typing import List, Optional, Set, Tuple, Any
from unittest import TestCase
from unittest.mock import Mock

import boto3
import botocore.exceptions
from botocore.exceptions import ClientError

from samcli.lib.pipeline.bootstrap.stage import Stage
Expand Down Expand Up @@ -59,21 +61,39 @@ def setUpClass(cls):
def setUp(self):
self.stack_names = []
super().setUp()
shutil.rmtree(os.path.join(os.getcwd(), ".aws-sam", "pipeline"), ignore_errors=True)

def tearDown(self):
for stack_name in self.stack_names:
self._cleanup_s3_buckets(stack_name)
self.cf_client.delete_stack(StackName=stack_name)
shutil.rmtree(os.path.join(os.getcwd(), ".aws-sam", "pipeline"), ignore_errors=True)
super().tearDown()

def _cleanup_s3_buckets(self, stack_name):
try:
stack_resources = self.cf_client.describe_stack_resources(StackName=stack_name)
buckets = [
resource
for resource in stack_resources["StackResources"]
if resource["ResourceType"] == "AWS::S3::Bucket"
]
s3_client = boto3.client("s3")
for bucket in buckets:
s3_client.delete_bucket(Bucket=bucket.get("PhysicalResourceId"))
except botocore.exceptions.ClientError:
"""No need to fail in cleanup"""

def get_bootstrap_command_list(
self,
no_interactive: bool = False,
stage_name: Optional[str] = None,
profile_name: Optional[str] = None,
region: Optional[str] = None,
pipeline_user: Optional[str] = None,
pipeline_execution_role: Optional[str] = None,
cloudformation_execution_role: Optional[str] = None,
artifacts_bucket: Optional[str] = None,
bucket: Optional[str] = None,
create_image_repository: bool = False,
image_repository: Optional[str] = None,
no_confirm_changeset: bool = False,
Expand All @@ -84,14 +104,18 @@ def get_bootstrap_command_list(
command_list += ["--no-interactive"]
if stage_name:
command_list += ["--stage", stage_name]
if profile_name:
command_list += ["--profile", profile_name]
if region:
command_list += ["--region", region]
if pipeline_user:
command_list += ["--pipeline-user", pipeline_user]
if pipeline_execution_role:
command_list += ["--pipeline-execution-role", pipeline_execution_role]
if cloudformation_execution_role:
command_list += ["--cloudformation-execution-role", cloudformation_execution_role]
if artifacts_bucket:
command_list += ["--artifacts-bucket", artifacts_bucket]
if bucket:
command_list += ["--bucket", bucket]
if create_image_repository:
command_list += ["--create-image-repository"]
if image_repository:
Expand All @@ -101,9 +125,9 @@ def get_bootstrap_command_list(

return command_list

def _extract_created_resource_logical_ids(self, stack_name: str) -> Set[str]:
def _extract_created_resource_logical_ids(self, stack_name: str) -> List[str]:
response = self.cf_client.describe_stack_resources(StackName=stack_name)
return {resource["LogicalResourceId"] for resource in response["StackResources"]}
return [resource["LogicalResourceId"] for resource in response["StackResources"]]

def _stack_exists(self, stack_name) -> bool:
try:
Expand Down
81 changes: 51 additions & 30 deletions tests/integration/pipeline/test_bootstrap_command.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,41 +18,53 @@
# This is to restrict tests to run outside of CI/CD, when the branch is not master or tests are not run by Canary
SKIP_BOOTSTRAP_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY

# In order to run bootstrap integration test locally make sure your test account is configured as `default` account.
CREDENTIAL_PROFILE = "2" if not RUN_BY_CANARY else "1"


@skipIf(SKIP_BOOTSTRAP_TESTS, "Skip bootstrap tests in CI/CD only")
class TestBootstrap(BootstrapIntegBase):
@parameterized.expand([("create_image_repository",), (False,)])
def test_interactive_with_no_resources_provided(self, create_image_repository: bool):
def test_interactive_with_no_resources_provided(self, create_image_repository):
stage_name, stack_name = self._get_stage_and_stack_name()
self.stack_names = [stack_name]

bootstrap_command_list = self.get_bootstrap_command_list()

inputs = [
stage_name,
CREDENTIAL_PROFILE,
"us-east-1", # region
"", # pipeline user
"", # Pipeline execution role
"", # CloudFormation execution role
"", # Artifacts bucket
"2" if create_image_repository else "1", # Should we create ECR repo, 1 - No, 2 - Yes
"y", # proceed
"y" if create_image_repository else "N", # Should we create ECR repo
]

if create_image_repository:
inputs.append("") # Create image repository

inputs.append("") # Confirm summary
inputs.append("y") # Create resources

bootstrap_process_execute = run_command_with_inputs(bootstrap_command_list, inputs)

self.assertEqual(bootstrap_process_execute.process.returncode, 0)
stdout = bootstrap_process_execute.stdout.decode()
self.assertIn("We have created the following resources", stdout)
# make sure pipeline user's credential is printed
self.assertIn("ACCESS_KEY_ID", stdout)
self.assertIn("SECRET_ACCESS_KEY", stdout)

common_resources = {
"PipelineUser",
"PipelineUserAccessKey",
"PipelineUserSecretKey",
"CloudFormationExecutionRole",
"PipelineExecutionRole",
"ArtifactsBucket",
"ArtifactsLoggingBucket",
"ArtifactsLoggingBucketPolicy",
"ArtifactsBucketPolicy",
"PipelineExecutionRolePermissionPolicy",
}
Expand All @@ -62,13 +74,13 @@ def test_interactive_with_no_resources_provided(self, create_image_repository: b
*common_resources,
"ImageRepository",
},
self._extract_created_resource_logical_ids(stack_name),
set(self._extract_created_resource_logical_ids(stack_name)),
)
else:
self.assertSetEqual(common_resources, self._extract_created_resource_logical_ids(stack_name))
self.assertSetEqual(common_resources, set(self._extract_created_resource_logical_ids(stack_name)))

@parameterized.expand([("create_image_repository",), (False,)])
def test_non_interactive_with_no_resources_provided(self, create_image_repository: bool):
def test_non_interactive_with_no_resources_provided(self, create_image_repository):
stage_name, stack_name = self._get_stage_and_stack_name()
self.stack_names = [stack_name]

Expand All @@ -90,13 +102,14 @@ def test_interactive_with_all_required_resources_provided(self):

inputs = [
stage_name,
CREDENTIAL_PROFILE,
"us-east-1", # region
"arn:aws:iam::123:user/user-name", # pipeline user
"arn:aws:iam::123:role/role-name", # Pipeline execution role
"arn:aws:iam::123:role/role-name", # CloudFormation execution role
"arn:aws:s3:::bucket-name", # Artifacts bucket
"3", # Should we create ECR repo, 3 - specify one
"arn:aws:ecr:::repository/repo-name", # ecr repo
"y", # proceed
"N", # Should we create ECR repo, 3 - specify one
"",
]

bootstrap_process_execute = run_command_with_inputs(bootstrap_command_list, inputs)
Expand All @@ -115,7 +128,7 @@ def test_no_interactive_with_all_required_resources_provided(self):
pipeline_user="arn:aws:iam::123:user/user-name", # pipeline user
pipeline_execution_role="arn:aws:iam::123:role/role-name", # Pipeline execution role
cloudformation_execution_role="arn:aws:iam::123:role/role-name", # CloudFormation execution role
artifacts_bucket="arn:aws:s3:::bucket-name", # Artifacts bucket
bucket="arn:aws:s3:::bucket-name", # Artifacts bucket
image_repository="arn:aws:ecr:::repository/repo-name", # ecr repo
)

Expand All @@ -126,7 +139,7 @@ def test_no_interactive_with_all_required_resources_provided(self):
self.assertIn("skipping creation", stdout)

@parameterized.expand([("confirm_changeset",), (False,)])
def test_no_interactive_with_some_required_resources_provided(self, confirm_changeset):
def test_no_interactive_with_some_required_resources_provided(self, confirm_changeset: bool):
stage_name, stack_name = self._get_stage_and_stack_name()
self.stack_names = [stack_name]

Expand All @@ -136,7 +149,7 @@ def test_no_interactive_with_some_required_resources_provided(self, confirm_chan
pipeline_user="arn:aws:iam::123:user/user-name", # pipeline user
pipeline_execution_role="arn:aws:iam::123:role/role-name", # Pipeline execution role
# CloudFormation execution role missing
artifacts_bucket="arn:aws:s3:::bucket-name", # Artifacts bucket
bucket="arn:aws:s3:::bucket-name", # Artifacts bucket
image_repository="arn:aws:ecr:::repository/repo-name", # ecr repo
no_confirm_changeset=not confirm_changeset,
)
Expand All @@ -150,7 +163,7 @@ def test_no_interactive_with_some_required_resources_provided(self, confirm_chan
self.assertEqual(bootstrap_process_execute.process.returncode, 0)
stdout = bootstrap_process_execute.stdout.decode()
self.assertIn("Successfully created!", stdout)
self.assertSetEqual({"CloudFormationExecutionRole"}, self._extract_created_resource_logical_ids(stack_name))
self.assertIn("CloudFormationExecutionRole", self._extract_created_resource_logical_ids(stack_name))

def test_interactive_cancelled_by_user(self):
stage_name, stack_name = self._get_stage_and_stack_name()
Expand All @@ -160,19 +173,22 @@ def test_interactive_cancelled_by_user(self):

inputs = [
stage_name,
CREDENTIAL_PROFILE,
"us-east-1", # region
"arn:aws:iam::123:user/user-name", # pipeline user
"", # Pipeline execution role
"arn:aws:iam::123:role/role-name", # Pipeline execution role
"", # CloudFormation execution role
"", # Artifacts bucket
"1", # Should we create ECR repo, 1 - No
"N", # cancel
"arn:aws:s3:::bucket-name", # Artifacts bucket
"N", # Do you have Lambda with package type Image
"",
"", # Create resources confirmation
]

bootstrap_process_execute = run_command_with_inputs(bootstrap_command_list, inputs)

self.assertEqual(bootstrap_process_execute.process.returncode, 0)
stdout = bootstrap_process_execute.stdout.decode()
self.assertTrue(stdout.strip().endswith("Should we proceed with the creation? [y/N]:"))
self.assertTrue(stdout.strip().endswith("Canceling pipeline bootstrap creation."))
self.assertFalse(self._stack_exists(stack_name))

def test_interactive_with_some_required_resources_provided(self):
Expand All @@ -183,13 +199,15 @@ def test_interactive_with_some_required_resources_provided(self):

inputs = [
stage_name,
CREDENTIAL_PROFILE,
"us-east-1", # region
"arn:aws:iam::123:user/user-name", # pipeline user
"arn:aws:iam::123:role/role-name", # Pipeline execution role
"", # CloudFormation execution role
"arn:aws:s3:::bucket-name", # Artifacts bucket
"3", # Should we create ECR repo, 3 - specify one
"arn:aws:ecr:::repository/repo-name", # ecr repo
"y", # proceed
"N", # Do you have Lambda with package type Image
"",
"y", # Create resources confirmation
]

bootstrap_process_execute = run_command_with_inputs(bootstrap_command_list, inputs)
Expand All @@ -198,7 +216,7 @@ def test_interactive_with_some_required_resources_provided(self):
stdout = bootstrap_process_execute.stdout.decode()
self.assertIn("Successfully created!", stdout)
# make sure the not provided resource is the only resource created.
self.assertSetEqual({"CloudFormationExecutionRole"}, self._extract_created_resource_logical_ids(stack_name))
self.assertIn("CloudFormationExecutionRole", self._extract_created_resource_logical_ids(stack_name))

def test_interactive_pipeline_user_only_created_once(self):
"""
Expand All @@ -216,12 +234,15 @@ def test_interactive_pipeline_user_only_created_once(self):
for i, stage_name in enumerate(stage_names):
inputs = [
stage_name,
CREDENTIAL_PROFILE,
"us-east-1", # region
*([""] if i == 0 else []), # pipeline user
"arn:aws:iam::123:role/role-name", # Pipeline execution role
"arn:aws:iam::123:role/role-name", # CloudFormation execution role
"arn:aws:s3:::bucket-name", # Artifacts bucket
"1", # Should we create ECR repo, 1 - No, 2 - Yes
"y", # proceed
"N", # Should we create ECR repo, 3 - specify one
"",
"y", # Create resources confirmation
]

bootstrap_process_execute = run_command_with_input(
Expand All @@ -233,11 +254,11 @@ def test_interactive_pipeline_user_only_created_once(self):

# Only first environment creates pipeline user
if i == 0:
self.assertIn("We have created the following resources", stdout)
self.assertSetEqual(
{"PipelineUser", "PipelineUserAccessKey"},
self._extract_created_resource_logical_ids(self.stack_names[i]),
)
self.assertIn("The following resources were created in your account:", stdout)
resources = self._extract_created_resource_logical_ids(self.stack_names[i])
self.assertTrue("PipelineUser" in resources)
self.assertTrue("PipelineUserAccessKey" in resources)
self.assertTrue("PipelineUserSecretKey" in resources)
Comment on lines +259 to +261
Copy link
Contributor

@aahung aahung Jul 15, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

self.assertIn

else:
self.assertIn("skipping creation", stdout)

Expand Down