From 89383f2da556ce1f193626a63872416a172eca51 Mon Sep 17 00:00:00 2001 From: Petros Kalos Date: Mon, 20 May 2024 13:18:11 +0300 Subject: [PATCH 1/4] automate bootstrapping of integrations tests * introduce a secret that holds the test userdata * move ConfigureCognito step from frontend to backend * add logic in ConfigureCognito to create the test users and assigned them to groups if `with_approval_tests=True` (reads info from the secret) * tests are reading the userdata from the secret --- deploy/configs/cognito_urls_config.py | 108 +++++++++++++----------- deploy/stacks/backend_stack.py | 2 + deploy/stacks/backend_stage.py | 2 + deploy/stacks/cognito.py | 4 +- deploy/stacks/pipeline.py | 21 +++-- tests_new/integration_tests/README.md | 13 ++- tests_new/integration_tests/conftest.py | 32 ++++--- 7 files changed, 112 insertions(+), 70 deletions(-) diff --git a/deploy/configs/cognito_urls_config.py b/deploy/configs/cognito_urls_config.py index 2562cfbdd..b78db5584 100644 --- a/deploy/configs/cognito_urls_config.py +++ b/deploy/configs/cognito_urls_config.py @@ -20,6 +20,7 @@ def setup_cognito( internet_facing='True', custom_domain='False', enable_cw_canaries='False', + with_approval_tests='False', ): ssm = boto3.client('ssm', region_name=region) user_pool_id = ssm.get_parameter(Name=f'/dataall/{envname}/cognito/userpool')['Parameter']['Value'] @@ -94,61 +95,68 @@ def setup_cognito( sm = boto3.client('secretsmanager', region_name=region) secret = sm.get_secret_value(SecretId=f'{resource_prefix}-{envname}-cognito-canary-user') creds = json.loads(secret['SecretString']) - username = creds['username'] - print('Creating Canaries user...') - try: - response = cognito.admin_create_user( - UserPoolId=user_pool_id, - Username=username, - UserAttributes=[{'Name': 'email', 'Value': f'{username}@amazonaws.com'}], - TemporaryPassword='da@' - + shuffle_password( - random.SystemRandom().choice(string.ascii_uppercase) - + random.SystemRandom().choice(string.digits) - + ''.join( - random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(11) - ) - ), - MessageAction='SUPPRESS', - ) - print(f'User Created Successfully...: {response}') - except ClientError as e: - if 'UsernameExistsException' in str(e): - print('User already exists') - else: - raise e - - print('Updating Canaries user password...') - response = cognito.admin_set_user_password( - UserPoolId=user_pool_id, - Username=username, - Password=creds['password'], - Permanent=True, - ) - print(f'User password updated Successfully...: {response}') - try: - response = cognito.create_group( - GroupName='CWCanaries', - UserPoolId=user_pool_id, - Description='CW Canary group', - ) - print(f'Canaries group created Successfully...: {response}') - except ClientError as e: - if 'GroupExistsException' in str(e): - print('Group already exists') - else: - raise e - - response = cognito.admin_add_user_to_group( - GroupName='CWCanaries', UserPoolId=user_pool_id, Username=username - ) - print(f'User added to group Successfully...: {response}') + create_user(cognito, user_pool_id, creds['username'], creds['password'], ['CWCanaries']) + + if with_approval_tests == 'True': + sm = boto3.client('secretsmanager', region_name=region) + secret = sm.get_secret_value(SecretId=f'{resource_prefix}-{envname}-cognito-test-users') + users = json.loads(secret['SecretString']) + for username, data in users.items(): + create_user(cognito, user_pool_id, username, data['password'], data['groups']) except ClientError as e: print(f'Failed to setup cognito due to: {e}') raise e +def create_user(cognito, user_pool_id, username, password, groups=[]): + print('Creating user...') + try: + response = cognito.admin_create_user( + UserPoolId=user_pool_id, + Username=username, + UserAttributes=[{'Name': 'email', 'Value': f'{username}@amazonaws.com'}], + TemporaryPassword='da@' + + shuffle_password( + random.SystemRandom().choice(string.ascii_uppercase) + + random.SystemRandom().choice(string.digits) + + ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(11)) + ), + MessageAction='SUPPRESS', + ) + print(f'User Created Successfully...: {response}') + except ClientError as e: + if 'UsernameExistsException' in str(e): + print('User already exists') + else: + raise e + + print('Updating Canaries user password...') + response = cognito.admin_set_user_password( + UserPoolId=user_pool_id, + Username=username, + Password=password, + Permanent=True, + ) + print(f'User password updated Successfully...: {response}') + + for group in groups: + try: + response = cognito.create_group( + GroupName=group, + UserPoolId=user_pool_id, + ) + print(f'Group created Successfully...: {response}') + except ClientError as e: + if 'GroupExistsException' in str(e): + print('Group already exists') + else: + raise e + + response = cognito.admin_add_user_to_group(GroupName=group, UserPoolId=user_pool_id, Username=username) + print(f'User added to group Successfully...: {response}') + + if __name__ == '__main__': print('Starting Cognito Configuration...') envname = os.environ.get('envname') @@ -157,6 +165,7 @@ def setup_cognito( custom_domain = os.environ.get('custom_domain') enable_cw_canaries = os.environ.get('enable_cw_canaries') resource_prefix = os.environ.get('resource_prefix') + with_approval_tests = os.environ.get('with_approval_tests') setup_cognito( region, resource_prefix, @@ -164,5 +173,6 @@ def setup_cognito( internet_facing, custom_domain, enable_cw_canaries, + with_approval_tests, ) print('Cognito Configuration Finished Successfully') diff --git a/deploy/stacks/backend_stack.py b/deploy/stacks/backend_stack.py index 7b0a96709..54dc80a1f 100644 --- a/deploy/stacks/backend_stack.py +++ b/deploy/stacks/backend_stack.py @@ -56,6 +56,7 @@ def __init__( cognito_user_session_timeout_inmins=43200, custom_auth=None, custom_waf_rules=None, + with_approval_tests=False, **kwargs, ): super().__init__(scope, id, **kwargs) @@ -126,6 +127,7 @@ def __init__( vpc=vpc, cognito_user_session_timeout_inmins=cognito_user_session_timeout_inmins, custom_waf_rules=custom_waf_rules, + with_approval_tests=with_approval_tests, **kwargs, ) else: diff --git a/deploy/stacks/backend_stage.py b/deploy/stacks/backend_stage.py index 0c8a5720c..81d2e4ac3 100644 --- a/deploy/stacks/backend_stage.py +++ b/deploy/stacks/backend_stage.py @@ -36,6 +36,7 @@ def __init__( cognito_user_session_timeout_inmins=43200, custom_auth=None, custom_waf_rules=None, + with_approval_tests=False, **kwargs, ): super().__init__(scope, id, **kwargs) @@ -69,6 +70,7 @@ def __init__( cognito_user_session_timeout_inmins=cognito_user_session_timeout_inmins, custom_auth=custom_auth, custom_waf_rules=custom_waf_rules, + with_approval_tests=with_approval_tests, **kwargs, ) diff --git a/deploy/stacks/cognito.py b/deploy/stacks/cognito.py index bf5487c3e..4b2b80e45 100644 --- a/deploy/stacks/cognito.py +++ b/deploy/stacks/cognito.py @@ -12,6 +12,7 @@ Duration, CustomResource, ) +from aws_cdk.aws_cognito import AuthFlow from .pyNestedStack import pyNestedClass from .solution_bundling import SolutionBundling @@ -32,6 +33,7 @@ def __init__( tooling_account_id=None, enable_cw_rum=False, cognito_user_session_timeout_inmins=43200, + with_approval_tests=False, **kwargs, ): super().__init__(scope, id, **kwargs) @@ -93,11 +95,11 @@ def __init__( domain_prefix=f"{resource_prefix.replace('-', '')}{envname}{self.region.replace('-', '')}{self.account}" ), ) - self.client = cognito.UserPoolClient( self, f'AppClient-{envname}', user_pool=self.user_pool, + auth_flows=AuthFlow(user_password=with_approval_tests, user_srp=True, custom=True), prevent_user_existence_errors=True, refresh_token_validity=Duration.minutes(cognito_user_session_timeout_inmins), ) diff --git a/deploy/stacks/pipeline.py b/deploy/stacks/pipeline.py index fc4159629..8bd930c9b 100644 --- a/deploy/stacks/pipeline.py +++ b/deploy/stacks/pipeline.py @@ -9,6 +9,7 @@ from aws_cdk import aws_iam as iam from aws_cdk import aws_s3 as s3 from aws_cdk import pipelines +from aws_cdk.aws_codebuild import BuildEnvironmentVariable, BuildEnvironmentVariableType from aws_cdk.pipelines import CodePipelineSource from .albfront_stage import AlbFrontStage @@ -648,8 +649,11 @@ def set_backend_stage(self, target_env, repository_name): cognito_user_session_timeout_inmins=target_env.get('cognito_user_session_timeout_inmins', 43200), custom_auth=target_env.get('custom_auth', None), custom_waf_rules=target_env.get('custom_waf_rules', None), + with_approval_tests=target_env.get('with_approval_tests', False), ) ) + if target_env.get('custom_auth') is None: + backend_stage.add_post(self.cognito_config_action(target_env)) return backend_stage def set_approval_tests_stage( @@ -667,6 +671,12 @@ def set_approval_tests_stage( id='ApprovalTests', build_environment=codebuild.BuildEnvironment( build_image=codebuild.LinuxBuildImage.AMAZON_LINUX_2_5, + environment_variables={ + 'USERDATA': BuildEnvironmentVariable( + value=f'{self.resource_prefix}-{target_env["envname"]}-cognito-test-users', + type=BuildEnvironmentVariableType.SECRETS_MANAGER, + ), + }, ), partial_build_spec=codebuild.BuildSpec.from_object( dict( @@ -767,7 +777,7 @@ def set_cloudfront_stage(self, target_env): f'export internet_facing={target_env.get("internet_facing", True)}', f'export custom_domain={str(True) if target_env.get("custom_domain") else str(False)}', f'export deployment_region={target_env.get("region", self.region)}', - f'export enable_cw_rum={target_env.get("enable_cw_rum", False) and target_env.get("custom_auth", None) is None }', + f'export enable_cw_rum={target_env.get("enable_cw_rum", False) and target_env.get("custom_auth", None) is None}', f'export resource_prefix={self.resource_prefix}', f'export reauth_ttl={str(target_env.get("reauth_config", {}).get("ttl", 5))}', f'export custom_auth_provider={str(target_env.get("custom_auth", {}).get("provider", "None"))}', @@ -803,11 +813,6 @@ def set_cloudfront_stage(self, target_env): vpc=self.vpc, ), ) - if target_env.get('custom_auth', None) is None: - front_stage_actions = ( - *front_stage_actions, - self.cognito_config_action(target_env), - ) if target_env.get('enable_cw_rum', False) and target_env.get('custom_auth', None) is None: front_stage_actions = ( *front_stage_actions, @@ -881,6 +886,7 @@ def cognito_config_action(self, target_env): f'export custom_domain={str(True) if target_env.get("custom_domain") else str(False)}', f'export deployment_region={target_env.get("region", self.region)}', f'export enable_cw_canaries={target_env.get("enable_cw_canaries", False)}', + f'export with_approval_tests={target_env.get("with_approval_tests", False)}', 'mkdir ~/.aws/ && touch ~/.aws/config', 'echo "[profile buildprofile]" > ~/.aws/config', f'echo "role_arn = arn:aws:iam::{target_env["account"]}:role/{self.resource_prefix}-{target_env["envname"]}-cognito-config-role" >> ~/.aws/config', @@ -971,9 +977,6 @@ def set_albfront_stage(self, target_env, repository_name): if target_env.get('custom_auth') is None: albfront_stage.add_pre(self.user_guide_pre_build_alb(repository_name)) - if target_env.get('custom_auth') is None: - albfront_stage.add_post(self.cognito_config_action(target_env)) - if target_env.get('enable_cw_rum', False) and target_env.get('custom_auth', None) is None: albfront_stage.add_post(self.cw_rum_config_action(target_env)) diff --git a/tests_new/integration_tests/README.md b/tests_new/integration_tests/README.md index 4b3a658bc..5e38c6f14 100644 --- a/tests_new/integration_tests/README.md +++ b/tests_new/integration_tests/README.md @@ -3,7 +3,18 @@ The purpose of these tests is to automatically validate functionalities of data. ## Pre-requisites - A real deployment of data.all in AWS -- 4 Cognito users (at the moment only Cognito is supported) like the ones defined in `conftest`(e.g. `testUser1` with password `Pass1Word!`) +- A SecretManager secret (`'{self.resource_prefix}-{target_env["envname"]}-cognito-test-users'`) with the followings contents + ``` + { + "testUserTenant": {"password": "yourPassword", "groups": ["DAAdministrators"]}, + "testUser1": {"password": "yourPassword", "groups": ["testGroup1"]}, + "testUser2": {"password": "yourPassword", "groups": ["testGroup2"]}, + "testUser3": {"password": "yourPassword", "groups": ["testGroup3"]}, + "testUser4": {"password": "yourPassword", "groups": ["testGroup4"]} + } + ``` +- If you are not using Cognito then you must manually create the users/groups +- If you are using Cognito the pipeline will create the users/groups ## Run tests diff --git a/tests_new/integration_tests/conftest.py b/tests_new/integration_tests/conftest.py index 22ff24c38..531b4d814 100644 --- a/tests_new/integration_tests/conftest.py +++ b/tests_new/integration_tests/conftest.py @@ -1,4 +1,7 @@ +import json +import os from dataclasses import dataclass +from typing import List import pytest @@ -11,36 +14,45 @@ class User: username: str password: str + @staticmethod + def from_userdata(userdata, username): + return User(username, userdata[username]['password']) + + +@pytest.fixture(scope='module', autouse=True) +def userdata(): + yield json.loads(os.getenv('USERDATA')) + @pytest.fixture(scope='module', autouse=True) -def userTenant(): +def userTenant(userdata): # Existing user with name and password # This user needs to belong to `DAAdministrators` group - yield User('testUserTenant', 'Pass1Word!') + yield User.from_userdata(userdata, 'testUserTenant') @pytest.fixture(scope='module', autouse=True) -def user1(): +def user1(userdata): # Existing user with name and password - yield User('testUser1', 'Pass1Word!') + yield User.from_userdata(userdata, 'testUser1') @pytest.fixture(scope='module', autouse=True) -def user2(): +def user2(userdata): # Existing user with name and password - yield User('testUser2', 'Pass1Word!') + yield User.from_userdata(userdata, 'testUser2') @pytest.fixture(scope='module', autouse=True) -def user3(): +def user3(userdata): # Existing user with name and password - yield User('testUser3', 'Pass1Word!') + yield User.from_userdata(userdata, 'testUser3') @pytest.fixture(scope='module', autouse=True) -def user4(): +def user4(userdata): # Existing user with name and password - yield User('testUser4', 'Pass1Word!') + yield User.from_userdata(userdata, 'testUser4') @pytest.fixture(scope='module', autouse=True) From 56867f68f604e70f710cb9a52a87af100e5beaa9 Mon Sep 17 00:00:00 2001 From: Petros Kalos Date: Tue, 21 May 2024 11:54:31 +0300 Subject: [PATCH 2/4] convert print to log --- deploy/configs/cognito_urls_config.py | 136 +++++++++++++------------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/deploy/configs/cognito_urls_config.py b/deploy/configs/cognito_urls_config.py index b78db5584..54b6b4477 100644 --- a/deploy/configs/cognito_urls_config.py +++ b/deploy/configs/cognito_urls_config.py @@ -1,11 +1,16 @@ import json +import logging import os import random import string +import sys import boto3 from botocore.exceptions import ClientError +logging.basicConfig(stream=sys.stdout, level=os.environ.get('LOG_LEVEL', 'INFO')) +log = logging.getLogger(os.path.basename(__file__)) + def shuffle_password(pwd): chars = list(pwd) @@ -24,11 +29,11 @@ def setup_cognito( ): ssm = boto3.client('ssm', region_name=region) user_pool_id = ssm.get_parameter(Name=f'/dataall/{envname}/cognito/userpool')['Parameter']['Value'] - print(f'Cognito Pool ID: {user_pool_id}') + log.info(f'Cognito Pool ID: {user_pool_id}') app_client = ssm.get_parameter(Name=f'/dataall/{envname}/cognito/appclient')['Parameter']['Value'] if custom_domain == 'False' and internet_facing == 'True': - print('Switching to us-east-1 region...') + log.info('Switching to us-east-1 region...') ssm = boto3.client('ssm', region_name='us-east-1') signin_singout_link = ssm.get_parameter(Name=f'/dataall/{envname}/CloudfrontDistributionDomainName')[ 'Parameter' @@ -44,73 +49,68 @@ def setup_cognito( 'Value' ] - print(f'UI: {signin_singout_link}') - print(f'USERGUIDE: {user_guide_link}') + log.info(f'UI: {signin_singout_link}') + log.info(f'USERGUIDE: {user_guide_link}') cognito = boto3.client('cognito-idp', region_name=region) - try: - user_pool = cognito.describe_user_pool_client(UserPoolId=user_pool_id, ClientId=app_client) + user_pool = cognito.describe_user_pool_client(UserPoolId=user_pool_id, ClientId=app_client) + + del user_pool['UserPoolClient']['CreationDate'] + del user_pool['UserPoolClient']['LastModifiedDate'] + + config_callbacks = [ + f'https://{signin_singout_link}', + f'https://{user_guide_link}/parseauth', + ] + existing_callbacks = user_pool['UserPoolClient'].get('CallbackURLs', []) + if 'https://example.com' in existing_callbacks: + existing_callbacks.remove('https://example.com') + updated_callbacks = existing_callbacks + list(set(config_callbacks) - set(existing_callbacks)) + log.info(f'Updated CallBackUrls: {updated_callbacks}') + + config_logout_urls = [f'https://{signin_singout_link}'] + existing_logout_urls = user_pool['UserPoolClient'].get('LogoutURLs', []) + updated_logout_urls = existing_logout_urls + list(set(config_logout_urls) - set(existing_logout_urls)) + log.info(f'Updated LogOutUrls: {updated_logout_urls}') + + user_pool['UserPoolClient']['CallbackURLs'] = updated_callbacks + user_pool['UserPoolClient']['LogoutURLs'] = updated_logout_urls + + response = cognito.update_user_pool_client( + **user_pool['UserPoolClient'], + ) - del user_pool['UserPoolClient']['CreationDate'] - del user_pool['UserPoolClient']['LastModifiedDate'] + log.info(f'CallbackUrls and LogOutUrls updated successfully: {response}') - config_callbacks = [ - f'https://{signin_singout_link}', - f'https://{user_guide_link}/parseauth', - ] - existing_callbacks = user_pool['UserPoolClient'].get('CallbackURLs', []) - if 'https://example.com' in existing_callbacks: - existing_callbacks.remove('https://example.com') - updated_callbacks = existing_callbacks + list(set(config_callbacks) - set(existing_callbacks)) - print(f'Updated CallBackUrls: {updated_callbacks}') - - config_logout_urls = [f'https://{signin_singout_link}'] - existing_logout_urls = user_pool['UserPoolClient'].get('LogoutURLs', []) - updated_logout_urls = existing_logout_urls + list(set(config_logout_urls) - set(existing_logout_urls)) - print(f'Updated LogOutUrls: {updated_logout_urls}') - - user_pool['UserPoolClient']['CallbackURLs'] = updated_callbacks - user_pool['UserPoolClient']['LogoutURLs'] = updated_logout_urls - - response = cognito.update_user_pool_client( - **user_pool['UserPoolClient'], + try: + response = cognito.create_group( + GroupName='DAAdministrators', + UserPoolId=user_pool_id, + Description='administrators group', ) + log.info(f'Administrators group created Successfully...: {response}') + except ClientError as e: + if 'GroupExistsException' in str(e): + log.info('Group already exists') + else: + raise e - print(f'CallbackUrls and LogOutUrls updated successfully: {response}') - - try: - response = cognito.create_group( - GroupName='DAAdministrators', - UserPoolId=user_pool_id, - Description='administrators group', - ) - print(f'Administrators group created Successfully...: {response}') - except ClientError as e: - if 'GroupExistsException' in str(e): - print('Group already exists') - else: - raise e - - if enable_cw_canaries == 'True': - sm = boto3.client('secretsmanager', region_name=region) - secret = sm.get_secret_value(SecretId=f'{resource_prefix}-{envname}-cognito-canary-user') - creds = json.loads(secret['SecretString']) - create_user(cognito, user_pool_id, creds['username'], creds['password'], ['CWCanaries']) - - if with_approval_tests == 'True': - sm = boto3.client('secretsmanager', region_name=region) - secret = sm.get_secret_value(SecretId=f'{resource_prefix}-{envname}-cognito-test-users') - users = json.loads(secret['SecretString']) - for username, data in users.items(): - create_user(cognito, user_pool_id, username, data['password'], data['groups']) + if enable_cw_canaries == 'True': + sm = boto3.client('secretsmanager', region_name=region) + secret = sm.get_secret_value(SecretId=f'{resource_prefix}-{envname}-cognito-canary-user') + creds = json.loads(secret['SecretString']) + create_user(cognito, user_pool_id, creds['username'], creds['password'], ['CWCanaries']) - except ClientError as e: - print(f'Failed to setup cognito due to: {e}') - raise e + if with_approval_tests == 'True': + sm = boto3.client('secretsmanager', region_name=region) + secret = sm.get_secret_value(SecretId=f'{resource_prefix}-{envname}-cognito-test-users') + users = json.loads(secret['SecretString']) + for username, data in users.items(): + create_user(cognito, user_pool_id, username, data['password'], data['groups']) def create_user(cognito, user_pool_id, username, password, groups=[]): - print('Creating user...') + log.info('Creating user...') try: response = cognito.admin_create_user( UserPoolId=user_pool_id, @@ -124,21 +124,21 @@ def create_user(cognito, user_pool_id, username, password, groups=[]): ), MessageAction='SUPPRESS', ) - print(f'User Created Successfully...: {response}') + log.info(f'User Created Successfully...: {response}') except ClientError as e: if 'UsernameExistsException' in str(e): - print('User already exists') + log.info('User already exists') else: raise e - print('Updating Canaries user password...') + log.info('Updating user password...') response = cognito.admin_set_user_password( UserPoolId=user_pool_id, Username=username, Password=password, Permanent=True, ) - print(f'User password updated Successfully...: {response}') + log.info(f'User password updated Successfully...: {response}') for group in groups: try: @@ -146,19 +146,19 @@ def create_user(cognito, user_pool_id, username, password, groups=[]): GroupName=group, UserPoolId=user_pool_id, ) - print(f'Group created Successfully...: {response}') + log.info(f'Group created Successfully...: {response}') except ClientError as e: if 'GroupExistsException' in str(e): - print('Group already exists') + log.info('Group already exists') else: raise e response = cognito.admin_add_user_to_group(GroupName=group, UserPoolId=user_pool_id, Username=username) - print(f'User added to group Successfully...: {response}') + log.info(f'User added to group Successfully...: {response}') if __name__ == '__main__': - print('Starting Cognito Configuration...') + log.info('Starting Cognito Configuration...') envname = os.environ.get('envname') region = os.environ.get('deployment_region') internet_facing = os.environ.get('internet_facing') @@ -175,4 +175,4 @@ def create_user(cognito, user_pool_id, username, password, groups=[]): enable_cw_canaries, with_approval_tests, ) - print('Cognito Configuration Finished Successfully') + log.info('Cognito Configuration Finished Successfully') From 42b458b4f39f46e25c1d9eedd53bd5a7858d613c Mon Sep 17 00:00:00 2001 From: Petros Kalos Date: Tue, 21 May 2024 12:22:52 +0300 Subject: [PATCH 3/4] move to SSM --- deploy/configs/cognito_urls_config.py | 7 ++++--- deploy/stacks/pipeline.py | 5 +++-- tests_new/integration_tests/README.md | 10 ++++++++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/deploy/configs/cognito_urls_config.py b/deploy/configs/cognito_urls_config.py index 54b6b4477..613d34ad1 100644 --- a/deploy/configs/cognito_urls_config.py +++ b/deploy/configs/cognito_urls_config.py @@ -102,9 +102,10 @@ def setup_cognito( create_user(cognito, user_pool_id, creds['username'], creds['password'], ['CWCanaries']) if with_approval_tests == 'True': - sm = boto3.client('secretsmanager', region_name=region) - secret = sm.get_secret_value(SecretId=f'{resource_prefix}-{envname}-cognito-test-users') - users = json.loads(secret['SecretString']) + ssm = boto3.client('ssm', region_name=region) + users = json.loads( + ssm.get_parameter(Name=os.path.join('/dataall', envname, 'cognito-test-users'))['Parameter']['Value'] + ) for username, data in users.items(): create_user(cognito, user_pool_id, username, data['password'], data['groups']) diff --git a/deploy/stacks/pipeline.py b/deploy/stacks/pipeline.py index 8bd930c9b..547d54e9c 100644 --- a/deploy/stacks/pipeline.py +++ b/deploy/stacks/pipeline.py @@ -1,3 +1,4 @@ +import os import re import uuid from typing import List @@ -673,8 +674,8 @@ def set_approval_tests_stage( build_image=codebuild.LinuxBuildImage.AMAZON_LINUX_2_5, environment_variables={ 'USERDATA': BuildEnvironmentVariable( - value=f'{self.resource_prefix}-{target_env["envname"]}-cognito-test-users', - type=BuildEnvironmentVariableType.SECRETS_MANAGER, + value=os.path.join('/', self.resource_prefix, target_env['envname'], 'cognito-test-users'), + type=BuildEnvironmentVariableType.PARAMETER_STORE, ), }, ), diff --git a/tests_new/integration_tests/README.md b/tests_new/integration_tests/README.md index 5e38c6f14..4e0aa2e3a 100644 --- a/tests_new/integration_tests/README.md +++ b/tests_new/integration_tests/README.md @@ -1,9 +1,11 @@ # Integration tests + The purpose of these tests is to automatically validate functionalities of data.all on a real deployment. ## Pre-requisites + - A real deployment of data.all in AWS -- A SecretManager secret (`'{self.resource_prefix}-{target_env["envname"]}-cognito-test-users'`) with the followings contents +- An SSM parameter (`/{resource_prefix/{env_name}/cognito-test-users`) with the following contents ``` { "testUserTenant": {"password": "yourPassword", "groups": ["DAAdministrators"]}, @@ -18,9 +20,11 @@ The purpose of these tests is to automatically validate functionalities of data. ## Run tests -The tests are executed in CodeBuild as part of the CICD pipeline if the cdk.json parameter `with_approval_tests` is set to True. +The tests are executed in CodeBuild as part of the CICD pipeline if the cdk.json parameter `with_approval_tests` is set +to True. But you can also run the tests locally with deployment account credentials: + ```bash export ENVNAME = "Introduce deployment environment name" export AWS_REGION = "Introduce backend region" @@ -28,6 +32,7 @@ make integration-tests ``` or run the tests locally without credentials: + ```bash export ENVNAME = "Introduce deployment environment name" export AWS_REGION = "Introduce backend region" @@ -37,4 +42,5 @@ make integration-tests ``` ## Coverage + At the moment integration tests only cover Organizations module as an example. \ No newline at end of file From 5dfeeddf22988c35cb4884275a2710d975e56b18 Mon Sep 17 00:00:00 2001 From: Petros Kalos Date: Tue, 21 May 2024 14:43:54 +0300 Subject: [PATCH 4/4] codebuild to trigger --- .../cognito_config/__init__.py | 0 .../cognito_config}/cognito_urls_config.py | 8 +- .../cognito_config/requirements.txt | 0 deploy/stacks/cognito.py | 77 +++++++++++++++++++ deploy/stacks/pipeline.py | 30 -------- 5 files changed, 81 insertions(+), 34 deletions(-) create mode 100644 deploy/custom_resources/cognito_config/__init__.py rename deploy/{configs => custom_resources/cognito_config}/cognito_urls_config.py (97%) create mode 100644 deploy/custom_resources/cognito_config/requirements.txt diff --git a/deploy/custom_resources/cognito_config/__init__.py b/deploy/custom_resources/cognito_config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/deploy/configs/cognito_urls_config.py b/deploy/custom_resources/cognito_config/cognito_urls_config.py similarity index 97% rename from deploy/configs/cognito_urls_config.py rename to deploy/custom_resources/cognito_config/cognito_urls_config.py index 613d34ad1..585ef0fdd 100644 --- a/deploy/configs/cognito_urls_config.py +++ b/deploy/custom_resources/cognito_config/cognito_urls_config.py @@ -3,13 +3,13 @@ import os import random import string -import sys import boto3 from botocore.exceptions import ClientError -logging.basicConfig(stream=sys.stdout, level=os.environ.get('LOG_LEVEL', 'INFO')) -log = logging.getLogger(os.path.basename(__file__)) +logger = logging.getLogger() +logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO')) +log = logging.getLogger(__name__) def shuffle_password(pwd): @@ -158,7 +158,7 @@ def create_user(cognito, user_pool_id, username, password, groups=[]): log.info(f'User added to group Successfully...: {response}') -if __name__ == '__main__': +def handler(event, context) -> None: log.info('Starting Cognito Configuration...') envname = os.environ.get('envname') region = os.environ.get('deployment_region') diff --git a/deploy/custom_resources/cognito_config/requirements.txt b/deploy/custom_resources/cognito_config/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/deploy/stacks/cognito.py b/deploy/stacks/cognito.py index 4b2b80e45..00f782c1d 100644 --- a/deploy/stacks/cognito.py +++ b/deploy/stacks/cognito.py @@ -13,6 +13,7 @@ CustomResource, ) from aws_cdk.aws_cognito import AuthFlow +from aws_cdk.triggers import TriggerFunction from .pyNestedStack import pyNestedClass from .solution_bundling import SolutionBundling @@ -313,6 +314,82 @@ def __init__( sync_cr.node.add_dependency(domain_name) sync_cr.node.add_dependency(pool_arn) + cognito_config_assets = os.path.realpath( + os.path.join( + os.path.dirname(__file__), + '..', + 'custom_resources', + 'cognito_config', + ) + ) + + cognito_config_code = _lambda.Code.from_asset( + path=cognito_config_assets, + bundling=BundlingOptions( + image=_lambda.Runtime.PYTHON_3_9.bundling_image, + local=SolutionBundling(source_path=cognito_config_assets), + ), + ) + + TriggerFunction( + self, + 'TriggerFunction-CognitoConfig', + function_name=f'{resource_prefix}-{envname}-cognito_config', + description='dataall CognitoConfig trigger function', + initial_policy=[ + iam.PolicyStatement( + effect=iam.Effect.ALLOW, + actions=[ + 'cognito-idp:AddCustomAttributes', + 'cognito-idp:UpdateUserPool', + 'cognito-idp:DescribeUserPoolClient', + 'cognito-idp:CreateGroup', + 'cognito-idp:UpdateUserPoolClient', + 'cognito-idp:AdminSetUserPassword', + 'cognito-idp:AdminCreateUser', + 'cognito-idp:DescribeUserPool', + 'cognito-idp:AdminAddUserToGroup', + 'secretsmanager:DescribeSecret', + 'secretsmanager:GetSecretValue', + 'ssm:GetParameterHistory', + 'ssm:GetParameters', + 'ssm:GetParameter', + 'ssm:GetParametersByPath', + 'kms:Decrypt', + 'kms:GenerateDataKey', + 'kms:DescribeKey', + 'rum:GetAppMonitor', + ], + resources=[ + self.user_pool.user_pool_arn, + f'arn:aws:kms:{self.region}:{self.account}:key/*', + f'arn:aws:ssm:*:{self.account}:parameter/*dataall*', + f'arn:aws:secretsmanager:{self.region}:{self.account}:secret:*dataall*', + f'arn:aws:rum:{self.region}:{self.account}:appmonitor/*dataall*', + ], + ), + ], + code=cognito_config_code, + vpc=vpc, + memory_size=256, + timeout=Duration.minutes(15), + environment={ + 'envname': envname, + 'deployment_region': self.region, + 'internet_facing': str(internet_facing), + 'custom_domain': str(not domain_name), + 'enable_cw_canaries': str(enable_cw_rum), + 'resource_prefix': resource_prefix, + 'with_approval_tests': str(with_approval_tests), + }, + tracing=_lambda.Tracing.ACTIVE, + retry_attempts=0, + runtime=_lambda.Runtime.PYTHON_3_9, + handler='cognito_urls_config.handler', + execute_after=[self.client], + execute_on_handler_change=True, + ) + CfnOutput( self, 'CognitoDomainName', diff --git a/deploy/stacks/pipeline.py b/deploy/stacks/pipeline.py index 547d54e9c..2cc1382d9 100644 --- a/deploy/stacks/pipeline.py +++ b/deploy/stacks/pipeline.py @@ -653,8 +653,6 @@ def set_backend_stage(self, target_env, repository_name): with_approval_tests=target_env.get('with_approval_tests', False), ) ) - if target_env.get('custom_auth') is None: - backend_stage.add_post(self.cognito_config_action(target_env)) return backend_stage def set_approval_tests_stage( @@ -874,34 +872,6 @@ def cw_rum_config_action(self, target_env): vpc=self.vpc, ) - def cognito_config_action(self, target_env): - return pipelines.CodeBuildStep( - id='ConfigureCognito', - build_environment=codebuild.BuildEnvironment( - build_image=codebuild.LinuxBuildImage.AMAZON_LINUX_2_5, - ), - commands=[ - f'export envname={target_env["envname"]}', - f'export resource_prefix={self.resource_prefix}', - f'export internet_facing={target_env.get("internet_facing", True)}', - f'export custom_domain={str(True) if target_env.get("custom_domain") else str(False)}', - f'export deployment_region={target_env.get("region", self.region)}', - f'export enable_cw_canaries={target_env.get("enable_cw_canaries", False)}', - f'export with_approval_tests={target_env.get("with_approval_tests", False)}', - 'mkdir ~/.aws/ && touch ~/.aws/config', - 'echo "[profile buildprofile]" > ~/.aws/config', - f'echo "role_arn = arn:aws:iam::{target_env["account"]}:role/{self.resource_prefix}-{target_env["envname"]}-cognito-config-role" >> ~/.aws/config', - 'echo "credential_source = EcsContainer" >> ~/.aws/config', - 'aws sts get-caller-identity --profile buildprofile', - 'export AWS_PROFILE=buildprofile', - 'pip install --upgrade pip', - 'pip install boto3==1.34.35', - 'python deploy/configs/cognito_urls_config.py', - ], - role=self.expanded_codebuild_role.without_policy_updates(), - vpc=self.vpc, - ) - def set_albfront_stage(self, target_env, repository_name): if target_env.get('custom_auth', None) is None: frontend_deployment_role_arn = f'arn:aws:iam::{target_env["account"]}:role/{self.resource_prefix}-{target_env["envname"]}-cognito-config-role'