922 - Replace IAM inline policies by configurable Managed Policies fo…

…r folder and bucket sharing (#1068)

### Feature or Bugfix
- Feature


### Detail
- For each consumption role and group role, invited to the env 1 managed
policy is created
- User can choose, if the consumption role is data.all managed or not
- If data.all managed, the created policy is automatically attached to
role
- without this policy attached user can not create a share for this
consumption role
- policy attachment and managed options are displayed in Env->Teams
- bucket and accesspoint shares are managed through this managed
policies

### Relates
- [922](#922)

### SecurityN/A

How to test:
See backwards compatibility and local testing comments below.
 
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.


@dlpzx Update:
- ⚠️ When upgrading customers need to update their environment stacks to
update the pivot role if created through CDK. If the pivot role is
manually created, they need to update its permissions manually with the
latest version
- ⚠️ For backwards compatibility: for all existing roles both created
and imported by data.all. The share-policy will be created in any of the
next 3 scenarios:
a) if a new share request is created
b) if new items are added to an existing share request 
c) if items are revoked from an existing share request 

More details in the comments

---------

Co-authored-by: Sofia Sazonova <sazonova@amazon.co.uk>
Co-authored-by: dlpzx <dlpzx@amazon.com>
Co-authored-by: Noah Paige <noahpaig@amazon.com>

.checkov.baseline

@@ -62,12 +62,6 @@
                         "CKV_AWS_109",
                         "CKV_AWS_111"
                     ]
-                },
-                {
-                    "resource": "AWS::IAM::ManagedPolicy.PivotRolepolicy3",
-                    "check_ids": [
-                        "CKV_AWS_109"
-                    ]
                 }
             ]
         },

backend/dataall/base/aws/iam.py

@@ -1,8 +1,8 @@
 import logging
+from botocore.exceptions import ClientError
 
 from .sts import SessionHelper
 
-
 log = logging.getLogger(__name__)
 
 
@@ -16,12 +16,14 @@ def client(account_id: str, role=None):
     def get_role(account_id: str, role_arn: str, role=None):
         log.info(f"Getting IAM role = {role_arn}")
         try:
-            iamcli = IAM.client(account_id=account_id, role=role)
-            response = iamcli.get_role(
+            client = IAM.client(account_id=account_id, role=role)
+            response = client.get_role(
                 RoleName=role_arn.split("/")[-1]
             )
             assert response['Role']['Arn'] == role_arn, "Arn doesn't match the role name. Check Arn and try again."
-        except Exception as e:
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(f'Data.all Environment Pivot Role does not have permissions to get role {role_arn}: {e}')
             log.error(
                 f'Failed to get role {role_arn} due to: {e}'
             )
@@ -33,71 +35,247 @@ def get_role(account_id: str, role_arn: str, role=None):
     def get_role_arn_by_name(account_id: str, role_name: str, role=None):
         log.info(f"Getting IAM role name= {role_name}")
         try:
-            iamcli = IAM.client(account_id=account_id, role=role)
-            response = iamcli.get_role(
+            client = IAM.client(account_id=account_id, role=role)
+            response = client.get_role(
                 RoleName=role_name
             )
-        except Exception as e:
+            return response["Role"]["Arn"]
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(f'Data.all Environment Pivot Role does not have permissions to get role {role_name}: {e}')
             log.error(
                 f'Failed to get role {role_name} due to: {e}'
             )
             return None
-        else:
-            return response["Role"]["Arn"]
 
     @staticmethod
-    def update_role_policy(
-        account_id: str,
-        role_name: str,
-        policy_name: str,
-        policy: str,
+    def get_role_policy(
+            account_id: str,
+            role_name: str,
+            policy_name: str,
     ):
         try:
-            iamcli = IAM.client(account_id)
-            iamcli.put_role_policy(
+            client = IAM.client(account_id)
+            response = client.get_role_policy(
                 RoleName=role_name,
                 PolicyName=policy_name,
-                PolicyDocument=policy,
             )
-        except Exception as e:
+            return response["PolicyDocument"]
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(f'Data.all Environment Pivot Role does not have permissions to get policy {policy_name} of role {role_name}: {e}')
             log.error(
-                f'Failed to add S3 bucket access to target role {account_id}/{role_name} : {e}'
+                f'Failed to get policy {policy_name} of role {role_name} : {e}'
             )
-            raise e
+            return None
 
     @staticmethod
-    def get_role_policy(
-        account_id: str,
-        role_name: str,
-        policy_name: str,
+    def delete_role_policy(
+            account_id: str,
+            role_name: str,
+            policy_name: str,
     ):
         try:
-            iamcli = IAM.client(account_id)
-            response = iamcli.get_role_policy(
+            client = IAM.client(account_id)
+            client.delete_role_policy(
                 RoleName=role_name,
                 PolicyName=policy_name,
             )
-        except Exception as e:
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(f'Data.all Environment Pivot Role does not have permissions to delete policy {policy_name} of role {role_name}: {e}')
             log.error(
-                f'Failed to get policy {policy_name} of role {role_name} : {e}'
+                f'Failed to delete policy {policy_name} of role {role_name} : {e}'
+            )
+
+    @staticmethod
+    def get_managed_policy_by_name(
+            account_id: str,
+            policy_name: str
+    ):
+        try:
+            arn = f'arn:aws:iam::{account_id}:policy/{policy_name}'
+            client = IAM.client(account_id)
+            response = client.get_policy(PolicyArn=arn)
+            return response['Policy']
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to to get policy {policy_name}: {e}')
+            log.error(
+                f'Failed to get policy {policy_name}: {e}'
             )
             return None
-        else:
-            return response["PolicyDocument"]
 
     @staticmethod
-    def delete_role_policy(
-        account_id: str,
-        role_name: str,
-        policy_name: str,
+    def create_managed_policy(
+            account_id: str,
+            policy_name: str,
+            policy: str
     ):
         try:
-            iamcli = IAM.client(account_id)
-            iamcli.delete_role_policy(
-                RoleName=role_name,
+            client = IAM.client(account_id)
+            response = client.create_policy(
                 PolicyName=policy_name,
+                PolicyDocument=policy,
+            )
+            arn = response['Policy']['Arn']
+            log.info(
+                f'Created managed policy {arn}'
+            )
+            return arn
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(f'Data.all Environment Pivot Role does not have permissions to create managed policy {policy_name}: {e}')
+            raise Exception(f'Failed to create managed policy {policy_name} : {e}')
+
+    @staticmethod
+    def delete_managed_policy_by_name(
+            account_id: str,
+            policy_name
+    ):
+        try:
+            arn = f'arn:aws:iam::{account_id}:policy/{policy_name}'
+            client = IAM.client(account_id)
+            client.delete_policy(
+                PolicyArn=arn
+            )
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(f'Data.all Environment Pivot Role does not have permissions to delete managed policy {policy_name}: {e}')
+            raise Exception(f'Failed to delete managed policy {policy_name} : {e}')
+
+    @staticmethod
+    def get_managed_policy_default_version(
+            account_id: str,
+            policy_name: str
+    ):
+        try:
+            arn = f'arn:aws:iam::{account_id}:policy/{policy_name}'
+            client = IAM.client(account_id)
+            response = client.get_policy(PolicyArn=arn)
+            versionId = response['Policy']['DefaultVersionId']
+            policyVersion = client.get_policy_version(PolicyArn=arn, VersionId=versionId)
+            policyDocument = policyVersion['PolicyVersion']['Document']
+            return versionId, policyDocument
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(f'Data.all Environment Pivot Role does not have permissions to get policy {policy_name}: {e}')
+            log.error(f'Failed to get policy {policy_name} : {e}')
+            return None, None
+
+    @staticmethod
+    def update_managed_policy_default_version(
+            account_id: str,
+            policy_name: str,
+            old_version_id: str,
+            policy_document: str):
+        try:
+            arn = f'arn:aws:iam::{account_id}:policy/{policy_name}'
+            client = IAM.client(account_id)
+            client.create_policy_version(
+                PolicyArn=arn,
+                PolicyDocument=policy_document,
+                SetAsDefault=True
+            )
+
+            client.delete_policy_version(PolicyArn=arn, VersionId=old_version_id)
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to update policy {policy_name}: {e}')
+            raise Exception(f'Failed to update policy {policy_name} : {e}')
+
+    @staticmethod
+    def delete_managed_policy_non_default_versions(
+            account_id: str,
+            policy_name: str,
+    ):
+        try:
+            arn = f'arn:aws:iam::{account_id}:policy/{policy_name}'
+            client = IAM.client(account_id)
+
+            # List all policy versions
+            paginator = client.get_paginator('list_policy_versions')
+            pages = paginator.paginate(
+                PolicyArn=arn
+            )
+            versions = []
+            for page in pages:
+                versions += page['Versions']
+            non_default_versions = [version['VersionId'] for version in versions if version['IsDefaultVersion'] is False]
+            # Delete all non-default versions
+            for version_id in non_default_versions:
+                client.delete_policy_version(PolicyArn=arn, VersionId=version_id)
+
+            return True
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(f'Data.all Environment Pivot Role does not have permissions to get policy {policy_name}: {e}')
+            log.error(f'Failed to get policy {policy_name} : {e}')
+            return None, None
+
+    @staticmethod
+    def is_policy_attached(
+            account_id: str,
+            policy_name: str,
+            role_name: str
+    ):
+        try:
+            client = IAM.client(account_id)
+            paginator = client.get_paginator('list_attached_role_policies')
+            pages = paginator.paginate(
+                RoleName=role_name
             )
-        except Exception as e:
+            policies = []
+            for page in pages:
+                policies += page['AttachedPolicies']
+            return policy_name in [p['PolicyName'] for p in policies]
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to to get the list of attached policies to the role {role_name}: {e}')
             log.error(
-                f'Failed to delete policy {policy_name} of role {role_name} : {e}'
+                f'Failed to get the list of attached policies to the role {role_name}: {e}'
+            )
+            return False
+
+    @staticmethod
+    def attach_role_policy(
+            account_id,
+            role_name,
+            policy_arn
+    ):
+        try:
+            client = IAM.client(account_id)
+            response = client.attach_role_policy(
+                RoleName=role_name,
+                PolicyArn=policy_arn
+            )
+            return True
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to to attach policy {policy_arn} to the role {role_name}: {e}')
+            log.error(
+                f'Failed to attach policy {policy_arn} to the role {role_name}: {e}'
+            )
+            raise e
+
+    @staticmethod
+    def detach_policy_from_role(
+            account_id: str,
+            role_name: str,
+            policy_name: str):
+        try:
+            arn = f'arn:aws:iam::{account_id}:policy/{policy_name}'
+            client = IAM.client(account_id)
+            client.detach_role_policy(
+                RoleName=role_name,
+                PolicyArn=arn
             )
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'AccessDenied':
+                raise Exception(
+                    f'Data.all Environment Pivot Role does not have permissions to detach policy {policy_name} from role {role_name}: {e}')
+            raise Exception(f'Failed to detach policy {policy_name} from role {role_name}: {e}')

backend/dataall/base/utils/naming_convention.py

@@ -6,7 +6,8 @@
 class NamingConventionPattern(Enum):
 
     S3 = {'regex': '[^a-zA-Z0-9-]', 'separator': '-', 'max_length': 63}
-    IAM = {'regex': '[^a-zA-Z0-9-_]', 'separator': '-', 'max_length': 63}
+    IAM = {'regex': '[^a-zA-Z0-9-_]', 'separator': '-', 'max_length': 63}  # Role names up to 64 chars
+    IAM_POLICY = {'regex': '[^a-zA-Z0-9-_]', 'separator': '-', 'max_length': 128}  # Policy names up to 128 chars
     GLUE = {'regex': '[^a-zA-Z0-9_]', 'separator': '_', 'max_length': 240}  # Limit 255 - 15 extra chars buffer
     GLUE_ETL = {'regex': '[^a-zA-Z0-9-]', 'separator': '-', 'max_length': 52}
     NOTEBOOK = {'regex': '[^a-zA-Z0-9-]', 'separator': '-', 'max_length': 63}

backend/dataall/core/environment/api/input_types.py

@@ -106,6 +106,8 @@ class EnvironmentSortField(GraphQLEnumMapper):
         gql.Argument('groupUri', gql.NonNullableType(gql.String)),
         gql.Argument('IAMRoleArn', gql.NonNullableType(gql.String)),
         gql.Argument('environmentUri', gql.NonNullableType(gql.String)),
+        gql.Argument('dataallManaged', gql.NonNullableType(gql.Boolean)),
+
     ],
 )