Merge pull request #117 from LEUNGUU/feat/replace-s3-bucketpolicy-wit…

…h-accesspoints

Feature: Change to use s3 access point while sharing folders

backend/dataall/api/Objects/ShareObject/resolvers.py

@@ -84,7 +84,7 @@ def approve_share_object(context: Context, source, shareUri: str = None):
         session.add(approve_share_task)
 
     # call cdk to update bucket policy of the dataset for folder shares
-    stack_helper.deploy_stack(context, share.datasetUri)
+    # stack_helper.deploy_stack(context, share.datasetUri)
 
     Worker.queue(engine=context.engine, task_ids=[approve_share_task.taskUri])
 
@@ -109,7 +109,7 @@ def reject_share_object(context: Context, source, shareUri: str = None):
         )
         session.add(reject_share_task)
 
-    stack_helper.deploy_stack(context, share.datasetUri)
+    # stack_helper.deploy_stack(context, share.datasetUri)
 
     Worker.queue(engine=context.engine, task_ids=[reject_share_task.taskUri])
 

backend/dataall/aws/handlers/iam.py

@@ -0,0 +1,70 @@
+import logging
+
+from .sts import SessionHelper
+
+
+log = logging.getLogger(__name__)
+
+
+class IAM:
+    @staticmethod
+    def client(account_id: str):
+        session = SessionHelper.remote_session(account_id)
+        return session.client('iam')
+
+    @staticmethod
+    def update_role_policy(
+        account_id: str,
+        role_name: str,
+        policy_name: str,
+        policy: str,
+    ):
+        try:
+            iamcli = IAM.client(account_id)
+            iamcli.put_role_policy(
+                RoleName=role_name,
+                PolicyName=policy_name,
+                PolicyDocument=policy,
+            )
+        except Exception as e:
+            log.error(
+                f'Failed to add S3 bucket access to target role {account_id}/{role_name} : {e}'
+            )
+            raise e
+
+    @staticmethod
+    def get_role_policy(
+        account_id: str,
+        role_name: str,
+        policy_name: str,
+    ):
+        try:
+            iamcli = IAM.client(account_id)
+            response = iamcli.get_role_policy(
+                RoleName=role_name,
+                PolicyName=policy_name,
+            )
+        except Exception as e:
+            log.error(
+                f'Failed to get policy {policy_name} of role {role_name} : {e}'
+            )
+            return None
+        else:
+            return response["PolicyDocument"]
+
+    @staticmethod
+    def delete_role_policy(
+        account_id: str,
+        role_name: str,
+        policy_name: str,
+    ):
+        try:
+            iamcli = IAM.client(account_id)
+            iamcli.delete_role_policy(
+                RoleName=role_name,
+                PolicyName=policy_name,
+            )
+        except Exception as e:
+            log.error(
+                f'Failed to delete policy {policy_name} of role {role_name} : {e}'
+            )

backend/dataall/aws/handlers/kms.py

@@ -0,0 +1,71 @@
+import logging
+
+from .sts import SessionHelper
+
+log = logging.getLogger(__name__)
+
+
+class KMS:
+
+    @staticmethod
+    def client(account_id: str):
+        session = SessionHelper.remote_session(accountid=account_id)
+        return session.client('kms')
+
+    @staticmethod
+    def put_key_policy(
+        account_id: str,
+        key_id: str,
+        policy_name: str,
+        policy: str,
+    ):
+        try:
+            kms_client = KMS.client(account_id)
+            kms_client.put_key_policy(
+                KeyId=key_id,
+                PolicyName=policy_name,
+                Policy=policy,
+            )
+        except Exception as e:
+            log.error(
+                f'Failed to attach policy to KMS key {key_id} on {account_id} : {e} '
+            )
+            raise e
+
+    @staticmethod
+    def get_key_policy(
+        account_id: str,
+        key_id: str,
+        policy_name: str,
+    ):
+        try:
+            kms_client = KMS.client(account_id)
+            response = kms_client.get_key_policy(
+                KeyId=key_id,
+                PolicyName=policy_name,
+            )
+        except Exception as e:
+            log.error(
+                f'Failed to get kms key policy of key {key_id} : {e}'
+            )
+            return None
+        else:
+            return response['Policy']
+
+    @staticmethod
+    def get_key_id(
+        account_id: str,
+        key_alias: str,
+    ):
+        try:
+            kms_client = KMS.client(account_id)
+            response = kms_client.describe_key(
+                KeyId=key_alias,
+            )
+        except Exception as e:
+            log.error(
+                f'Failed to get kms key id of {key_alias} : {e}'
+            )
+            return None
+        else:
+            return response['KeyMetadata']['KeyId']

backend/dataall/aws/handlers/s3.py

@@ -19,12 +19,16 @@ def create_dataset_location(engine, task: models.Task):
             S3.create_bucket_prefix(location)
             return location
 
+    @staticmethod
+    def client(account_id: str, client_type: str):
+        session = SessionHelper.remote_session(accountid=account_id)
+        return session.client(client_type)
+
     @staticmethod
     def create_bucket_prefix(location):
         try:
             accountid = location.AWSAccountId
-            aws_session = SessionHelper.remote_session(accountid=accountid)
-            s3cli = aws_session.client('s3')
+            s3cli = S3.client(account_id=accountid, client_type='s3')
             response = s3cli.put_object(
                 Bucket=location.S3BucketName, Body='', Key=location.S3Prefix + '/'
             )
@@ -39,3 +43,155 @@ def create_bucket_prefix(location):
                 f'Dataset storage location creation failed on S3 for dataset location {location.locationUri} : {e}'
             )
             raise e
+
+    @staticmethod
+    def create_bucket_policy(account_id: str, bucket_name: str, policy: str):
+        try:
+            s3cli = S3.client(account_id=account_id, client_type='s3')
+            s3cli.put_bucket_policy(
+                Bucket=bucket_name,
+                Policy=policy,
+                ConfirmRemoveSelfBucketAccess=False,
+                ExpectedBucketOwner=account_id,
+            )
+            log.info(
+                f'Created bucket policy of {bucket_name} on {account_id} successfully'
+            )
+        except Exception as e:
+            log.error(
+                f'Bucket policy created failed on bucket {bucket_name} of {account_id} : {e}'
+            )
+            raise e
+
+    @staticmethod
+    def get_bucket_policy(account_id: str, bucket_name: str):
+        try:
+            s3cli = S3.client(account_id=account_id, client_type='s3')
+            response = s3cli.get_bucket_policy(Bucket=bucket_name, ExpectedBucketOwner=account_id)
+        except Exception as e:
+            log.warning(
+                f'Failed to get bucket policy of {bucket_name} : {e}'
+            )
+            return None
+        else:
+            return response['Policy']
+
+    @staticmethod
+    def get_bucket_access_point_arn(account_id: str, access_point_name: str):
+        try:
+            s3control = S3.client(account_id, 's3control')
+            access_point = s3control.get_access_point(
+                AccountId=account_id,
+                Name=access_point_name,
+            )
+        except Exception as e:
+            log.info(
+                f'Failed to get S3 bucket access point {access_point_name} on {account_id} : {e}'
+            )
+            return None
+        else:
+            return access_point["AccessPointArn"]
+
+    @staticmethod
+    def create_bucket_access_point(account_id: str, bucket_name: str, access_point_name: str):
+        try:
+            s3control = S3.client(account_id, 's3control')
+            access_point = s3control.create_access_point(
+                AccountId=account_id,
+                Name=access_point_name,
+                Bucket=bucket_name,
+            )
+        except Exception as e:
+            log.error(
+                f'S3 bucket access point creation failed for location {bucket_name} : {e}'
+            )
+            raise e
+        else:
+            return access_point["AccessPointArn"]
+
+    @staticmethod
+    def delete_bucket_access_point(account_id: str, access_point_name: str):
+        try:
+            s3control = S3.client(account_id, 's3control')
+            s3control.delete_access_point(
+                AccountId=account_id,
+                Name=access_point_name,
+            )
+        except Exception as e:
+            log.error(
+                f'Failed to delete S3 bucket access point {access_point_name}/{account_id} : {e}'
+            )
+            raise e
+
+    @staticmethod
+    def get_access_point_policy(account_id: str, access_point_name: str):
+        try:
+            s3control = S3.client(account_id, 's3control')
+            response = s3control.get_access_point_policy(
+                AccountId=account_id,
+                Name=access_point_name,
+            )
+        except Exception as e:
+            log.info(
+                f'Failed to get policy of access point {access_point_name} on {account_id} : {e}'
+            )
+            return None
+        else:
+            return response['Policy']
+
+    @staticmethod
+    def attach_access_point_policy(account_id: str, access_point_name: str, policy: str):
+        try:
+            s3control = S3.client(account_id, 's3control')
+            s3control.put_access_point_policy(
+                AccountId=account_id,
+                Name=access_point_name,
+                Policy=policy
+            )
+        except Exception as e:
+            log.error(
+                f'S3 bucket access point policy creation failed : {e}'
+            )
+            raise e
+
+    @staticmethod
+    def generate_access_point_policy_template(
+        principal_id: str,
+        access_point_arn: str,
+        s3_prefix: str,
+    ):
+        policy = {
+            'Version': '2012-10-17',
+            "Statement": [
+                {
+                    "Sid": f"{principal_id}0",
+                    "Effect": "Allow",
+                    "Principal": {
+                        "AWS": "*"
+                    },
+                    "Action": "s3:ListBucket",
+                    "Resource": f"{access_point_arn}",
+                    "Condition": {
+                        "StringLike": {
+                            "s3:prefix": [f"{s3_prefix}/*"],
+                            "aws:userId": [f"{principal_id}:*"]
+                        }
+                    }
+                },
+                {
+                    "Sid": f"{principal_id}1",
+                    "Effect": "Allow",
+                    "Principal": {
+                        "AWS": "*"
+                    },
+                    "Action": "s3:GetObject",
+                    "Resource": [f"{access_point_arn}/object/{s3_prefix}/*"],
+                    "Condition": {
+                        "StringLike": {
+                            "aws:userId": [f"{principal_id}:*"]
+                        }
+                    }
+                }
+            ]
+        }
+        return policy