feat: S3 delete (#86)

* feat: add s3 delete methods

* feat: add test case

Author: wphyojpl

mdps_ds_lib/lib/aws/aws_s3.py

@@ -222,3 +222,55 @@ def read_small_txt_file(self):
         """
         bytestream = BytesIO(self.get_stream().read())  # get the bytes stream of zipped file
         return bytestream.read().decode('UTF-8')
+
+    def delete_one(self):
+        response = self.__s3_client.delete_object(
+            Bucket=self.__target_bucket,
+            Key=self.__target_key,
+            # MFA='string',
+            # VersionId='string',
+            # RequestPayer='requester',
+            # BypassGovernanceRetention=True | False,
+            # ExpectedBucketOwner='string',
+            # IfMatch='string',
+            # IfMatchLastModifiedTime=datetime(2015, 1, 1),
+            # IfMatchSize=123
+        )
+        return response
+
+    def delete_multiple(self, s3_urls: list=[], s3_bucket: str='', s3_paths: list=[]):
+        if len(s3_urls) < 1 and len(s3_paths) < 1:
+            raise ValueError(f'unable to delete empty list of URLs or Paths')
+        if len(s3_urls) < 1:
+            if s3_bucket == '':
+                raise ValueError(f'empty s3 bucket for paths')
+        else:
+            s3_splits = [self.split_s3_url(k) for k in s3_urls]
+            s3_bucket = list(set([k[0] for k in s3_splits]))
+            if len(s3_bucket) > 1:
+                raise ValueError(f'unable to delete multiple s3 buckets: {s3_bucket}')
+            s3_bucket = s3_bucket[0]
+            s3_paths = list(set([k[1] for k in s3_splits]))
+        s3_paths = [{'Key': k,
+                     # 'VersionId': 'string',
+                     # 'ETag': 'string',
+                     # 'LastModifiedTime': datetime(2015, 1, 1),
+                     # 'Size': 123
+                     } for k in s3_paths]
+        response = self.__s3_client.delete_objects(
+            Bucket=s3_bucket,
+            Delete={
+                'Objects': s3_paths,
+                'Quiet': True,  # True | False
+            },
+            # MFA='string',
+            # VersionId='string',
+            # RequestPayer='requester',
+            # BypassGovernanceRetention=True | False,
+            # ExpectedBucketOwner='string',
+            # IfMatch='string',
+            # IfMatchLastModifiedTime=datetime(2015, 1, 1),
+            # IfMatchSize=123
+        )
+        return response
+

tests/mdps_ds_lib/lib/aws/test_aws_s3.py

@@ -0,0 +1,58 @@
+from unittest import TestCase
+
+from mdps_ds_lib.lib.aws.aws_s3 import AwsS3
+
+
+class TestAwsS3(TestCase):
+    def test_delete_one_01(self):
+        bucket = 'uds-sbx-cumulus-staging'
+        s3 = AwsS3()
+        single_path = 'tmp/unit-test/file1.txt'
+        s3.set_s3_url(f's3://{bucket}/{single_path}').upload_bytes('This is a test'.encode())
+        d = s3.delete_one()
+        print(d)
+        return
+
+    def test_delete_multiple_01(self):
+        bucket = 'uds-sbx-cumulus-staging'
+        s3 = AwsS3()
+        deleting_s3_urls = []
+        for i in range(10):
+            single_path = f'tmp/unit-test/file{i}.txt'
+            deleting_s3_urls.append(f's3://{bucket}/{single_path}')
+            s3.set_s3_url(deleting_s3_urls[-1]).upload_bytes(f'This is a test - {i}'.encode())
+        d = s3.delete_multiple(s3_urls=deleting_s3_urls)
+        print(d)
+        return
+
+    def test_delete_multiple_02(self):
+        bucket = 'uds-sbx-cumulus-staging'
+        s3 = AwsS3()
+        deleting_s3_paths = []
+        for i in range(10):
+            single_path = f'tmp/unit-test/file{i}.txt'
+            deleting_s3_paths.append(single_path)
+            s3.set_s3_url(f's3://{bucket}/{single_path}').upload_bytes(f'This is a test - {i}'.encode())
+        d = s3.delete_multiple(s3_bucket=bucket, s3_paths=deleting_s3_paths)
+        print(d)
+        return
+
+    def test_delete_multiple_03(self):
+        bucket = 'uds-sbx-cumulus-staging'
+        s3 = AwsS3()
+        with self.assertRaises(ValueError) as context:
+            s3.delete_multiple(s3_bucket=bucket, s3_paths=[])
+        self.assertTrue(str(context.exception).startswith('unable to delete empty list of URLs or Paths'))
+
+        with self.assertRaises(ValueError) as context:
+            s3.delete_multiple(s3_urls=[])
+        self.assertTrue(str(context.exception).startswith('unable to delete empty list of URLs or Paths'))
+
+        with self.assertRaises(ValueError) as context:
+            s3.delete_multiple(s3_bucket='', s3_paths=['a', 'b', 'c'])
+        self.assertTrue(str(context.exception).startswith('empty s3 bucket for paths'))
+
+        with self.assertRaises(ValueError) as context:
+            s3.delete_multiple(s3_urls=['s3://a/b', 's3://b/c'])
+        self.assertTrue(str(context.exception).startswith('unable to delete multiple s3 buckets'))
+        return