Bugfix 932: Investigate why some shares did not go to failed state, but remained stuck or in-progress

backend/dataall/modules/dataset_sharing/db/share_object_repositories.py

@@ -288,14 +288,18 @@ def update_state(self, session, share_uri, new_state):
             return True
 
     def update_state_single_item(self, session, share_item, new_state):
-        logger.info(f"Updating share item in DB {share_item.shareItemUri} status to {new_state}")
-        ShareObjectRepository.update_share_item_status(
-            session=session,
-            uri=share_item.shareItemUri,
-            status=new_state
-        )
-        self._state = new_state
-        return True
+        try:
+            logger.info(f"Updating share item in DB {share_item.shareItemUri} status to {new_state}")
+            ShareObjectRepository.update_share_item_status(
+                session=session,
+                uri=share_item.shareItemUri,
+                status=new_state
+            )
+            self._state = new_state
+            return True
+        except Exception as e:
+            logger.error("Could not update share item status: ", exc_info=True)
+            raise e
 
     @staticmethod
     def get_share_item_shared_states():

backend/dataall/modules/dataset_sharing/services/dataset_alarm_service.py

@@ -19,11 +19,9 @@ def trigger_table_sharing_failure_alarm(
             target_environment: Environment,
     ):
         log.info('Triggering share failure alarm...')
-        subject = (
-            f'ALARM: DATAALL Table {table.GlueTableName} Sharing Failure Notification'
-        )
+        subject = f'Data.all Share Failure for Table {table.GlueTableName}'[:100]
         message = f"""
-    You are receiving this email because your DATAALL {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to share the table {table.GlueTableName} with Lake Formation.
+    You are receiving this email because your Data.all {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to share the table {table.GlueTableName} with Lake Formation.
 
     Alarm Details:
         - State Change:               	OK -> ALARM
@@ -51,9 +49,9 @@ def trigger_revoke_table_sharing_failure_alarm(
             target_environment: Environment,
     ):
         log.info('Triggering share failure alarm...')
-        subject = f'ALARM: DATAALL Table {table.GlueTableName} Revoking LF permissions Failure Notification'
+        subject = f'Data.all Revoke LF Permissions Failure for Table {table.GlueTableName}'[:100]
         message = f"""
-    You are receiving this email because your DATAALL {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to revoke Lake Formation permissions for table {table.GlueTableName} with Lake Formation.
+    You are receiving this email because your Data.all {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to revoke Lake Formation permissions for table {table.GlueTableName} with Lake Formation.
 
     Alarm Details:
         - State Change:               	OK -> ALARM
@@ -76,11 +74,9 @@ def trigger_revoke_table_sharing_failure_alarm(
 
     def trigger_dataset_sync_failure_alarm(self, dataset: Dataset, error: str):
         log.info(f'Triggering dataset {dataset.name} tables sync failure alarm...')
-        subject = (
-            f'ALARM: DATAALL Dataset {dataset.name} Tables Sync Failure Notification'
-        )
+        subject = f'Data.all Dataset Tables Sync Failure for {dataset.name}'[:100]
         message = f"""
-You are receiving this email because your DATAALL {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to synchronize Dataset {dataset.name} tables from AWS Glue to the Search Catalog.
+You are receiving this email because your Data.all {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to synchronize Dataset {dataset.name} tables from AWS Glue to the Search Catalog.
 
 Alarm Details:
     - State Change:               	OK -> ALARM
@@ -101,11 +97,9 @@ def trigger_folder_sharing_failure_alarm(
         target_environment: Environment,
     ):
         log.info('Triggering share failure alarm...')
-        subject = (
-            f'ALARM: DATAALL Folder {folder.S3Prefix} Sharing Failure Notification'
-        )
+        subject = f'Data.all Folder Share Failure for {folder.S3Prefix}'[:100]
         message = f"""
-You are receiving this email because your DATAALL {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to share the folder {folder.S3Prefix} with S3 Access Point.
+You are receiving this email because your Data.all {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to share the folder {folder.S3Prefix} with S3 Access Point.
 Alarm Details:
     - State Change:               	OK -> ALARM
     - Reason for State Change:      S3 Folder sharing failure
@@ -129,11 +123,9 @@ def trigger_revoke_folder_sharing_failure_alarm(
         target_environment: Environment,
     ):
         log.info('Triggering share failure alarm...')
-        subject = (
-            f'ALARM: DATAALL Folder {folder.S3Prefix} Sharing Revoke Failure Notification'
-        )
+        subject = f'Data.all Folder Share Revoke Failure for {folder.S3Prefix}'[:100]
         message = f"""
-You are receiving this email because your DATAALL {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to share the folder {folder.S3Prefix} with S3 Access Point.
+You are receiving this email because your Data.all {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to share the folder {folder.S3Prefix} with S3 Access Point.
 Alarm Details:
     - State Change:               	OK -> ALARM
     - Reason for State Change:      S3 Folder sharing Revoke failure
@@ -173,11 +165,9 @@ def handle_bucket_sharing_failure(self, bucket: DatasetBucket,
                                       target_environment: Environment,
                                       alarm_type: str):
         log.info(f'Triggering {alarm_type} failure alarm...')
-        subject = (
-            f'ALARM: DATAALL S3 Bucket {bucket.S3BucketName} {alarm_type} Failure Notification'
-        )
+        subject = f'Data.all S3 Bucket Failure for {bucket.S3BucketName} {alarm_type}'[:100]
         message = f"""
-You are receiving this email because your DATAALL {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to {alarm_type} the S3 Bucket {bucket.S3BucketName}.
+You are receiving this email because your Data.all {self.envname} environment in the {self.region} region has entered the ALARM state, because it failed to {alarm_type} the S3 Bucket {bucket.S3BucketName}.
 Alarm Details:
     - State Change:               	OK -> ALARM
     - Reason for State Change:      S3 Bucket {alarm_type} failure

backend/dataall/modules/dataset_sharing/services/share_managers/lf_share_manager.py

@@ -480,17 +480,21 @@ def handle_share_failure(
         -------
         True if alarm published successfully
         """
-        logging.error(
-            f'Failed to share table {table.GlueTableName} '
-            f'from source account {self.source_environment.AwsAccountId}//{self.source_environment.region} '
-            f'with target account {self.target_environment.AwsAccountId}/{self.target_environment.region}'
-            f'due to: {error}'
-        )
+        try:
+            logging.error(
+                f'Failed to share table {table.GlueTableName} '
+                f'from source account {self.source_environment.AwsAccountId}//{self.source_environment.region} '
+                f'with target account {self.target_environment.AwsAccountId}/{self.target_environment.region}'
+                f'due to: {error}'
+            )
 
-        DatasetAlarmService().trigger_table_sharing_failure_alarm(
-            table, self.share, self.target_environment
-        )
-        return True
+            DatasetAlarmService().trigger_table_sharing_failure_alarm(
+                table, self.share, self.target_environment
+            )
+            return True
+        except Exception as e:
+            logger.error("Could not process dataset alarms: ", exc_info=True)
+            return False
 
     def handle_revoke_failure(
             self,
@@ -504,16 +508,20 @@ def handle_revoke_failure(
         -------
         True if alarm published successfully
         """
-        logger.error(
-            f'Failed to revoke S3 permissions to table {table.GlueTableName} '
-            f'from source account {self.source_environment.AwsAccountId}//{self.source_environment.region} '
-            f'with target account {self.target_environment.AwsAccountId}/{self.target_environment.region} '
-            f'due to: {error}'
-        )
-        DatasetAlarmService().trigger_revoke_table_sharing_failure_alarm(
-            table, self.share, self.target_environment
-        )
-        return True
+        try:
+            logger.error(
+                f'Failed to revoke S3 permissions to table {table.GlueTableName} '
+                f'from source account {self.source_environment.AwsAccountId}//{self.source_environment.region} '
+                f'with target account {self.target_environment.AwsAccountId}/{self.target_environment.region} '
+                f'due to: {error}'
+            )
+            DatasetAlarmService().trigger_revoke_table_sharing_failure_alarm(
+                table, self.share, self.target_environment
+            )
+            return True
+        except Exception as e:
+            logger.error("Could not process dataset alarms: ", exc_info=True)
+            return False
 
     def glue_client(self):
         return GlueClient(

backend/dataall/modules/dataset_sharing/services/share_managers/s3_access_point_share_manager.py

@@ -510,22 +510,27 @@ def handle_share_failure(self, error: Exception) -> None:
             self.target_folder, self.share, self.target_environment
         )
 
-    def handle_revoke_failure(self, error: Exception) -> None:
+    def handle_revoke_failure(self, error: Exception) -> bool:
         """
         Handles share failure by raising an alarm to alarmsTopic
         Returns
         -------
         True if alarm published successfully
         """
-        logger.error(
-            f'Failed to revoke S3 permissions to folder {self.s3_prefix} '
-            f'from source account {self.source_environment.AwsAccountId}//{self.source_environment.region} '
-            f'with target account {self.target_environment.AwsAccountId}/{self.target_environment.region} '
-            f'due to: {error}'
-        )
-        DatasetAlarmService().trigger_revoke_folder_sharing_failure_alarm(
-            self.target_folder, self.share, self.target_environment
-        )
+        try:
+            logger.error(
+                f'Failed to revoke S3 permissions to folder {self.s3_prefix} '
+                f'from source account {self.source_environment.AwsAccountId}//{self.source_environment.region} '
+                f'with target account {self.target_environment.AwsAccountId}/{self.target_environment.region} '
+                f'due to: {error}'
+            )
+            DatasetAlarmService().trigger_revoke_folder_sharing_failure_alarm(
+                self.target_folder, self.share, self.target_environment
+            )
+            return True
+        except Exception as e:
+            logger.error("Could not process dataset alarms: ", exc_info=True)
+            return False
 
     @staticmethod
     def generate_default_kms_decrypt_policy_statement(target_requester_arn):

backend/dataall/modules/dataset_sharing/services/share_managers/s3_bucket_share_manager.py

@@ -438,16 +438,20 @@ def handle_revoke_failure(self, error: Exception) -> bool:
         -------
         True if alarm published successfully
         """
-        logger.error(
-            f'Failed to revoke S3 permissions to bucket {self.bucket_name} '
-            f'from source account {self.source_environment.AwsAccountId}//{self.source_environment.region} '
-            f'with target account {self.target_environment.AwsAccountId}/{self.target_environment.region} '
-            f'due to: {error}'
-        )
-        DatasetAlarmService().trigger_revoke_folder_sharing_failure_alarm(
-            self.target_bucket, self.share, self.target_environment
-        )
-        return True
+        try:
+            logger.error(
+                f'Failed to revoke S3 permissions to bucket {self.bucket_name} '
+                f'from source account {self.source_environment.AwsAccountId}//{self.source_environment.region} '
+                f'with target account {self.target_environment.AwsAccountId}/{self.target_environment.region} '
+                f'due to: {error}'
+            )
+            DatasetAlarmService().trigger_revoke_s3_bucket_sharing_failure_alarm(
+                self.target_bucket, self.share, self.target_environment
+            )
+            return True
+        except Exception as e:
+            logger.error("Could not process dataset alarms: ", exc_info=True)
+            return False
 
     @staticmethod
     def generate_default_bucket_read_policy_statement(s3_bucket_name, target_requester_arn):

backend/dataall/modules/dataset_sharing/services/share_processors/s3_bucket_process_share.py

@@ -161,9 +161,9 @@ def process_revoked_shares(
                 revoked_item_SM.update_state_single_item(session, removing_item, new_state)
 
             except Exception as e:
-                removing_bucket.handle_revoke_failure(e)
                 new_state = revoked_item_SM.run_transition(ShareItemActions.Failure.value)
                 revoked_item_SM.update_state_single_item(session, removing_item, new_state)
                 success = False
+                removing_bucket.handle_revoke_failure(e)
 
         return success