diff --git a/backend/dataall/api/Objects/Dataset/resolvers.py b/backend/dataall/api/Objects/Dataset/resolvers.py index 018712aa8..9b191e816 100644 --- a/backend/dataall/api/Objects/Dataset/resolvers.py +++ b/backend/dataall/api/Objects/Dataset/resolvers.py @@ -305,6 +305,9 @@ def sync_tables(context: Context, source, datasetUri: str = None): indexers.upsert_dataset_tables( session=session, es=context.es, datasetUri=dataset.datasetUri ) + indexers.remove_deleted_tables( + session=session, es=context.es, datasetUri=dataset.datasetUri + ) return Dataset.paginated_dataset_tables( session=session, username=context.username, diff --git a/backend/dataall/db/api/dataset.py b/backend/dataall/db/api/dataset.py index 672ab5268..3fe83f24c 100644 --- a/backend/dataall/db/api/dataset.py +++ b/backend/dataall/db/api/dataset.py @@ -287,7 +287,12 @@ def paginated_dataset_tables( ) -> dict: query = ( session.query(models.DatasetTable) - .filter(models.DatasetTable.datasetUri == uri) + .filter( + and_( + models.DatasetTable.datasetUri == uri, + models.DatasetTable.LastGlueTableStatus != 'Deleted', + ) + ) .order_by(models.DatasetTable.created.desc()) ) if data and data.get('term'): diff --git a/backend/dataall/searchproxy/indexers.py b/backend/dataall/searchproxy/indexers.py index 981d8faf7..286f2f9d1 100644 --- a/backend/dataall/searchproxy/indexers.py +++ b/backend/dataall/searchproxy/indexers.py @@ -319,7 +319,12 @@ def upsert_dashboard(session, es, dashboardUri: str): def upsert_dataset_tables(session, es, datasetUri: str): tables = ( session.query(models.DatasetTable) - .filter(models.DatasetTable.datasetUri == datasetUri) + .filter( + and_( + models.DatasetTable.datasetUri == datasetUri, + models.DatasetTable.LastGlueTableStatus != 'Deleted', + ) + ) .all() ) for table in tables: @@ -327,6 +332,22 @@ def upsert_dataset_tables(session, es, datasetUri: str): return tables +def remove_deleted_tables(session, es, datasetUri: str): + tables = ( + session.query(models.DatasetTable) + .filter( + and_( + models.DatasetTable.datasetUri == datasetUri, + models.DatasetTable.LastGlueTableStatus == 'Deleted', + ) + ) + .all() + ) + for table in tables: + delete_doc(es, doc_id=table.tableUri) + return tables + + def upsert_dataset_folders(session, es, datasetUri: str): folders = ( session.query(models.DatasetStorageLocation)