From 021cd89c072bb49ded07edee88c8de91ac3a9e89 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Fri, 22 Nov 2024 14:21:43 +0100 Subject: [PATCH 1/5] Remove unused function --- .../s3_datasets/services/dataset_profiling_service.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/backend/dataall/modules/s3_datasets/services/dataset_profiling_service.py b/backend/dataall/modules/s3_datasets/services/dataset_profiling_service.py index 063d30959..a19eaffa2 100644 --- a/backend/dataall/modules/s3_datasets/services/dataset_profiling_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_profiling_service.py @@ -67,13 +67,6 @@ def resolve_profiling_run_status(run_uri): session.add(task) Worker.queue(engine=context.db_engine, task_ids=[task.taskUri]) - @staticmethod - @ResourcePolicyService.has_resource_permission(GET_DATASET) - @is_feature_enabled('modules.s3_datasets.features.metrics_data') - def list_profiling_runs(uri): - with get_context().db_engine.scoped_session() as session: - return DatasetProfilingRepository.list_profiling_runs(session, uri) - @classmethod @is_feature_enabled('modules.s3_datasets.features.metrics_data') def get_dataset_table_profiling_run(cls, uri: str): From 6c011627e6ef6b048822fa19448488f8e1954c1f Mon Sep 17 00:00:00 2001 From: dlpzx Date: Fri, 22 Nov 2024 14:27:09 +0100 Subject: [PATCH 2/5] Remove unused function --- .../modules/s3_datasets/services/dataset_table_service.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/backend/dataall/modules/s3_datasets/services/dataset_table_service.py b/backend/dataall/modules/s3_datasets/services/dataset_table_service.py index 386ab252e..5efffe767 100644 --- a/backend/dataall/modules/s3_datasets/services/dataset_table_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_table_service.py @@ -39,11 +39,6 @@ def _get_dataset_uri(session, table_uri): table = DatasetTableRepository.get_dataset_table_by_uri(session, table_uri) return table.datasetUri - @staticmethod - def get_table(uri: str): - with get_context().db_engine.scoped_session() as session: - return DatasetTableRepository.get_dataset_table_by_uri(session, uri) - @staticmethod @TenantPolicyService.has_tenant_permission(MANAGE_DATASETS) @ResourcePolicyService.has_resource_permission(UPDATE_DATASET_TABLE, parent_resource=_get_dataset_uri) From ae854e040dc926633e716c38480ad5536b87684c Mon Sep 17 00:00:00 2001 From: dlpzx Date: Fri, 22 Nov 2024 16:06:01 +0100 Subject: [PATCH 3/5] Add LIST_ENVIRONMENT_DATASETS permission for listing shared datasets --- .../s3_datasets_shares/api/resolvers.py | 6 ++--- .../services/s3_share_service.py | 22 +++++++++++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/backend/dataall/modules/s3_datasets_shares/api/resolvers.py b/backend/dataall/modules/s3_datasets_shares/api/resolvers.py index b531cc29d..7f171acb9 100644 --- a/backend/dataall/modules/s3_datasets_shares/api/resolvers.py +++ b/backend/dataall/modules/s3_datasets_shares/api/resolvers.py @@ -65,13 +65,11 @@ def get_s3_consumption_data(context: Context, source, shareUri: str): def list_shared_databases_tables_with_env_group(context: Context, source, environmentUri: str, groupUri: str): - return S3ShareService.list_shared_databases_tables_with_env_group(environmentUri=environmentUri, groupUri=groupUri) + return S3ShareService.list_shared_databases_tables_with_env_group(uri=environmentUri, group_uri=groupUri) def resolve_shared_db_name(context: Context, source, **kwargs): - return S3ShareService.resolve_shared_db_name( - source.GlueDatabaseName, source.shareUri, source.targetEnvAwsAccountId, source.targetEnvRegion - ) + return S3ShareService.resolve_shared_db_name(source.GlueDatabaseName, source.shareUri) def list_shared_table_columns(context: Context, source, tableUri: str, shareUri: str, filter: dict): diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_service.py index 309750265..4d28686f4 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_service.py @@ -1,6 +1,6 @@ import logging -from dataall.base.db import utils +from dataall.base.db import utils, exceptions from dataall.base.context import get_context from dataall.base.aws.sts import SessionHelper from dataall.base.aws.iam import IAM @@ -10,6 +10,7 @@ from dataall.core.tasks.db.task_models import Task from dataall.core.tasks.service_handlers import Worker from dataall.modules.datasets_base.db.dataset_repositories import DatasetBaseRepository +from dataall.modules.datasets_base.services.dataset_list_permissions import LIST_ENVIRONMENT_DATASETS from dataall.modules.shares_base.db.share_object_models import ShareObject from dataall.modules.shares_base.db.share_object_repositories import ShareObjectRepository from dataall.modules.shares_base.db.share_object_item_repositories import ShareObjectItemRepository @@ -173,12 +174,13 @@ def reapply_share_items_for_dataset(uri: str): return True @staticmethod - def list_shared_tables_by_env_dataset(dataset_uri: str, env_uri: str): + @ResourcePolicyService.has_resource_permission(LIST_ENVIRONMENT_DATASETS) + def list_shared_tables_by_env_dataset(uri: str, dataset_uri: str): context = get_context() with context.db_engine.scoped_session() as session: log.info( S3ShareObjectRepository.query_dataset_tables_shared_with_env( - session, env_uri, dataset_uri, context.username, context.groups + session, uri, dataset_uri, context.username, context.groups ) ) return [ @@ -188,7 +190,7 @@ def list_shared_tables_by_env_dataset(dataset_uri: str, env_uri: str): + (f'_{res.resourceLinkSuffix}' if res.resourceLinkSuffix else ''), } for res in S3ShareObjectRepository.query_dataset_tables_shared_with_env( - session, env_uri, dataset_uri, context.username, context.groups + session, uri, dataset_uri, context.username, context.groups ) ] @@ -259,11 +261,17 @@ def get_s3_consumption_data(uri): } @staticmethod - def list_shared_databases_tables_with_env_group(environmentUri: str, groupUri: str): + @ResourcePolicyService.has_resource_permission(LIST_ENVIRONMENT_DATASETS) + def list_shared_databases_tables_with_env_group(uri: str, group_uri: str): context = get_context() + if group_uri not in context.groups: + raise exceptions.UnauthorizedOperation( + action='LIST_ENVIRONMENT_GROUP_DATASETS', + message=f'User: {context.username} is not a member of the team {group_uri}', + ) with context.db_engine.scoped_session() as session: return S3ShareObjectRepository.query_shared_glue_databases( - session=session, groups=context.groups, env_uri=environmentUri, group_uri=groupUri + session=session, groups=context.groups, env_uri=uri, group_uri=group_uri ) @staticmethod @@ -303,7 +311,7 @@ def list_table_data_filters_by_attached(uri: str, data: dict): ) @staticmethod - def resolve_shared_db_name(GlueDatabaseName: str, shareUri: str, targetEnvAwsAccountId: str, targetEnvRegion: str): + def resolve_shared_db_name(GlueDatabaseName: str, shareUri: str): with get_context().db_engine.scoped_session() as session: share = ShareObjectRepository.get_share_by_uri(session, shareUri) dataset = DatasetBaseRepository.get_dataset_by_uri(session, share.datasetUri) From 2ed5b834f1f359d70d98687715471aaa5b518b57 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Thu, 28 Nov 2024 09:10:15 +0100 Subject: [PATCH 4/5] Revert "Remove unused function" This reverts commit 6c011627e6ef6b048822fa19448488f8e1954c1f. --- .../modules/s3_datasets/services/dataset_table_service.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/dataall/modules/s3_datasets/services/dataset_table_service.py b/backend/dataall/modules/s3_datasets/services/dataset_table_service.py index 5efffe767..386ab252e 100644 --- a/backend/dataall/modules/s3_datasets/services/dataset_table_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_table_service.py @@ -39,6 +39,11 @@ def _get_dataset_uri(session, table_uri): table = DatasetTableRepository.get_dataset_table_by_uri(session, table_uri) return table.datasetUri + @staticmethod + def get_table(uri: str): + with get_context().db_engine.scoped_session() as session: + return DatasetTableRepository.get_dataset_table_by_uri(session, uri) + @staticmethod @TenantPolicyService.has_tenant_permission(MANAGE_DATASETS) @ResourcePolicyService.has_resource_permission(UPDATE_DATASET_TABLE, parent_resource=_get_dataset_uri) From aa6ff46caabed6e0b542224a25b83a998f6b683b Mon Sep 17 00:00:00 2001 From: dlpzx Date: Thu, 28 Nov 2024 09:12:24 +0100 Subject: [PATCH 5/5] PR changes --- .../s3_datasets/db/dataset_profiling_repositories.py | 11 ----------- .../s3_datasets_shares/services/s3_share_service.py | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/backend/dataall/modules/s3_datasets/db/dataset_profiling_repositories.py b/backend/dataall/modules/s3_datasets/db/dataset_profiling_repositories.py index 001fcb1b6..6676d42ee 100644 --- a/backend/dataall/modules/s3_datasets/db/dataset_profiling_repositories.py +++ b/backend/dataall/modules/s3_datasets/db/dataset_profiling_repositories.py @@ -50,17 +50,6 @@ def get_profiling_run(session, profiling_run_uri=None, glue_job_run_id=None, glu ) return run - @staticmethod - def list_profiling_runs(session, dataset_uri): - # TODO filter is always default - filter = {} - q = ( - session.query(DatasetProfilingRun) - .filter(DatasetProfilingRun.datasetUri == dataset_uri) - .order_by(DatasetProfilingRun.created.desc()) - ) - return paginate(q, page=filter.get('page', 1), page_size=filter.get('pageSize', 20)).to_dict() - @staticmethod def list_table_profiling_runs(session, table_uri): # TODO filter is always default diff --git a/backend/dataall/modules/s3_datasets_shares/services/s3_share_service.py b/backend/dataall/modules/s3_datasets_shares/services/s3_share_service.py index 4d28686f4..e7a1a27f1 100644 --- a/backend/dataall/modules/s3_datasets_shares/services/s3_share_service.py +++ b/backend/dataall/modules/s3_datasets_shares/services/s3_share_service.py @@ -267,7 +267,7 @@ def list_shared_databases_tables_with_env_group(uri: str, group_uri: str): if group_uri not in context.groups: raise exceptions.UnauthorizedOperation( action='LIST_ENVIRONMENT_GROUP_DATASETS', - message=f'User: {context.username} is not a member of the team {group_uri}', + message=f'User: {context.username} is not a member of the owner team', ) with context.db_engine.scoped_session() as session: return S3ShareObjectRepository.query_shared_glue_databases(