diff --git a/backend/dataall/modules/dataset_sharing/__init__.py b/backend/dataall/modules/dataset_sharing/__init__.py index cad070ece..bffa8a903 100644 --- a/backend/dataall/modules/dataset_sharing/__init__.py +++ b/backend/dataall/modules/dataset_sharing/__init__.py @@ -24,11 +24,13 @@ def __init__(self): from dataall.modules.dataset_sharing import api from dataall.modules.dataset_sharing.services.managed_share_policy_service import SharePolicyService from dataall.modules.s3_datasets.services.dataset_service import DatasetService + from dataall.modules.datasets_base.services.dataset_list_service import DatasetListService from dataall.modules.dataset_sharing.services.dataset_sharing_service import DatasetSharingService from dataall.modules.dataset_sharing.db.share_object_repositories import ShareEnvironmentResource EnvironmentResourceManager.register(ShareEnvironmentResource()) DatasetService.register(DatasetSharingService()) + DatasetListService.register(DatasetSharingService()) log.info('API of dataset sharing has been imported') diff --git a/backend/dataall/modules/dataset_sharing/db/share_object_repositories.py b/backend/dataall/modules/dataset_sharing/db/share_object_repositories.py index c8ba2eb16..9bbc1723c 100644 --- a/backend/dataall/modules/dataset_sharing/db/share_object_repositories.py +++ b/backend/dataall/modules/dataset_sharing/db/share_object_repositories.py @@ -22,6 +22,7 @@ from dataall.modules.dataset_sharing.db.share_object_models import ShareObjectItem, ShareObject from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, S3Dataset, DatasetBucket +from dataall.modules.datasets_base.db.dataset_models import DatasetBase logger = logging.getLogger(__name__) @@ -1002,10 +1003,10 @@ def delete_shares_with_no_shared_items(session, dataset_uri): def query_user_shared_datasets(session, username, groups) -> Query: share_item_shared_states = ShareItemSM.get_share_item_shared_states() query = ( - session.query(S3Dataset) + session.query(DatasetBase) .outerjoin( ShareObject, - ShareObject.datasetUri == S3Dataset.datasetUri, + ShareObject.datasetUri == DatasetBase.datasetUri, ) .outerjoin(ShareObjectItem, ShareObjectItem.shareUri == ShareObject.shareUri) .filter( @@ -1021,7 +1022,7 @@ def query_user_shared_datasets(session, username, groups) -> Query: ) ) ) - return query.distinct(S3Dataset.datasetUri) + return query.distinct(DatasetBase.datasetUri) @staticmethod def find_dataset_shares(session, dataset_uri): diff --git a/backend/dataall/modules/dataset_sharing/services/dataset_sharing_service.py b/backend/dataall/modules/dataset_sharing/services/dataset_sharing_service.py index 4ee0854b4..3e7fc713a 100644 --- a/backend/dataall/modules/dataset_sharing/services/dataset_sharing_service.py +++ b/backend/dataall/modules/dataset_sharing/services/dataset_sharing_service.py @@ -20,10 +20,9 @@ DELETE_DATASET_FOLDER, CREDENTIALS_DATASET, ) - from dataall.modules.s3_datasets.db.dataset_models import S3Dataset -from dataall.modules.datasets_base.services.datasets_enums import DatasetRole -from dataall.modules.s3_datasets.services.dataset_service import DatasetServiceInterface +from dataall.modules.datasets_base.services.datasets_enums import DatasetRole, DatasetType +from dataall.modules.datasets_base.services.dataset_service_interface import DatasetServiceInterface import logging @@ -32,6 +31,10 @@ class DatasetSharingService(DatasetServiceInterface): + @property + def dataset_type(self): + return DatasetType.S3 + @staticmethod def resolve_additional_dataset_user_role(session, uri, username, groups): """Implemented as part of the DatasetServiceInterface""" diff --git a/backend/dataall/modules/datasets_base/__init__.py b/backend/dataall/modules/datasets_base/__init__.py index a79574166..4e848a957 100644 --- a/backend/dataall/modules/datasets_base/__init__.py +++ b/backend/dataall/modules/datasets_base/__init__.py @@ -6,7 +6,6 @@ class DatasetBaseModuleInterface(ModuleInterface): @staticmethod def is_supported(modes: Set[ImportMode]) -> bool: supported_modes = { - ImportMode.API, ImportMode.CDK, ImportMode.HANDLERS, ImportMode.STACK_UPDATER_TASK, @@ -16,3 +15,15 @@ def is_supported(modes: Set[ImportMode]) -> bool: def __init__(self): import dataall.modules.datasets_base.services.datasets_enums + + +class DatasetBaseApiModuleInterface(ModuleInterface): + """Implements ModuleInterface for MLStudio GraphQl lambda""" + + @classmethod + def is_supported(cls, modes): + return ImportMode.API in modes + + def __init__(self): + import dataall.modules.datasets_base.api + import dataall.modules.datasets_base.services.datasets_enums diff --git a/backend/dataall/modules/datasets_base/api/__init__.py b/backend/dataall/modules/datasets_base/api/__init__.py index e69de29bb..b6bd1a0d2 100644 --- a/backend/dataall/modules/datasets_base/api/__init__.py +++ b/backend/dataall/modules/datasets_base/api/__init__.py @@ -0,0 +1,5 @@ +"""The package defines the schema for Dataset_base lists""" + +from dataall.modules.datasets_base.api import input_types, queries, types, resolvers + +__all__ = ['types', 'input_types', 'queries', 'resolvers'] diff --git a/backend/dataall/modules/datasets_base/api/input_types.py b/backend/dataall/modules/datasets_base/api/input_types.py new file mode 100644 index 000000000..ba7e10a53 --- /dev/null +++ b/backend/dataall/modules/datasets_base/api/input_types.py @@ -0,0 +1,27 @@ +from dataall.base.api import gql +from dataall.base.api.constants import SortDirection +from dataall.modules.datasets_base.services.datasets_enums import DatasetSortField + + +DatasetSortCriteria = gql.InputType( + name='DatasetSortCriteria', + arguments=[ + gql.Argument(name='field', type=gql.NonNullableType(DatasetSortField.toGraphQLEnum())), + gql.Argument(name='direction', type=SortDirection.toGraphQLEnum()), + ], +) + + +DatasetFilter = gql.InputType( + name='DatasetFilter', + arguments=[ + gql.Argument('term', gql.String), + gql.Argument('roles', gql.ArrayType(gql.Ref('DatasetRole'))), + gql.Argument('InProject', gql.String), + gql.Argument('notInProject', gql.String), + gql.Argument('displayArchived', gql.Boolean), + gql.Argument('sort', gql.ArrayType(DatasetSortCriteria)), + gql.Argument('page', gql.Integer), + gql.Argument('pageSize', gql.Integer), + ], +) diff --git a/backend/dataall/modules/datasets_base/api/queries.py b/backend/dataall/modules/datasets_base/api/queries.py new file mode 100644 index 000000000..7a0938a1c --- /dev/null +++ b/backend/dataall/modules/datasets_base/api/queries.py @@ -0,0 +1,13 @@ +from dataall.base.api import gql +from dataall.modules.datasets_base.api.input_types import DatasetFilter +from dataall.modules.datasets_base.api.resolvers import ( + list_all_user_datasets, +) +from dataall.modules.datasets_base.api.types import DatasetBaseSearchResult + +listDatasets = gql.QueryField( + name='listDatasets', + args=[gql.Argument('filter', DatasetFilter)], + type=DatasetBaseSearchResult, + resolver=list_all_user_datasets, +) diff --git a/backend/dataall/modules/datasets_base/api/resolvers.py b/backend/dataall/modules/datasets_base/api/resolvers.py new file mode 100644 index 000000000..a2f872cb1 --- /dev/null +++ b/backend/dataall/modules/datasets_base/api/resolvers.py @@ -0,0 +1,71 @@ +import logging + +from dataall.base.api.context import Context +from dataall.core.environment.services.environment_service import EnvironmentService +from dataall.core.organizations.db.organization_repositories import OrganizationRepository +from dataall.core.stacks.services.stack_service import StackService +from dataall.modules.datasets_base.services.dataset_list_service import DatasetListService +from dataall.modules.datasets_base.services.datasets_enums import DatasetRole +from dataall.modules.datasets_base.db.dataset_models import DatasetBase + +log = logging.getLogger(__name__) + + +def list_all_user_datasets(context: Context, source, filter: dict = None): + if not filter: + filter = {'page': 1, 'pageSize': 5} + return DatasetListService.list_all_user_datasets(filter) + + +def resolve_user_role(context: Context, source: DatasetBase, **kwargs): + if not source: + return None + if source.owner == context.username: + return DatasetRole.Creator.value + elif source.SamlAdminGroupName in context.groups: + return DatasetRole.Admin.value + elif source.stewards in context.groups: + return DatasetRole.DataSteward.value + else: + with context.engine.scoped_session() as session: + other_modules_user_role = DatasetListService.get_other_modules_dataset_user_role( + session, source.datasetUri, context.username, context.groups + ) + if other_modules_user_role is not None: + return other_modules_user_role + return DatasetRole.NoPermission.value + + +def get_dataset_organization(context, source: DatasetBase, **kwargs): + if not source: + return None + with context.engine.scoped_session() as session: + return OrganizationRepository.get_organization_by_uri(session, source.organizationUri) + + +def get_dataset_environment(context, source: DatasetBase, **kwargs): + if not source: + return None + with context.engine.scoped_session() as session: + return EnvironmentService.get_environment_by_uri(session, source.environmentUri) + + +def get_dataset_owners_group(context, source: DatasetBase, **kwargs): + if not source: + return None + return source.SamlAdminGroupName + + +def get_dataset_stewards_group(context, source: DatasetBase, **kwargs): + if not source: + return None + return source.stewards + + +def resolve_dataset_stack(context: Context, source: DatasetBase, **kwargs): + if not source: + return None + return StackService.get_stack_with_cfn_resources( + targetUri=source.datasetUri, + environmentUri=source.environmentUri, + ) diff --git a/backend/dataall/modules/datasets_base/api/types.py b/backend/dataall/modules/datasets_base/api/types.py new file mode 100644 index 000000000..553cd61ec --- /dev/null +++ b/backend/dataall/modules/datasets_base/api/types.py @@ -0,0 +1,79 @@ +from dataall.base.api import gql +from dataall.modules.datasets_base.services.datasets_enums import DatasetRole +from dataall.modules.datasets_base.api.resolvers import ( + get_dataset_environment, + get_dataset_organization, + get_dataset_owners_group, + get_dataset_stewards_group, + resolve_user_role, + resolve_dataset_stack, +) +from dataall.core.environment.api.enums import EnvironmentPermission + +DatasetBase = gql.ObjectType( + name='DatasetBase', + fields=[ + gql.Field(name='datasetUri', type=gql.ID), + gql.Field(name='datasetType', type=gql.String), + gql.Field(name='label', type=gql.String), + gql.Field(name='name', type=gql.String), + gql.Field(name='description', type=gql.String), + gql.Field(name='tags', type=gql.ArrayType(gql.String)), + gql.Field(name='owner', type=gql.String), + gql.Field(name='created', type=gql.String), + gql.Field(name='updated', type=gql.String), + gql.Field(name='admins', type=gql.ArrayType(gql.String)), + gql.Field(name='AwsAccountId', type=gql.String), + gql.Field(name='region', type=gql.String), + gql.Field(name='SamlAdminGroupName', type=gql.String), + gql.Field(name='businessOwnerEmail', type=gql.String), + gql.Field(name='businessOwnerDelegationEmails', type=gql.ArrayType(gql.String)), + gql.Field(name='imported', type=gql.Boolean), + gql.Field( + name='environment', + type=gql.Ref('Environment'), + resolver=get_dataset_environment, + ), + gql.Field( + name='organization', + type=gql.Ref('Organization'), + resolver=get_dataset_organization, + ), + gql.Field( + name='owners', + type=gql.String, + resolver=get_dataset_owners_group, + ), + gql.Field( + name='stewards', + type=gql.String, + resolver=get_dataset_stewards_group, + ), + gql.Field( + name='userRoleForDataset', + type=DatasetRole.toGraphQLEnum(), + resolver=resolve_user_role, + ), + gql.Field(name='userRoleInEnvironment', type=EnvironmentPermission.toGraphQLEnum()), + gql.Field(name='topics', type=gql.ArrayType(gql.Ref('Topic'))), + gql.Field(name='confidentiality', type=gql.String), + gql.Field(name='language', type=gql.Ref('Language')), + gql.Field(name='autoApprovalEnabled', type=gql.Boolean), + gql.Field(name='stack', type=gql.Ref('Stack'), resolver=resolve_dataset_stack), + ], +) + +DatasetBaseSearchResult = gql.ObjectType( + name='DatasetBaseSearchResult', + fields=[ + gql.Field(name='count', type=gql.Integer), + gql.Field(name='nodes', type=gql.ArrayType(DatasetBase)), + gql.Field(name='pageSize', type=gql.Integer), + gql.Field(name='nextPage', type=gql.Integer), + gql.Field(name='pages', type=gql.Integer), + gql.Field(name='page', type=gql.Integer), + gql.Field(name='previousPage', type=gql.Integer), + gql.Field(name='hasNext', type=gql.Boolean), + gql.Field(name='hasPrevious', type=gql.Boolean), + ], +) diff --git a/backend/dataall/modules/datasets_base/db/dataset_repositories.py b/backend/dataall/modules/datasets_base/db/dataset_repositories.py index 76b3dd357..15f4d16dd 100644 --- a/backend/dataall/modules/datasets_base/db/dataset_repositories.py +++ b/backend/dataall/modules/datasets_base/db/dataset_repositories.py @@ -1,4 +1,8 @@ import logging +from typing import List +from sqlalchemy import and_, or_ +from sqlalchemy.orm import Query +from dataall.base.db import paginate from dataall.core.activity.db.activity_models import Activity from dataall.modules.datasets_base.db.dataset_models import DatasetBase, DatasetLock @@ -32,3 +36,40 @@ def update_dataset_activity(session, dataset: DatasetBase, username): ) session.add(activity) session.commit() + + +class DatasetListRepository: + """DAO layer for Listing Datasets in Environments""" + + @staticmethod + def paginated_all_user_datasets(session, username, groups, all_subqueries: List[Query], data=None) -> dict: + return paginate( + query=DatasetListRepository._query_all_user_datasets(session, username, groups, all_subqueries, data), + page=data.get('page', 1), + page_size=data.get('pageSize', 10), + ).to_dict() + + @staticmethod + def _query_all_user_datasets(session, username, groups, all_subqueries: List[Query], filter: dict = None) -> Query: + query = session.query(DatasetBase).filter( + or_( + DatasetBase.owner == username, + DatasetBase.SamlAdminGroupName.in_(groups), + DatasetBase.stewards.in_(groups), + ) + ) + if query.first() is not None: + all_subqueries.append(query) + if len(all_subqueries) == 1: + query = all_subqueries[0] + elif len(all_subqueries) > 1: + query = all_subqueries[0].union(*all_subqueries[1:]) + + if filter and filter.get('term'): + query = query.filter( + or_( + DatasetBase.description.ilike(filter.get('term') + '%%'), + DatasetBase.label.ilike(filter.get('term') + '%%'), + ) + ) + return query.order_by(DatasetBase.label).distinct(DatasetBase.datasetUri, DatasetBase.label) diff --git a/backend/dataall/modules/datasets_base/services/dataset_list_service.py b/backend/dataall/modules/datasets_base/services/dataset_list_service.py new file mode 100644 index 000000000..b4d82f300 --- /dev/null +++ b/backend/dataall/modules/datasets_base/services/dataset_list_service.py @@ -0,0 +1,46 @@ +import logging +from sqlalchemy.orm import Query +from typing import List +from dataall.modules.datasets_base.services.dataset_service_interface import DatasetServiceInterface +from dataall.base.context import get_context +from dataall.modules.datasets_base.db.dataset_repositories import DatasetListRepository + +log = logging.getLogger(__name__) + + +class DatasetListService: + _interfaces: List[DatasetServiceInterface] = [] + + @classmethod + def register(cls, interface: DatasetServiceInterface): + cls._interfaces.append(interface) + + @classmethod + def _list_all_user_interface_datasets(cls, session, username, groups) -> List[Query]: + """All list_datasets from other modules that need to be appended to the list of datasets""" + return [ + query + for interface in cls._interfaces + for query in [interface.append_to_list_user_datasets(session, username, groups)] + if query.first() is not None + ] + + @classmethod + def get_other_modules_dataset_user_role(cls, session, uri, username, groups) -> str: + """All other user role types that might come from other modules""" + for interface in cls._interfaces: + role = interface.resolve_additional_dataset_user_role(session, uri, username, groups) + if role is not None: + return role + return None + + @staticmethod + def list_all_user_datasets(data: dict): + context = get_context() + with context.db_engine.scoped_session() as session: + all_subqueries = DatasetListService._list_all_user_interface_datasets( + session, context.username, context.groups + ) + return DatasetListRepository.paginated_all_user_datasets( + session, context.username, context.groups, all_subqueries, data=data + ) diff --git a/backend/dataall/modules/datasets_base/services/dataset_service_interface.py b/backend/dataall/modules/datasets_base/services/dataset_service_interface.py new file mode 100644 index 000000000..66a83f05b --- /dev/null +++ b/backend/dataall/modules/datasets_base/services/dataset_service_interface.py @@ -0,0 +1,52 @@ +import logging +from abc import ABC, abstractmethod +from dataall.modules.datasets_base.services.datasets_enums import DatasetType + +log = logging.getLogger(__name__) + + +class DatasetServiceInterface(ABC): + """ + Interface for modules that depend on datasets to insert code in datasets and avoid circular dependencies + For example, we might check dataset_shares (in dataset_sharing module) before deleting (datasets_module) + """ + + @property + @abstractmethod + def dataset_type(self) -> DatasetType: ... + + @staticmethod + @abstractmethod + def check_before_delete(session, uri, **kwargs) -> bool: + """Abstract method to be implemented by dependent modules that want to add checks before deletion for dataset objects""" + ... + + @staticmethod + @abstractmethod + def execute_on_delete(session, uri, **kwargs) -> bool: + """Abstract method to be implemented by dependent modules that want to add clean-up actions when a dataset object is deleted""" + ... + + @staticmethod + @abstractmethod + def append_to_list_user_datasets(session, username, groups): + """Abstract method to be implemented by dependent modules that want to add datasets to the list_datasets that list all datasets that the user has access to""" + ... + + @staticmethod + @abstractmethod + def resolve_additional_dataset_user_role(session, uri, username, groups): + """Abstract method to be implemented by dependent modules that want to add new types of user role in relation to a Dataset""" + ... + + @staticmethod + @abstractmethod + def extend_attach_steward_permissions(session, dataset, new_stewards) -> bool: + """Abstract method to be implemented by dependent modules that want to attach additional permissions to Dataset stewards""" + ... + + @staticmethod + @abstractmethod + def extend_delete_steward_permissions(session, dataset, new_stewards) -> bool: + """Abstract method to be implemented by dependent modules that want to attach additional permissions to Dataset stewards""" + ... diff --git a/backend/dataall/modules/s3_datasets/__init__.py b/backend/dataall/modules/s3_datasets/__init__.py index 998e3857b..c828e7993 100644 --- a/backend/dataall/modules/s3_datasets/__init__.py +++ b/backend/dataall/modules/s3_datasets/__init__.py @@ -17,13 +17,13 @@ def is_supported(modes): @staticmethod def depends_on() -> List[Type['ModuleInterface']]: - from dataall.modules.datasets_base import DatasetBaseModuleInterface + from dataall.modules.datasets_base import DatasetBaseApiModuleInterface from dataall.modules.catalog import CatalogApiModuleInterface from dataall.modules.feed import FeedApiModuleInterface from dataall.modules.vote import VoteApiModuleInterface return [ - DatasetBaseModuleInterface, + DatasetBaseApiModuleInterface, CatalogApiModuleInterface, FeedApiModuleInterface, VoteApiModuleInterface, diff --git a/backend/dataall/modules/s3_datasets/api/dataset/input_types.py b/backend/dataall/modules/s3_datasets/api/dataset/input_types.py index ba9c0c495..93395154c 100644 --- a/backend/dataall/modules/s3_datasets/api/dataset/input_types.py +++ b/backend/dataall/modules/s3_datasets/api/dataset/input_types.py @@ -1,6 +1,4 @@ from dataall.base.api import gql -from dataall.base.api.constants import SortDirection -from dataall.modules.datasets_base.services.datasets_enums import DatasetSortField NewDatasetInput = gql.InputType( @@ -41,31 +39,6 @@ ], ) -DatasetSortCriteria = gql.InputType( - name='DatasetSortCriteria', - arguments=[ - gql.Argument(name='field', type=gql.NonNullableType(DatasetSortField.toGraphQLEnum())), - gql.Argument(name='direction', type=SortDirection.toGraphQLEnum()), - ], -) - - -DatasetFilter = gql.InputType( - name='DatasetFilter', - arguments=[ - gql.Argument('term', gql.String), - gql.Argument('roles', gql.ArrayType(gql.Ref('DatasetRole'))), - gql.Argument('InProject', gql.String), - gql.Argument('notInProject', gql.String), - gql.Argument('displayArchived', gql.Boolean), - # gql.Argument("organization", gql.String), - # gql.Argument("environment", gql.String), - gql.Argument('sort', gql.ArrayType(DatasetSortCriteria)), - gql.Argument('page', gql.Integer), - gql.Argument('pageSize', gql.Integer), - ], -) - DatasetPresignedUrlInput = gql.InputType( name='DatasetPresignedUrlInput', arguments=[ diff --git a/backend/dataall/modules/s3_datasets/api/dataset/queries.py b/backend/dataall/modules/s3_datasets/api/dataset/queries.py index 6e688bcd1..e74c70772 100644 --- a/backend/dataall/modules/s3_datasets/api/dataset/queries.py +++ b/backend/dataall/modules/s3_datasets/api/dataset/queries.py @@ -1,8 +1,7 @@ from dataall.base.api import gql -from dataall.modules.s3_datasets.api.dataset.input_types import DatasetFilter +from dataall.modules.datasets_base.api.input_types import DatasetFilter from dataall.modules.s3_datasets.api.dataset.resolvers import ( get_dataset, - list_all_user_datasets, list_owned_datasets, get_dataset_assume_role_url, get_file_upload_presigned_url, @@ -20,14 +19,6 @@ ) -listDatasets = gql.QueryField( - name='listDatasets', - args=[gql.Argument('filter', DatasetFilter)], - type=DatasetSearchResult, - resolver=list_all_user_datasets, - test_scope='Dataset', -) - listOwnedDatasets = gql.QueryField( name='listOwnedDatasets', args=[gql.Argument('filter', DatasetFilter)], diff --git a/backend/dataall/modules/s3_datasets/api/dataset/resolvers.py b/backend/dataall/modules/s3_datasets/api/dataset/resolvers.py index 948a1e250..8d30a7440 100644 --- a/backend/dataall/modules/s3_datasets/api/dataset/resolvers.py +++ b/backend/dataall/modules/s3_datasets/api/dataset/resolvers.py @@ -58,12 +58,6 @@ def get_file_upload_presigned_url(context, source, datasetUri: str = None, input return DatasetService.get_file_upload_presigned_url(uri=datasetUri, data=input) -def list_all_user_datasets(context: Context, source, filter: dict = None): - if not filter: - filter = {'page': 1, 'pageSize': 5} - return DatasetService.list_all_user_datasets(filter) - - def list_owned_datasets(context: Context, source, filter: dict = None): if not filter: filter = {'page': 1, 'pageSize': 5} diff --git a/backend/dataall/modules/s3_datasets/db/dataset_repositories.py b/backend/dataall/modules/s3_datasets/db/dataset_repositories.py index be8e04d4e..9f9a2ba58 100644 --- a/backend/dataall/modules/s3_datasets/db/dataset_repositories.py +++ b/backend/dataall/modules/s3_datasets/db/dataset_repositories.py @@ -10,7 +10,6 @@ from dataall.modules.datasets_base.services.datasets_enums import ConfidentialityClassification, Language from dataall.core.environment.services.environment_resource_manager import EnvironmentResource from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, S3Dataset -from dataall.modules.datasets_base.db.dataset_models import DatasetLock from dataall.base.utils.naming_convention import ( NamingConventionService, NamingConventionPattern, @@ -139,39 +138,6 @@ def _set_dataset_aws_resources(dataset: S3Dataset, data, environment): dataset.GlueDataQualityTriggerName = f'{glue_etl_basename}-dqtrigger' return dataset - @staticmethod - def paginated_all_user_datasets(session, username, groups, all_subqueries, data=None) -> dict: - return paginate( - query=DatasetRepository._query_all_user_datasets(session, username, groups, all_subqueries, data), - page=data.get('page', 1), - page_size=data.get('pageSize', 10), - ).to_dict() - - @staticmethod - def _query_all_user_datasets(session, username, groups, all_subqueries, filter) -> Query: - query = session.query(S3Dataset).filter( - or_( - S3Dataset.owner == username, - S3Dataset.SamlAdminGroupName.in_(groups), - S3Dataset.stewards.in_(groups), - ) - ) - if query.first() is not None: - all_subqueries.append(query) - if len(all_subqueries) == 1: - query = all_subqueries[0] - elif len(all_subqueries) > 1: - query = all_subqueries[0].union(*all_subqueries[1:]) - - if filter and filter.get('term'): - union_query = query.filter( - or_( - S3Dataset.description.ilike(filter.get('term') + '%%'), - S3Dataset.label.ilike(filter.get('term') + '%%'), - ) - ) - return query.order_by(S3Dataset.label).distinct(S3Dataset.datasetUri, S3Dataset.label) - @staticmethod def paginated_dataset_tables(session, uri, data=None) -> dict: query = ( diff --git a/backend/dataall/modules/s3_datasets/services/dataset_service.py b/backend/dataall/modules/s3_datasets/services/dataset_service.py index 3cabbd01f..240f45553 100644 --- a/backend/dataall/modules/s3_datasets/services/dataset_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_service.py @@ -2,7 +2,6 @@ import json import logging from typing import List -from abc import ABC, abstractmethod from dataall.base.aws.quicksight import QuicksightClient from dataall.base.db import exceptions from dataall.base.utils.naming_convention import NamingConventionPattern @@ -45,48 +44,11 @@ from dataall.modules.s3_datasets.db.dataset_models import S3Dataset, DatasetTable from dataall.modules.datasets_base.db.dataset_models import DatasetBase from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_TABLE_READ +from dataall.modules.datasets_base.services.dataset_service_interface import DatasetServiceInterface log = logging.getLogger(__name__) -class DatasetServiceInterface(ABC): - @staticmethod - @abstractmethod - def check_before_delete(session, uri, **kwargs) -> bool: - """Abstract method to be implemented by dependent modules that want to add checks before deletion for dataset objects""" - ... - - @staticmethod - @abstractmethod - def execute_on_delete(session, uri, **kwargs) -> bool: - """Abstract method to be implemented by dependent modules that want to add clean-up actions when a dataset object is deleted""" - ... - - @staticmethod - @abstractmethod - def append_to_list_user_datasets(session, username, groups): - """Abstract method to be implemented by dependent modules that want to add datasets to the list_datasets that list all datasets that the user has access to""" - ... - - @staticmethod - @abstractmethod - def resolve_additional_dataset_user_role(session, uri, username, groups): - """Abstract method to be implemented by dependent modules that want to add new types of user role in relation to a Dataset""" - ... - - @staticmethod - @abstractmethod - def extend_attach_steward_permissions(session, dataset, new_stewards) -> bool: - """Abstract method to be implemented by dependent modules that want to attach additional permissions to Dataset stewards""" - ... - - @staticmethod - @abstractmethod - def extend_delete_steward_permissions(session, dataset, new_stewards) -> bool: - """Abstract method to be implemented by dependent modules that want to attach additional permissions to Dataset stewards""" - ... - - class DatasetService: _interfaces: List[DatasetServiceInterface] = [] @@ -116,16 +78,6 @@ def execute_on_delete(cls, session, uri, **kwargs) -> bool: interface.execute_on_delete(session, uri, **kwargs) return True - @classmethod - def _list_all_user_interface_datasets(cls, session, username, groups) -> List: - """All list_datasets from other modules that need to be appended to the list of datasets""" - return [ - query - for interface in cls._interfaces - for query in [interface.append_to_list_user_datasets(session, username, groups)] - if query.first() is not None - ] - @classmethod def _attach_additional_steward_permissions(cls, session, dataset, new_stewards): """All permissions from other modules that need to be granted to stewards""" @@ -273,15 +225,6 @@ def get_file_upload_presigned_url(uri: str, data: dict): dataset = DatasetRepository.get_dataset_by_uri(session, uri) return S3DatasetClient(dataset).get_file_upload_presigned_url(data) - @staticmethod - def list_all_user_datasets(data: dict): - context = get_context() - with context.db_engine.scoped_session() as session: - all_subqueries = DatasetService._list_all_user_interface_datasets(session, context.username, context.groups) - return DatasetRepository.paginated_all_user_datasets( - session, context.username, context.groups, all_subqueries, data=data - ) - @staticmethod def list_owned_datasets(data: dict): context = get_context() diff --git a/frontend/src/modules/S3_Datasets/components/DatasetListItem.js b/frontend/src/modules/S3_Datasets/components/DatasetListItem.js index e3f72f822..2d24f7af2 100644 --- a/frontend/src/modules/S3_Datasets/components/DatasetListItem.js +++ b/frontend/src/modules/S3_Datasets/components/DatasetListItem.js @@ -10,19 +10,11 @@ import { } from '@mui/material'; import PropTypes from 'prop-types'; import React from 'react'; -import * as BsIcons from 'react-icons/bs'; -import { BsTable } from 'react-icons/bs'; import * as FaIcons from 'react-icons/fa'; import * as FiIcons from 'react-icons/fi'; import { useNavigate } from 'react-router'; import { Link as RouterLink } from 'react-router-dom'; -import { - IconAvatar, - Label, - StackStatus, - UpVotesReadOnly, - useCardStyle -} from 'design'; +import { IconAvatar, Label, StackStatus, useCardStyle } from 'design'; export const DatasetListItem = (props) => { const { dataset } = props; @@ -157,44 +149,6 @@ export const DatasetListItem = (props) => { - - - - - Tables - - - - - {dataset.statistics.tables} - - - - - - - - - Folders - - - - - {dataset.statistics.locations} - - - - { - diff --git a/frontend/src/modules/S3_Datasets/services/listDatasets.js b/frontend/src/modules/S3_Datasets/services/listDatasets.js index a16649215..648f936a0 100644 --- a/frontend/src/modules/S3_Datasets/services/listDatasets.js +++ b/frontend/src/modules/S3_Datasets/services/listDatasets.js @@ -22,7 +22,6 @@ export const listDatasets = ({ filter }) => ({ SamlAdminGroupName userRoleForDataset userRoleInEnvironment - GlueDatabaseName tags topics organization { @@ -38,11 +37,7 @@ export const listDatasets = ({ filter }) => ({ stack { status } - statistics { - tables - locations - upvotes - } + datasetType } } }