Generic dataset module and specific s3_datasets module - part 5 (Move DatasetServiceInterface to datasets_base, add property, create first list API for datasets_base)

backend/dataall/modules/dataset_sharing/__init__.py

@@ -24,11 +24,13 @@ def __init__(self):
         from dataall.modules.dataset_sharing import api
         from dataall.modules.dataset_sharing.services.managed_share_policy_service import SharePolicyService
         from dataall.modules.s3_datasets.services.dataset_service import DatasetService
+        from dataall.modules.datasets_base.services.dataset_list_service import DatasetListService
         from dataall.modules.dataset_sharing.services.dataset_sharing_service import DatasetSharingService
         from dataall.modules.dataset_sharing.db.share_object_repositories import ShareEnvironmentResource
 
         EnvironmentResourceManager.register(ShareEnvironmentResource())
         DatasetService.register(DatasetSharingService())
+        DatasetListService.register(DatasetSharingService())
         log.info('API of dataset sharing has been imported')
 
 

backend/dataall/modules/dataset_sharing/db/share_object_repositories.py

@@ -22,6 +22,7 @@
 from dataall.modules.dataset_sharing.db.share_object_models import ShareObjectItem, ShareObject
 from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository
 from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, S3Dataset, DatasetBucket
+from dataall.modules.datasets_base.db.dataset_models import DatasetBase
 
 logger = logging.getLogger(__name__)
 
@@ -1002,10 +1003,10 @@ def delete_shares_with_no_shared_items(session, dataset_uri):
     def query_user_shared_datasets(session, username, groups) -> Query:
         share_item_shared_states = ShareItemSM.get_share_item_shared_states()
         query = (
-            session.query(S3Dataset)
+            session.query(DatasetBase)
             .outerjoin(
                 ShareObject,
-                ShareObject.datasetUri == S3Dataset.datasetUri,
+                ShareObject.datasetUri == DatasetBase.datasetUri,
             )
             .outerjoin(ShareObjectItem, ShareObjectItem.shareUri == ShareObject.shareUri)
             .filter(
@@ -1021,7 +1022,7 @@ def query_user_shared_datasets(session, username, groups) -> Query:
                 )
             )
         )
-        return query.distinct(S3Dataset.datasetUri)
+        return query.distinct(DatasetBase.datasetUri)
 
     @staticmethod
     def find_dataset_shares(session, dataset_uri):

backend/dataall/modules/dataset_sharing/services/dataset_sharing_service.py

@@ -20,10 +20,9 @@
     DELETE_DATASET_FOLDER,
     CREDENTIALS_DATASET,
 )
-
 from dataall.modules.s3_datasets.db.dataset_models import S3Dataset
-from dataall.modules.datasets_base.services.datasets_enums import DatasetRole
-from dataall.modules.s3_datasets.services.dataset_service import DatasetServiceInterface
+from dataall.modules.datasets_base.services.datasets_enums import DatasetRole, DatasetType
+from dataall.modules.datasets_base.services.dataset_service_interface import DatasetServiceInterface
 
 
 import logging
@@ -32,6 +31,10 @@
 
 
 class DatasetSharingService(DatasetServiceInterface):
+    @property
+    def dataset_type(self):
+        return DatasetType.S3
+
     @staticmethod
     def resolve_additional_dataset_user_role(session, uri, username, groups):
         """Implemented as part of the DatasetServiceInterface"""

backend/dataall/modules/datasets_base/__init__.py

@@ -6,7 +6,6 @@ class DatasetBaseModuleInterface(ModuleInterface):
     @staticmethod
     def is_supported(modes: Set[ImportMode]) -> bool:
         supported_modes = {
-            ImportMode.API,
             ImportMode.CDK,
             ImportMode.HANDLERS,
             ImportMode.STACK_UPDATER_TASK,
@@ -16,3 +15,15 @@ def is_supported(modes: Set[ImportMode]) -> bool:
 
     def __init__(self):
         import dataall.modules.datasets_base.services.datasets_enums
+
+
+class DatasetBaseApiModuleInterface(ModuleInterface):
+    """Implements ModuleInterface for MLStudio GraphQl lambda"""
+
+    @classmethod
+    def is_supported(cls, modes):
+        return ImportMode.API in modes
+
+    def __init__(self):
+        import dataall.modules.datasets_base.api
+        import dataall.modules.datasets_base.services.datasets_enums

backend/dataall/modules/datasets_base/api/__init__.py

@@ -0,0 +1,5 @@
+"""The package defines the schema for Dataset_base lists"""
+
+from dataall.modules.datasets_base.api import input_types, queries, types, resolvers
+
+__all__ = ['types', 'input_types', 'queries', 'resolvers']

backend/dataall/modules/datasets_base/api/input_types.py

@@ -0,0 +1,27 @@
+from dataall.base.api import gql
+from dataall.base.api.constants import SortDirection
+from dataall.modules.datasets_base.services.datasets_enums import DatasetSortField
+
+
+DatasetSortCriteria = gql.InputType(
+    name='DatasetSortCriteria',
+    arguments=[
+        gql.Argument(name='field', type=gql.NonNullableType(DatasetSortField.toGraphQLEnum())),
+        gql.Argument(name='direction', type=SortDirection.toGraphQLEnum()),
+    ],
+)
+
+
+DatasetFilter = gql.InputType(
+    name='DatasetFilter',
+    arguments=[
+        gql.Argument('term', gql.String),
+        gql.Argument('roles', gql.ArrayType(gql.Ref('DatasetRole'))),
+        gql.Argument('InProject', gql.String),
+        gql.Argument('notInProject', gql.String),
+        gql.Argument('displayArchived', gql.Boolean),
+        gql.Argument('sort', gql.ArrayType(DatasetSortCriteria)),
+        gql.Argument('page', gql.Integer),
+        gql.Argument('pageSize', gql.Integer),
+    ],
+)

backend/dataall/modules/datasets_base/api/queries.py

@@ -0,0 +1,13 @@
+from dataall.base.api import gql
+from dataall.modules.datasets_base.api.input_types import DatasetFilter
+from dataall.modules.datasets_base.api.resolvers import (
+    list_all_user_datasets,
+)
+from dataall.modules.datasets_base.api.types import DatasetBaseSearchResult
+
+listDatasets = gql.QueryField(
+    name='listDatasets',
+    args=[gql.Argument('filter', DatasetFilter)],
+    type=DatasetBaseSearchResult,
+    resolver=list_all_user_datasets,
+)

backend/dataall/modules/datasets_base/api/resolvers.py

@@ -0,0 +1,71 @@
+import logging
+
+from dataall.base.api.context import Context
+from dataall.core.environment.services.environment_service import EnvironmentService
+from dataall.core.organizations.db.organization_repositories import OrganizationRepository
+from dataall.core.stacks.services.stack_service import StackService
+from dataall.modules.datasets_base.services.dataset_list_service import DatasetListService
+from dataall.modules.datasets_base.services.datasets_enums import DatasetRole
+from dataall.modules.datasets_base.db.dataset_models import DatasetBase
+
+log = logging.getLogger(__name__)
+
+
+def list_all_user_datasets(context: Context, source, filter: dict = None):
+    if not filter:
+        filter = {'page': 1, 'pageSize': 5}
+    return DatasetListService.list_all_user_datasets(filter)
+
+
+def resolve_user_role(context: Context, source: DatasetBase, **kwargs):
+    if not source:
+        return None
+    if source.owner == context.username:
+        return DatasetRole.Creator.value
+    elif source.SamlAdminGroupName in context.groups:
+        return DatasetRole.Admin.value
+    elif source.stewards in context.groups:
+        return DatasetRole.DataSteward.value
+    else:
+        with context.engine.scoped_session() as session:
+            other_modules_user_role = DatasetListService.get_other_modules_dataset_user_role(
+                session, source.datasetUri, context.username, context.groups
+            )
+            if other_modules_user_role is not None:
+                return other_modules_user_role
+    return DatasetRole.NoPermission.value
+
+
+def get_dataset_organization(context, source: DatasetBase, **kwargs):
+    if not source:
+        return None
+    with context.engine.scoped_session() as session:
+        return OrganizationRepository.get_organization_by_uri(session, source.organizationUri)
+
+
+def get_dataset_environment(context, source: DatasetBase, **kwargs):
+    if not source:
+        return None
+    with context.engine.scoped_session() as session:
+        return EnvironmentService.get_environment_by_uri(session, source.environmentUri)
+
+
+def get_dataset_owners_group(context, source: DatasetBase, **kwargs):
+    if not source:
+        return None
+    return source.SamlAdminGroupName
+
+
+def get_dataset_stewards_group(context, source: DatasetBase, **kwargs):
+    if not source:
+        return None
+    return source.stewards
+
+
+def resolve_dataset_stack(context: Context, source: DatasetBase, **kwargs):
+    if not source:
+        return None
+    return StackService.get_stack_with_cfn_resources(
+        targetUri=source.datasetUri,
+        environmentUri=source.environmentUri,
+    )

backend/dataall/modules/datasets_base/api/types.py

@@ -0,0 +1,79 @@
+from dataall.base.api import gql
+from dataall.modules.datasets_base.services.datasets_enums import DatasetRole
+from dataall.modules.datasets_base.api.resolvers import (
+    get_dataset_environment,
+    get_dataset_organization,
+    get_dataset_owners_group,
+    get_dataset_stewards_group,
+    resolve_user_role,
+    resolve_dataset_stack,
+)
+from dataall.core.environment.api.enums import EnvironmentPermission
+
+DatasetBase = gql.ObjectType(
+    name='DatasetBase',
+    fields=[
+        gql.Field(name='datasetUri', type=gql.ID),
+        gql.Field(name='datasetType', type=gql.String),
+        gql.Field(name='label', type=gql.String),
+        gql.Field(name='name', type=gql.String),
+        gql.Field(name='description', type=gql.String),
+        gql.Field(name='tags', type=gql.ArrayType(gql.String)),
+        gql.Field(name='owner', type=gql.String),
+        gql.Field(name='created', type=gql.String),
+        gql.Field(name='updated', type=gql.String),
+        gql.Field(name='admins', type=gql.ArrayType(gql.String)),
+        gql.Field(name='AwsAccountId', type=gql.String),
+        gql.Field(name='region', type=gql.String),
+        gql.Field(name='SamlAdminGroupName', type=gql.String),
+        gql.Field(name='businessOwnerEmail', type=gql.String),
+        gql.Field(name='businessOwnerDelegationEmails', type=gql.ArrayType(gql.String)),
+        gql.Field(name='imported', type=gql.Boolean),
+        gql.Field(
+            name='environment',
+            type=gql.Ref('Environment'),
+            resolver=get_dataset_environment,
+        ),
+        gql.Field(
+            name='organization',
+            type=gql.Ref('Organization'),
+            resolver=get_dataset_organization,
+        ),
+        gql.Field(
+            name='owners',
+            type=gql.String,
+            resolver=get_dataset_owners_group,
+        ),
+        gql.Field(
+            name='stewards',
+            type=gql.String,
+            resolver=get_dataset_stewards_group,
+        ),
+        gql.Field(
+            name='userRoleForDataset',
+            type=DatasetRole.toGraphQLEnum(),
+            resolver=resolve_user_role,
+        ),
+        gql.Field(name='userRoleInEnvironment', type=EnvironmentPermission.toGraphQLEnum()),
+        gql.Field(name='topics', type=gql.ArrayType(gql.Ref('Topic'))),
+        gql.Field(name='confidentiality', type=gql.String),
+        gql.Field(name='language', type=gql.Ref('Language')),
+        gql.Field(name='autoApprovalEnabled', type=gql.Boolean),
+        gql.Field(name='stack', type=gql.Ref('Stack'), resolver=resolve_dataset_stack),
+    ],
+)
+
+DatasetBaseSearchResult = gql.ObjectType(
+    name='DatasetBaseSearchResult',
+    fields=[
+        gql.Field(name='count', type=gql.Integer),
+        gql.Field(name='nodes', type=gql.ArrayType(DatasetBase)),
+        gql.Field(name='pageSize', type=gql.Integer),
+        gql.Field(name='nextPage', type=gql.Integer),
+        gql.Field(name='pages', type=gql.Integer),
+        gql.Field(name='page', type=gql.Integer),
+        gql.Field(name='previousPage', type=gql.Integer),
+        gql.Field(name='hasNext', type=gql.Boolean),
+        gql.Field(name='hasPrevious', type=gql.Boolean),
+    ],
+)

backend/dataall/modules/datasets_base/db/dataset_repositories.py

@@ -1,4 +1,8 @@
 import logging
+from typing import List
+from sqlalchemy import and_, or_
+from sqlalchemy.orm import Query
+from dataall.base.db import paginate
 from dataall.core.activity.db.activity_models import Activity
 from dataall.modules.datasets_base.db.dataset_models import DatasetBase, DatasetLock
 
@@ -32,3 +36,40 @@ def update_dataset_activity(session, dataset: DatasetBase, username):
         )
         session.add(activity)
         session.commit()
+
+
+class DatasetListRepository:
+    """DAO layer for Listing Datasets in Environments"""
+
+    @staticmethod
+    def paginated_all_user_datasets(session, username, groups, all_subqueries: List[Query], data=None) -> dict:
+        return paginate(
+            query=DatasetListRepository._query_all_user_datasets(session, username, groups, all_subqueries, data),
+            page=data.get('page', 1),
+            page_size=data.get('pageSize', 10),
+        ).to_dict()
+
+    @staticmethod
+    def _query_all_user_datasets(session, username, groups, all_subqueries: List[Query], filter: dict = None) -> Query:
+        query = session.query(DatasetBase).filter(
+            or_(
+                DatasetBase.owner == username,
+                DatasetBase.SamlAdminGroupName.in_(groups),
+                DatasetBase.stewards.in_(groups),
+            )
+        )
+        if query.first() is not None:
+            all_subqueries.append(query)
+        if len(all_subqueries) == 1:
+            query = all_subqueries[0]
+        elif len(all_subqueries) > 1:
+            query = all_subqueries[0].union(*all_subqueries[1:])
+
+        if filter and filter.get('term'):
+            query = query.filter(
+                or_(
+                    DatasetBase.description.ilike(filter.get('term') + '%%'),
+                    DatasetBase.label.ilike(filter.get('term') + '%%'),
+                )
+            )
+        return query.order_by(DatasetBase.label).distinct(DatasetBase.datasetUri, DatasetBase.label)

backend/dataall/modules/datasets_base/services/dataset_list_service.py

@@ -0,0 +1,46 @@
+import logging
+from sqlalchemy.orm import Query
+from typing import List
+from dataall.modules.datasets_base.services.dataset_service_interface import DatasetServiceInterface
+from dataall.base.context import get_context
+from dataall.modules.datasets_base.db.dataset_repositories import DatasetListRepository
+
+log = logging.getLogger(__name__)
+
+
+class DatasetListService:
+    _interfaces: List[DatasetServiceInterface] = []
+
+    @classmethod
+    def register(cls, interface: DatasetServiceInterface):
+        cls._interfaces.append(interface)
+
+    @classmethod
+    def _list_all_user_interface_datasets(cls, session, username, groups) -> List[Query]:
+        """All list_datasets from other modules that need to be appended to the list of datasets"""
+        return [
+            query
+            for interface in cls._interfaces
+            for query in [interface.append_to_list_user_datasets(session, username, groups)]
+            if query.first() is not None
+        ]
+
+    @classmethod
+    def get_other_modules_dataset_user_role(cls, session, uri, username, groups) -> str:
+        """All other user role types that might come from other modules"""
+        for interface in cls._interfaces:
+            role = interface.resolve_additional_dataset_user_role(session, uri, username, groups)
+            if role is not None:
+                return role
+        return None
+
+    @staticmethod
+    def list_all_user_datasets(data: dict):
+        context = get_context()
+        with context.db_engine.scoped_session() as session:
+            all_subqueries = DatasetListService._list_all_user_interface_datasets(
+                session, context.username, context.groups
+            )
+            return DatasetListRepository.paginated_all_user_datasets(
+                session, context.username, context.groups, all_subqueries, data=data
+            )