diff --git a/backend/dataall/api/Objects/Dashboard/resolvers.py b/backend/dataall/api/Objects/Dashboard/resolvers.py deleted file mode 100644 index 94372f5d1..000000000 --- a/backend/dataall/api/Objects/Dashboard/resolvers.py +++ /dev/null @@ -1,329 +0,0 @@ -import os -from .... import db -from ....api.constants import DashboardRole -from ....api.context import Context -from ....aws.handlers.quicksight import Quicksight -from ....aws.handlers.parameter_store import ParameterStoreManager -from ....db import permissions, models -from ....db.api import ResourcePolicy, Glossary, Vote -from ....searchproxy import indexers -from ....utils import Parameter -from dataall.searchproxy.indexers import DashboardIndexer - -param_store = Parameter() -ENVNAME = os.getenv("envname", "local") -DOMAIN_NAME = param_store.get_parameter(env=ENVNAME, path="frontend/custom_domain_name") if ENVNAME not in ["local", "dkrcompose"] else None -DOMAIN_URL = f"https://{DOMAIN_NAME}" if DOMAIN_NAME else "http://localhost:8080" - - -def get_quicksight_reader_url(context, source, dashboardUri: str = None): - with context.engine.scoped_session() as session: - dash: models.Dashboard = session.query(models.Dashboard).get(dashboardUri) - env: models.Environment = session.query(models.Environment).get( - dash.environmentUri - ) - ResourcePolicy.check_user_resource_permission( - session=session, - username=context.username, - groups=context.groups, - resource_uri=dash.dashboardUri, - permission_name=permissions.GET_DASHBOARD, - ) - if not env.dashboardsEnabled: - raise db.exceptions.UnauthorizedOperation( - action=permissions.GET_DASHBOARD, - message=f'Dashboards feature is disabled for the environment {env.label}', - ) - if dash.SamlGroupName in context.groups: - url = Quicksight.get_reader_session( - AwsAccountId=env.AwsAccountId, - region=env.region, - UserName=context.username, - DashboardId=dash.DashboardId, - domain_name=DOMAIN_URL, - ) - else: - shared_groups = db.api.Dashboard.query_all_user_groups_shareddashboard( - session=session, - username=context.username, - groups=context.groups, - uri=dashboardUri - ) - if not shared_groups: - raise db.exceptions.UnauthorizedOperation( - action=permissions.GET_DASHBOARD, - message='Dashboard has not been shared with your Teams', - ) - - session_type = ParameterStoreManager.get_parameter_value( - parameter_path=f"/dataall/{os.getenv('envname', 'local')}/quicksight/sharedDashboardsSessions" - ) - - if session_type == 'reader': - url = Quicksight.get_shared_reader_session( - AwsAccountId=env.AwsAccountId, - region=env.region, - UserName=context.username, - GroupName=shared_groups[0], - DashboardId=dash.DashboardId, - ) - else: - url = Quicksight.get_anonymous_session( - AwsAccountId=env.AwsAccountId, - region=env.region, - UserName=context.username, - DashboardId=dash.DashboardId, - ) - return url - - -def get_quicksight_designer_url( - context, source, environmentUri: str = None, dashboardUri: str = None -): - with context.engine.scoped_session() as session: - ResourcePolicy.check_user_resource_permission( - session=session, - username=context.username, - groups=context.groups, - resource_uri=environmentUri, - permission_name=permissions.CREATE_DASHBOARD, - ) - env: models.Environment = session.query(models.Environment).get(environmentUri) - if not env.dashboardsEnabled: - raise db.exceptions.UnauthorizedOperation( - action=permissions.CREATE_DASHBOARD, - message=f'Dashboards feature is disabled for the environment {env.label}', - ) - - url = Quicksight.get_author_session( - AwsAccountId=env.AwsAccountId, - region=env.region, - UserName=context.username, - UserRole='AUTHOR', - ) - - return url - - -def import_dashboard(context: Context, source, input: dict = None): - with context.engine.scoped_session() as session: - ResourcePolicy.check_user_resource_permission( - session=session, - username=context.username, - groups=context.groups, - resource_uri=input['environmentUri'], - permission_name=permissions.CREATE_DASHBOARD, - ) - env: models.Environment = db.api.Environment.get_environment_by_uri( - session, input['environmentUri'] - ) - - if not env.dashboardsEnabled: - raise db.exceptions.UnauthorizedOperation( - action=permissions.CREATE_DASHBOARD, - message=f'Dashboards feature is disabled for the environment {env.label}', - ) - - can_import = Quicksight.can_import_dashboard( - AwsAccountId=env.AwsAccountId, - region=env.region, - UserName=context.username, - DashboardId=input.get('dashboardId'), - ) - - if not can_import: - raise db.exceptions.UnauthorizedOperation( - action=permissions.CREATE_DASHBOARD, - message=f'User: {context.username} has not AUTHOR rights on quicksight for the environment {env.label}', - ) - - input['environment'] = env - dashboard = db.api.Dashboard.import_dashboard( - session=session, - username=context.username, - groups=context.groups, - uri=env.environmentUri, - data=input, - check_perm=True, - ) - - DashboardIndexer.upsert(session, dashboard_uri=dashboard.dashboardUri) - - return dashboard - - -def update_dashboard(context, source, input: dict = None): - with context.engine.scoped_session() as session: - dashboard = db.api.Dashboard.get_dashboard_by_uri( - session, input['dashboardUri'] - ) - input['dashboard'] = dashboard - db.api.Dashboard.update_dashboard( - session=session, - username=context.username, - groups=context.groups, - uri=dashboard.dashboardUri, - data=input, - check_perm=True, - ) - - DashboardIndexer.upsert(session, dashboard_uri=dashboard.dashboardUri) - - return dashboard - - -def list_dashboards(context: Context, source, filter: dict = None): - if not filter: - filter = {} - with context.engine.scoped_session() as session: - return db.api.Dashboard.paginated_user_dashboards( - session=session, - username=context.username, - groups=context.groups, - uri=None, - data=filter, - check_perm=True, - ) - - -def get_dashboard(context: Context, source, dashboardUri: str = None): - with context.engine.scoped_session() as session: - return db.api.Dashboard.get_dashboard( - session=session, - username=context.username, - groups=context.groups, - uri=dashboardUri, - data=None, - check_perm=True, - ) - - -def resolve_user_role(context: Context, source: models.Dashboard): - if context.username and source.owner == context.username: - return DashboardRole.Creator.value - elif context.groups and source.SamlGroupName in context.groups: - return DashboardRole.Admin.value - return DashboardRole.Shared.value - - -def get_dashboard_organization(context: Context, source: models.Dashboard, **kwargs): - with context.engine.scoped_session() as session: - org = session.query(models.Organization).get(source.organizationUri) - return org - - -def request_dashboard_share( - context: Context, - source: models.Dashboard, - principalId: str = None, - dashboardUri: str = None, -): - with context.engine.scoped_session() as session: - return db.api.Dashboard.request_dashboard_share( - session=session, - username=context.username, - groups=context.groups, - uri=dashboardUri, - data={'principalId': principalId}, - check_perm=True, - ) - - -def approve_dashboard_share( - context: Context, - source: models.Dashboard, - shareUri: str = None, -): - with context.engine.scoped_session() as session: - share = db.api.Dashboard.get_dashboard_share_by_uri(session, shareUri) - dashboard = db.api.Dashboard.get_dashboard_by_uri(session, share.dashboardUri) - return db.api.Dashboard.approve_dashboard_share( - session=session, - username=context.username, - groups=context.groups, - uri=dashboard.dashboardUri, - data={'share': share, 'shareUri': shareUri}, - check_perm=True, - ) - - -def reject_dashboard_share( - context: Context, - source: models.Dashboard, - shareUri: str = None, -): - with context.engine.scoped_session() as session: - share = db.api.Dashboard.get_dashboard_share_by_uri(session, shareUri) - dashboard = db.api.Dashboard.get_dashboard_by_uri(session, share.dashboardUri) - return db.api.Dashboard.reject_dashboard_share( - session=session, - username=context.username, - groups=context.groups, - uri=dashboard.dashboardUri, - data={'share': share, 'shareUri': shareUri}, - check_perm=True, - ) - - -def list_dashboard_shares( - context: Context, - source: models.Dashboard, - dashboardUri: str = None, - filter: dict = None, -): - if not filter: - filter = {} - with context.engine.scoped_session() as session: - return db.api.Dashboard.paginated_dashboard_shares( - session=session, - username=context.username, - groups=context.groups, - uri=dashboardUri, - data=filter, - check_perm=True, - ) - - -def share_dashboard( - context: Context, - source: models.Dashboard, - principalId: str = None, - dashboardUri: str = None, -): - with context.engine.scoped_session() as session: - return db.api.Dashboard.share_dashboard( - session=session, - username=context.username, - groups=context.groups, - uri=dashboardUri, - data={'principalId': principalId}, - check_perm=True, - ) - - -def delete_dashboard(context: Context, source, dashboardUri: str = None): - with context.engine.scoped_session() as session: - db.api.Dashboard.delete_dashboard( - session=session, - username=context.username, - groups=context.groups, - uri=dashboardUri, - data=None, - check_perm=True, - ) - DashboardIndexer.delete_doc(doc_id=dashboardUri) - return True - - -def resolve_glossary_terms(context: Context, source: models.Dashboard, **kwargs): - with context.engine.scoped_session() as session: - return Glossary.get_glossary_terms_links( - session, source.dashboardUri, 'Dashboard' - ) - - -def resolve_upvotes(context: Context, source: models.Dashboard, **kwargs): - with context.engine.scoped_session() as session: - return Vote.count_upvotes( - session, None, None, source.dashboardUri, data={'targetType': 'dashboard'} - ) diff --git a/backend/dataall/api/Objects/Environment/input_types.py b/backend/dataall/api/Objects/Environment/input_types.py index c9a6837de..19c9cf103 100644 --- a/backend/dataall/api/Objects/Environment/input_types.py +++ b/backend/dataall/api/Objects/Environment/input_types.py @@ -28,7 +28,6 @@ gql.Argument('description', gql.String), gql.Argument('AwsAccountId', gql.NonNullableType(gql.String)), gql.Argument('region', gql.NonNullableType(gql.String)), - gql.Argument('dashboardsEnabled', type=gql.Boolean), gql.Argument('warehousesEnabled', type=gql.Boolean), gql.Argument('vpcId', gql.String), gql.Argument('privateSubnetIds', gql.ArrayType(gql.String)), @@ -50,7 +49,6 @@ gql.Argument('vpcId', gql.String), gql.Argument('privateSubnetIds', gql.ArrayType(gql.String)), gql.Argument('publicSubnetIds', gql.ArrayType(gql.String)), - gql.Argument('dashboardsEnabled', type=gql.Boolean), gql.Argument('warehousesEnabled', type=gql.Boolean), gql.Argument('resourcePrefix', gql.String), gql.Argument('parameters', gql.ArrayType(ModifyEnvironmentParameterInput)) diff --git a/backend/dataall/api/Objects/Environment/resolvers.py b/backend/dataall/api/Objects/Environment/resolvers.py index af7976e62..903592cf2 100644 --- a/backend/dataall/api/Objects/Environment/resolvers.py +++ b/backend/dataall/api/Objects/Environment/resolvers.py @@ -14,6 +14,7 @@ from ....aws.handlers.cloudformation import CloudFormation from ....aws.handlers.iam import IAM from ....aws.handlers.parameter_store import ParameterStoreManager +from ....core.group.services.environment_resource_manager import EnvironmentResourceManager from ....db import exceptions, permissions from ....db.api import Environment, ResourcePolicy, Stack from ....utils.naming_convention import ( @@ -127,10 +128,10 @@ def update_environment( data=input, check_perm=True, ) - if input.get('dashboardsEnabled') or ( - environment.resourcePrefix != previous_resource_prefix - ): + + if EnvironmentResourceManager.deploy_updated_stack(session, previous_resource_prefix, environment): stack_helper.deploy_stack(targetUri=environment.environmentUri) + return environment diff --git a/backend/dataall/api/Objects/Environment/schema.py b/backend/dataall/api/Objects/Environment/schema.py index 01bb3c9bf..669365a56 100644 --- a/backend/dataall/api/Objects/Environment/schema.py +++ b/backend/dataall/api/Objects/Environment/schema.py @@ -83,7 +83,6 @@ resolver=resolve_user_role, ), gql.Field('validated', type=gql.Boolean), - gql.Field('dashboardsEnabled', type=gql.Boolean), gql.Field('warehousesEnabled', type=gql.Boolean), gql.Field('roleCreated', type=gql.Boolean), gql.Field('isOrganizationDefaultEnvironment', type=gql.Boolean), diff --git a/backend/dataall/api/Objects/Feed/registry.py b/backend/dataall/api/Objects/Feed/registry.py index 085768311..d3a583d09 100644 --- a/backend/dataall/api/Objects/Feed/registry.py +++ b/backend/dataall/api/Objects/Feed/registry.py @@ -34,6 +34,3 @@ def find_target(cls, obj: Resource): @classmethod def types(cls): return [gql.Ref(target_type) for target_type in cls._DEFINITIONS.keys()] - - -FeedRegistry.register(FeedDefinition("Dashboard", models.Dashboard)) diff --git a/backend/dataall/api/Objects/Glossary/registry.py b/backend/dataall/api/Objects/Glossary/registry.py index 3c27f3796..6405f7ed3 100644 --- a/backend/dataall/api/Objects/Glossary/registry.py +++ b/backend/dataall/api/Objects/Glossary/registry.py @@ -3,8 +3,7 @@ from dataall.api import gql from dataall.api.gql.graphql_union_type import UnionTypeRegistry -from dataall.db import Resource, models -from dataall.searchproxy.indexers import DashboardIndexer +from dataall.db import Resource from dataall.searchproxy.base_indexer import BaseIndexer @@ -59,11 +58,3 @@ def reindex(cls, session, target_type: str, target_uri: str): definition = cls._DEFINITIONS[target_type] if definition.reindexer: definition.reindexer.upsert(session, target_uri) - - -GlossaryRegistry.register(GlossaryDefinition( - target_type="Dashboard", - object_type="Dashboard", - model=models.Dashboard, - reindexer=DashboardIndexer -)) diff --git a/backend/dataall/api/Objects/Tenant/mutations.py b/backend/dataall/api/Objects/Tenant/mutations.py index 7f57a5050..f01033c09 100644 --- a/backend/dataall/api/Objects/Tenant/mutations.py +++ b/backend/dataall/api/Objects/Tenant/mutations.py @@ -13,15 +13,6 @@ resolver=update_group_permissions, ) -createQuicksightDataSourceSet = gql.MutationField( - name='createQuicksightDataSourceSet', - args=[ - gql.Argument(name='vpcConnectionId', type=gql.NonNullableType(gql.String)) - ], - type=gql.String, - resolver=create_quicksight_data_source_set, -) - updateSSMParameter = gql.MutationField( name='updateSSMParameter', args=[ diff --git a/backend/dataall/api/Objects/Tenant/queries.py b/backend/dataall/api/Objects/Tenant/queries.py index 62cac727f..4c4d6f089 100644 --- a/backend/dataall/api/Objects/Tenant/queries.py +++ b/backend/dataall/api/Objects/Tenant/queries.py @@ -16,33 +16,3 @@ type=gql.Ref('GroupSearchResult'), resolver=list_tenant_groups, ) - -getMonitoringDashboardId = gql.QueryField( - name='getMonitoringDashboardId', - type=gql.String, - resolver=get_monitoring_dashboard_id, -) - -getMonitoringVpcConnectionId = gql.QueryField( - name='getMonitoringVPCConnectionId', - type=gql.String, - resolver=get_monitoring_vpc_connection_id, -) - -getPlatformAuthorSession = gql.QueryField( - name='getPlatformAuthorSession', - args=[ - gql.Argument(name='awsAccount', type=gql.NonNullableType(gql.String)), - ], - type=gql.String, - resolver=get_quicksight_author_session, -) - -getPlatformReaderSession = gql.QueryField( - name='getPlatformReaderSession', - args=[ - gql.Argument(name='dashboardId', type=gql.NonNullableType(gql.String)), - ], - type=gql.String, - resolver=get_quicksight_reader_session, -) diff --git a/backend/dataall/api/Objects/Tenant/resolvers.py b/backend/dataall/api/Objects/Tenant/resolvers.py index 8bc57be62..7a46239b1 100644 --- a/backend/dataall/api/Objects/Tenant/resolvers.py +++ b/backend/dataall/api/Objects/Tenant/resolvers.py @@ -3,8 +3,6 @@ from .... import db from ....aws.handlers.sts import SessionHelper from ....aws.handlers.parameter_store import ParameterStoreManager -from ....aws.handlers.quicksight import Quicksight -from ....db import exceptions def update_group_permissions(context, source, input=None): @@ -47,86 +45,3 @@ def update_ssm_parameter(context, source, name: str = None, value: str = None): print(name) response = ParameterStoreManager.update_parameter(AwsAccountId=current_account, region=region, parameter_name=f'/dataall/{os.getenv("envname", "local")}/quicksightmonitoring/{name}', parameter_value=value) return response - - -def get_monitoring_dashboard_id(context, source): - current_account = SessionHelper.get_account() - region = os.getenv('AWS_REGION', 'eu-west-1') - dashboard_id = ParameterStoreManager.get_parameter_value(AwsAccountId=current_account, region=region, parameter_path=f'/dataall/{os.getenv("envname", "local")}/quicksightmonitoring/DashboardId') - if not dashboard_id: - raise exceptions.AWSResourceNotFound( - action='GET_DASHBOARD_ID', - message='Dashboard Id could not be found on AWS Parameter Store', - ) - return dashboard_id - - -def get_monitoring_vpc_connection_id(context, source): - current_account = SessionHelper.get_account() - region = os.getenv('AWS_REGION', 'eu-west-1') - vpc_connection_id = ParameterStoreManager.get_parameter_value(AwsAccountId=current_account, region=region, parameter_path=f'/dataall/{os.getenv("envname", "local")}/quicksightmonitoring/VPCConnectionId') - if not vpc_connection_id: - raise exceptions.AWSResourceNotFound( - action='GET_VPC_CONNECTION_ID', - message='Dashboard Id could not be found on AWS Parameter Store', - ) - return vpc_connection_id - - -def create_quicksight_data_source_set(context, source, vpcConnectionId: str = None): - current_account = SessionHelper.get_account() - region = os.getenv('AWS_REGION', 'eu-west-1') - user = Quicksight.register_user_in_group(AwsAccountId=current_account, UserName=context.username, GroupName='dataall', UserRole='AUTHOR') - - datasourceId = Quicksight.create_data_source_vpc(AwsAccountId=current_account, region=region, UserName=context.username, vpcConnectionId=vpcConnectionId) - # Data sets are not created programmatically. Too much overhead for the value added. However, an example API is provided: - # datasets = Quicksight.create_data_set_from_source(AwsAccountId=current_account, region=region, UserName='dataallTenantUser', dataSourceId=datasourceId, tablesToImport=['organization', 'environment', 'dataset', 'datapipeline', 'dashboard', 'share_object']) - - return datasourceId - - -def get_quicksight_author_session(context, source, awsAccount: str = None): - with context.engine.scoped_session() as session: - admin = db.api.TenantPolicy.is_tenant_admin(context.groups) - - if not admin: - raise db.exceptions.TenantUnauthorized( - username=context.username, - action=db.permissions.TENANT_ALL, - tenant_name=context.username, - ) - region = os.getenv('AWS_REGION', 'eu-west-1') - - url = Quicksight.get_author_session( - AwsAccountId=awsAccount, - region=region, - UserName=context.username, - UserRole='AUTHOR', - ) - - return url - - -def get_quicksight_reader_session(context, source, dashboardId: str = None): - with context.engine.scoped_session() as session: - admin = db.api.TenantPolicy.is_tenant_admin(context.groups) - - if not admin: - raise db.exceptions.TenantUnauthorized( - username=context.username, - action=db.permissions.TENANT_ALL, - tenant_name=context.username, - ) - - region = os.getenv('AWS_REGION', 'eu-west-1') - current_account = SessionHelper.get_account() - - url = Quicksight.get_reader_session( - AwsAccountId=current_account, - region=region, - UserName=context.username, - UserRole='READER', - DashboardId=dashboardId - ) - - return url diff --git a/backend/dataall/api/Objects/Vote/resolvers.py b/backend/dataall/api/Objects/Vote/resolvers.py index b9a14e117..5cbcff7c2 100644 --- a/backend/dataall/api/Objects/Vote/resolvers.py +++ b/backend/dataall/api/Objects/Vote/resolvers.py @@ -2,7 +2,6 @@ from dataall import db from dataall.api.context import Context -from dataall.searchproxy.indexers import DashboardIndexer from dataall.searchproxy.base_indexer import BaseIndexer _VOTE_TYPES: Dict[str, Type[BaseIndexer]] = {} @@ -51,7 +50,3 @@ def get_vote(context: Context, source, targetUri: str = None, targetType: str = data={'targetType': targetType}, check_perm=True, ) - - -# TODO should migrate after into the Dashboard module -add_vote_type("dashboard", DashboardIndexer) diff --git a/backend/dataall/api/Objects/__init__.py b/backend/dataall/api/Objects/__init__.py index 52196107b..57fbb8129 100644 --- a/backend/dataall/api/Objects/__init__.py +++ b/backend/dataall/api/Objects/__init__.py @@ -18,7 +18,6 @@ Activity, Group, Principal, - Dashboard, Organization, Stack, Test, diff --git a/backend/dataall/api/constants.py b/backend/dataall/api/constants.py index fdb0f1c05..a8218e8b6 100644 --- a/backend/dataall/api/constants.py +++ b/backend/dataall/api/constants.py @@ -67,13 +67,6 @@ class ProjectMemberRole(GraphQLEnumMapper): NotContributor = '000' -class DashboardRole(GraphQLEnumMapper): - Creator = '999' - Admin = '900' - Shared = '800' - NoPermission = '000' - - class GlossaryRole(GraphQLEnumMapper): # Permissions on a glossary Admin = '900' diff --git a/backend/dataall/aws/handlers/quicksight.py b/backend/dataall/aws/handlers/quicksight.py index 886482f3f..e7c59dfb0 100644 --- a/backend/dataall/aws/handlers/quicksight.py +++ b/backend/dataall/aws/handlers/quicksight.py @@ -1,20 +1,15 @@ import logging import re -import os -import ast -from botocore.exceptions import ClientError from .sts import SessionHelper -from .secrets_manager import SecretsManager -from .parameter_store import ParameterStoreManager logger = logging.getLogger('QuicksightHandler') logger.setLevel(logging.DEBUG) -class Quicksight: +class QuicksightClient: - _DEFAULT_GROUP_NAME = 'dataall' + DEFAULT_GROUP_NAME = 'dataall' def __init__(self): pass @@ -43,10 +38,10 @@ def get_identity_region(AwsAccountId): """ identity_region_rex = re.compile('Please use the (?P.*) endpoint.') identity_region = 'us-east-1' - client = Quicksight.get_quicksight_client(AwsAccountId=AwsAccountId, region=identity_region) + client = QuicksightClient.get_quicksight_client(AwsAccountId=AwsAccountId, region=identity_region) try: response = client.describe_group( - AwsAccountId=AwsAccountId, GroupName=Quicksight._DEFAULT_GROUP_NAME, Namespace='default' + AwsAccountId=AwsAccountId, GroupName=QuicksightClient.DEFAULT_GROUP_NAME, Namespace='default' ) except client.exceptions.AccessDeniedException as e: match = identity_region_rex.findall(str(e)) @@ -66,7 +61,7 @@ def get_quicksight_client_in_identity_region(AwsAccountId): Returns : boto3.client ("quicksight") """ - identity_region = Quicksight.get_identity_region(AwsAccountId) + identity_region = QuicksightClient.get_identity_region(AwsAccountId) session = SessionHelper.remote_session(accountid=AwsAccountId) return session.client('quicksight', region_name=identity_region) @@ -80,7 +75,7 @@ def check_quicksight_enterprise_subscription(AwsAccountId, region=None): True if Quicksight Enterprise Edition is enabled in the AWS Account """ logger.info(f'Checking Quicksight subscription in AWS account = {AwsAccountId}') - client = Quicksight.get_quicksight_client(AwsAccountId=AwsAccountId, region=region) + client = QuicksightClient.get_quicksight_client(AwsAccountId=AwsAccountId, region=region) try: response = client.describe_account_subscription(AwsAccountId=AwsAccountId) if not response['AccountInfo']: @@ -104,7 +99,7 @@ def check_quicksight_enterprise_subscription(AwsAccountId, region=None): return False @staticmethod - def create_quicksight_group(AwsAccountId, GroupName=_DEFAULT_GROUP_NAME): + def create_quicksight_group(AwsAccountId, GroupName=DEFAULT_GROUP_NAME): """Creates a Quicksight group called GroupName Args: AwsAccountId(str): aws account @@ -113,12 +108,12 @@ def create_quicksight_group(AwsAccountId, GroupName=_DEFAULT_GROUP_NAME): Returns:dict quicksight.describe_group response """ - client = Quicksight.get_quicksight_client_in_identity_region(AwsAccountId) - group = Quicksight.describe_group(client, AwsAccountId, GroupName) + client = QuicksightClient.get_quicksight_client_in_identity_region(AwsAccountId) + group = QuicksightClient.describe_group(client, AwsAccountId, GroupName) if not group: - if GroupName == Quicksight._DEFAULT_GROUP_NAME: + if GroupName == QuicksightClient.DEFAULT_GROUP_NAME: logger.info(f'Initializing data.all default group = {GroupName}') - Quicksight.check_quicksight_enterprise_subscription(AwsAccountId) + QuicksightClient.check_quicksight_enterprise_subscription(AwsAccountId) logger.info(f'Attempting to create Quicksight group `{GroupName}...') response = client.create_group( @@ -135,362 +130,18 @@ def create_quicksight_group(AwsAccountId, GroupName=_DEFAULT_GROUP_NAME): return group @staticmethod - def describe_group(client, AwsAccountId, GroupName=_DEFAULT_GROUP_NAME): + def describe_group(client, AwsAccountId, GroupName=DEFAULT_GROUP_NAME): try: response = client.describe_group( AwsAccountId=AwsAccountId, GroupName=GroupName, Namespace='default' ) logger.info( f'Quicksight {GroupName} group already exists in {AwsAccountId} ' - f'(using identity region {Quicksight.get_identity_region(AwsAccountId)}): ' + f'(using identity region {QuicksightClient.get_identity_region(AwsAccountId)}): ' f'{response}' ) return response except client.exceptions.ResourceNotFoundException: logger.info( - f'Creating Quicksight group in {AwsAccountId} (using identity region {Quicksight.get_identity_region(AwsAccountId)})' + f'Creating Quicksight group in {AwsAccountId} (using identity region {QuicksightClient.get_identity_region(AwsAccountId)})' ) - - @staticmethod - def describe_user(AwsAccountId, UserName): - """Describes a QS user, returns None if not found - Args: - AwsAccountId(str) : aws account - UserName(str) : name of the QS user - """ - client = Quicksight.get_quicksight_client_in_identity_region(AwsAccountId) - try: - response = client.describe_user( - UserName=UserName, AwsAccountId=AwsAccountId, Namespace='default' - ) - exists = True - except ClientError: - return None - return response.get('User') - - @staticmethod - def get_quicksight_group_arn(AwsAccountId): - default_group_arn = None - group = Quicksight.describe_group( - client=Quicksight.get_quicksight_client_in_identity_region( - AwsAccountId=AwsAccountId - ), - AwsAccountId=AwsAccountId, - ) - if group and group.get('Group', {}).get('Arn'): - default_group_arn = group.get('Group', {}).get('Arn') - - return default_group_arn - - @staticmethod - def list_user_groups(AwsAccountId, UserName): - client = Quicksight.get_quicksight_client_in_identity_region(AwsAccountId) - user = Quicksight.describe_user(AwsAccountId, UserName) - if not user: - return [] - response = client.list_user_groups( - UserName=UserName, AwsAccountId=AwsAccountId, Namespace='default' - ) - return response['GroupList'] - - @staticmethod - def register_user_in_group(AwsAccountId, UserName, GroupName, UserRole='READER'): - client = Quicksight.get_quicksight_client_in_identity_region( - AwsAccountId=AwsAccountId - ) - - Quicksight.create_quicksight_group(AwsAccountId, GroupName) - - exists = False - user = Quicksight.describe_user(AwsAccountId, UserName=UserName) - - if user is not None: - exists = True - - if exists: - response = client.update_user( - UserName=UserName, - AwsAccountId=AwsAccountId, - Namespace='default', - Email=UserName, - Role=UserRole, - ) - else: - response = client.register_user( - UserName=UserName, - Email=UserName, - AwsAccountId=AwsAccountId, - Namespace='default', - IdentityType='QUICKSIGHT', - UserRole=UserRole, - ) - member = False - - response = client.list_user_groups( - UserName=UserName, AwsAccountId=AwsAccountId, Namespace='default' - ) - logger.info( - f'list_user_groups for {UserName}: {response})' - ) - if GroupName not in [g['GroupName'] for g in response['GroupList']]: - logger.warning(f'Adding {UserName} to Quicksight group {GroupName} on {AwsAccountId}') - response = client.create_group_membership( - MemberName=UserName, - GroupName=GroupName, - AwsAccountId=AwsAccountId, - Namespace='default', - ) - return Quicksight.describe_user(AwsAccountId, UserName) - - @staticmethod - def get_reader_session(AwsAccountId, region, UserName, UserRole="READER", DashboardId=None, domain_name: str = None): - - client = Quicksight.get_quicksight_client(AwsAccountId, region) - user = Quicksight.describe_user(AwsAccountId, UserName) - if user is None: - user = Quicksight.register_user_in_group( - AwsAccountId=AwsAccountId, UserName=UserName, GroupName=Quicksight._DEFAULT_GROUP_NAME, UserRole=UserRole - ) - - response = client.generate_embed_url_for_registered_user( - AwsAccountId=AwsAccountId, - SessionLifetimeInMinutes=120, - UserArn=user.get("Arn"), - ExperienceConfiguration={ - "Dashboard": { - "InitialDashboardId": DashboardId, - }, - }, - AllowedDomains=[domain_name], - ) - return response.get('EmbedUrl') - - @staticmethod - def check_dashboard_permissions(AwsAccountId, region, DashboardId): - client = Quicksight.get_quicksight_client(AwsAccountId, region) - response = client.describe_dashboard_permissions( - AwsAccountId=AwsAccountId, - DashboardId=DashboardId - )['Permissions'] - logger.info(f"Dashboard initial permissions: {response}") - read_principals = [] - write_principals = [] - - for a, p in zip([p["Actions"] for p in response], [p["Principal"] for p in response]): - write_principals.append(p) if "Update" in str(a) else read_principals.append(p) - - logger.info(f"Dashboard updated permissions, Read principals: {read_principals}") - logger.info(f"Dashboard updated permissions, Write principals: {write_principals}") - - return read_principals, write_principals - - @staticmethod - def get_shared_reader_session( - AwsAccountId, region, UserName, GroupName, UserRole='READER', DashboardId=None - ): - - client = Quicksight.get_quicksight_client(AwsAccountId, region) - identity_region = Quicksight.get_identity_region(AwsAccountId) - groupPrincipal = f"arn:aws:quicksight:{identity_region}:{AwsAccountId}:group/default/{GroupName}" - - user = Quicksight.register_user_in_group( - AwsAccountId=AwsAccountId, UserName=UserName, GroupName=GroupName, UserRole=UserRole - ) - - read_principals, write_principals = Quicksight.check_dashboard_permissions( - AwsAccountId=AwsAccountId, - region=region, - DashboardId=DashboardId - ) - - if groupPrincipal not in read_principals: - permissions = client.update_dashboard_permissions( - AwsAccountId=AwsAccountId, - DashboardId=DashboardId, - GrantPermissions=[ - { - 'Principal': groupPrincipal, - 'Actions': [ - "quicksight:DescribeDashboard", - "quicksight:ListDashboardVersions", - "quicksight:QueryDashboard", - ] - }, - ] - ) - logger.info(f"Permissions granted: {permissions}") - - response = client.get_dashboard_embed_url( - AwsAccountId=AwsAccountId, - DashboardId=DashboardId, - IdentityType='QUICKSIGHT', - SessionLifetimeInMinutes=120, - UserArn=user.get('Arn'), - ) - return response.get('EmbedUrl') - - @staticmethod - def get_anonymous_session(AwsAccountId, region, UserName, DashboardId=None): - client = Quicksight.get_quicksight_client(AwsAccountId, region) - response = client.generate_embed_url_for_anonymous_user( - AwsAccountId=AwsAccountId, - SessionLifetimeInMinutes=120, - Namespace='default', - SessionTags=[ - {'Key': Quicksight._DEFAULT_GROUP_NAME, 'Value': UserName}, - ], - AuthorizedResourceArns=[ - f'arn:aws:quicksight:{region}:{AwsAccountId}:dashboard/{DashboardId}', - ], - ExperienceConfiguration={'Dashboard': {'InitialDashboardId': DashboardId}}, - ) - return response.get('EmbedUrl') - - @staticmethod - def get_author_session(AwsAccountId, region, UserName, UserRole='AUTHOR'): - client = Quicksight.get_quicksight_client(AwsAccountId, region) - user = Quicksight.describe_user(AwsAccountId, UserName=UserName) - if user is None: - user = Quicksight.register_user_in_group( - AwsAccountId=AwsAccountId, UserName=UserName, GroupName=Quicksight._DEFAULT_GROUP_NAME, UserRole=UserRole - ) - elif user.get("Role", None) not in ["AUTHOR", "ADMIN"]: - user = Quicksight.register_user_in_group( - AwsAccountId=AwsAccountId, UserName=UserName, GroupName=Quicksight._DEFAULT_GROUP_NAME, UserRole=UserRole - ) - else: - pass - response = client.get_session_embed_url( - AwsAccountId=AwsAccountId, - EntryPoint='/start/dashboards', - SessionLifetimeInMinutes=120, - UserArn=user['Arn'], - ) - return response['EmbedUrl'] - - @staticmethod - def can_import_dashboard(AwsAccountId, region, UserName, DashboardId): - client = Quicksight.get_quicksight_client(AwsAccountId, region) - user = Quicksight.describe_user(AwsAccountId, UserName) - if not user: - return False - - groups = Quicksight.list_user_groups(AwsAccountId, UserName) - grouparns = [g['Arn'] for g in groups] - try: - response = client.describe_dashboard_permissions( - AwsAccountId=AwsAccountId, DashboardId=DashboardId - ) - except ClientError as e: - raise e - - permissions = response.get('Permissions', []) - for p in permissions: - if p['Principal'] == user.get('Arn') or p['Principal'] in grouparns: - for a in p['Actions']: - if a in [ - 'quicksight:UpdateDashboard', - 'UpdateDashboardPermissions', - ]: - return True - - return False - - @staticmethod - def create_data_source_vpc(AwsAccountId, region, UserName, vpcConnectionId): - client = Quicksight.get_quicksight_client(AwsAccountId, region) - identity_region = 'us-east-1' - user = Quicksight.register_user_in_group( - AwsAccountId=AwsAccountId, UserName=UserName, GroupName=Quicksight._DEFAULT_GROUP_NAME, UserRole='AUTHOR' - ) - try: - response = client.describe_data_source( - AwsAccountId=AwsAccountId, DataSourceId="dataall-metadata-db" - ) - - except client.exceptions.ResourceNotFoundException: - aurora_secret_arn = ParameterStoreManager.get_parameter_value(AwsAccountId=AwsAccountId, region=region, parameter_path=f'/dataall/{os.getenv("envname", "local")}/aurora/secret_arn') - aurora_params = SecretsManager.get_secret_value( - AwsAccountId=AwsAccountId, region=region, secretId=aurora_secret_arn - ) - aurora_params_dict = ast.literal_eval(aurora_params) - response = client.create_data_source( - AwsAccountId=AwsAccountId, - DataSourceId="dataall-metadata-db", - Name="dataall-metadata-db", - Type="AURORA_POSTGRESQL", - DataSourceParameters={ - 'AuroraPostgreSqlParameters': { - 'Host': aurora_params_dict["host"], - 'Port': aurora_params_dict["port"], - 'Database': aurora_params_dict["dbname"] - } - }, - Credentials={ - "CredentialPair": { - "Username": aurora_params_dict["username"], - "Password": aurora_params_dict["password"], - } - }, - Permissions=[ - { - "Principal": f"arn:aws:quicksight:{region}:{AwsAccountId}:group/default/dataall", - "Actions": [ - "quicksight:UpdateDataSourcePermissions", - "quicksight:DescribeDataSource", - "quicksight:DescribeDataSourcePermissions", - "quicksight:PassDataSource", - "quicksight:UpdateDataSource", - "quicksight:DeleteDataSource" - ] - } - ], - VpcConnectionProperties={ - 'VpcConnectionArn': f"arn:aws:quicksight:{region}:{AwsAccountId}:vpcConnection/{vpcConnectionId}" - } - ) - - return "dataall-metadata-db" - - @staticmethod - def create_analysis(AwsAccountId, region, UserName): - client = Quicksight.get_quicksight_client(AwsAccountId, region) - user = Quicksight.describe_user(AwsAccountId, UserName) - if not user: - return False - - response = client.create_analysis( - AwsAccountId=AwsAccountId, - AnalysisId='dataallMonitoringAnalysis', - Name='dataallMonitoringAnalysis', - Permissions=[ - { - 'Principal': user.get('Arn'), - 'Actions': [ - 'quicksight:DescribeAnalysis', - 'quicksight:DescribeAnalysisPermissions', - 'quicksight:UpdateAnalysisPermissions', - 'quicksight:UpdateAnalysis' - ] - }, - ], - SourceEntity={ - 'SourceTemplate': { - 'DataSetReferences': [ - { - 'DataSetPlaceholder': 'environment', - 'DataSetArn': f"arn:aws:quicksight:{region}:{AwsAccountId}:dataset/" - }, - ], - 'Arn': ' int: - raise NotImplementedError("index is not implemented") \ No newline at end of file + raise NotImplementedError("index is not implemented") diff --git a/backend/dataall/core/group/services/environment_resource_manager.py b/backend/dataall/core/group/services/environment_resource_manager.py index d2f3beb96..d4c964514 100644 --- a/backend/dataall/core/group/services/environment_resource_manager.py +++ b/backend/dataall/core/group/services/environment_resource_manager.py @@ -5,31 +5,44 @@ class EnvironmentResource(ABC): @staticmethod def count_resources(session, environment, group_uri) -> int: - raise NotImplementedError() + return 0 @staticmethod def delete_env(session, environment): pass + @staticmethod + def update_env(session, environment): + return False + class EnvironmentResourceManager: """ - API for managing group resources + API for managing environment and environment group lifecycle. + Contains callbacks that are invoked when something is happened with the environment. """ _resources: List[EnvironmentResource] = [] - @staticmethod - def register(resource: EnvironmentResource): - EnvironmentResourceManager._resources.append(resource) + @classmethod + def register(cls, resource: EnvironmentResource): + cls._resources.append(resource) - @staticmethod - def count_group_resources(session, environment, group_uri) -> int: + @classmethod + def count_group_resources(cls, session, environment, group_uri) -> int: counter = 0 - for resource in EnvironmentResourceManager._resources: + for resource in cls._resources: counter += resource.count_resources(session, environment, group_uri) return counter - @staticmethod - def delete_env(session, environment): - for resource in EnvironmentResourceManager._resources: + @classmethod + def deploy_updated_stack(cls, session, prev_prefix, environment): + deploy_stack = prev_prefix != environment.resourcePrefix + for resource in cls._resources: + deploy_stack |= resource.update_env(session, environment) + + return deploy_stack + + @classmethod + def delete_env(cls, session, environment): + for resource in cls._resources: resource.delete_env(session, environment) diff --git a/backend/dataall/db/api/__init__.py b/backend/dataall/db/api/__init__.py index a91c9a3a4..62de5dde1 100644 --- a/backend/dataall/db/api/__init__.py +++ b/backend/dataall/db/api/__init__.py @@ -13,4 +13,3 @@ from .notification import Notification from .redshift_cluster import RedshiftCluster from .vpc import Vpc -from .dashboard import Dashboard diff --git a/backend/dataall/db/api/dashboard.py b/backend/dataall/db/api/dashboard.py deleted file mode 100644 index bf6950002..000000000 --- a/backend/dataall/db/api/dashboard.py +++ /dev/null @@ -1,433 +0,0 @@ -import logging - -from sqlalchemy import or_, and_ -from sqlalchemy.orm import Query - -from .. import models, exceptions, permissions, paginate -from . import ( - Environment, - has_tenant_perm, - has_resource_perm, - ResourcePolicy, - Glossary, - Vote, -) - -logger = logging.getLogger(__name__) - - -class Dashboard: - @staticmethod - @has_tenant_perm(permissions.MANAGE_DASHBOARDS) - @has_resource_perm(permissions.CREATE_DASHBOARD) - def import_dashboard( - session, - username: str, - groups: [str], - uri: str, - data: dict = None, - check_perm: bool = False, - ) -> models.Dashboard: - if not data: - raise exceptions.RequiredParameter(data) - if not data.get('environmentUri'): - raise exceptions.RequiredParameter('environmentUri') - if not data.get('SamlGroupName'): - raise exceptions.RequiredParameter('group') - if not data.get('dashboardId'): - raise exceptions.RequiredParameter('dashboardId') - if not data.get('label'): - raise exceptions.RequiredParameter('label') - - Environment.check_group_environment_permission( - session=session, - username=username, - groups=groups, - uri=uri, - group=data['SamlGroupName'], - permission_name=permissions.CREATE_DASHBOARD, - ) - - env: models.Environment = data.get( - 'environment', Environment.get_environment_by_uri(session, uri) - ) - dashboard: models.Dashboard = models.Dashboard( - label=data.get('label', 'untitled'), - environmentUri=data.get('environmentUri'), - organizationUri=env.organizationUri, - region=env.region, - DashboardId=data.get('dashboardId'), - AwsAccountId=env.AwsAccountId, - owner=username, - namespace='test', - tags=data.get('tags', []), - SamlGroupName=data['SamlGroupName'], - ) - session.add(dashboard) - session.commit() - - activity = models.Activity( - action='DASHBOARD:CREATE', - label='DASHBOARD:CREATE', - owner=username, - summary=f'{username} created dashboard {dashboard.label} in {env.label}', - targetUri=dashboard.dashboardUri, - targetType='dashboard', - ) - session.add(activity) - - Dashboard.set_dashboard_resource_policy( - session, env, dashboard, data['SamlGroupName'] - ) - - if 'terms' in data.keys(): - Glossary.set_glossary_terms_links( - session, - username, - dashboard.dashboardUri, - 'Dashboard', - data.get('terms', []), - ) - return dashboard - - @staticmethod - def set_dashboard_resource_policy(session, environment, dashboard, group): - ResourcePolicy.attach_resource_policy( - session=session, - group=group, - permissions=permissions.DASHBOARD_ALL, - resource_uri=dashboard.dashboardUri, - resource_type=models.Dashboard.__name__, - ) - if environment.SamlGroupName != dashboard.SamlGroupName: - ResourcePolicy.attach_resource_policy( - session=session, - group=environment.SamlGroupName, - permissions=permissions.DASHBOARD_ALL, - resource_uri=dashboard.dashboardUri, - resource_type=models.Dashboard.__name__, - ) - - @staticmethod - @has_tenant_perm(permissions.MANAGE_DASHBOARDS) - @has_resource_perm(permissions.GET_DASHBOARD) - def get_dashboard( - session, - username: str, - groups: [str], - uri: str, - data: dict = None, - check_perm: bool = False, - ) -> models.Dashboard: - return Dashboard.get_dashboard_by_uri(session, uri) - - @staticmethod - def get_dashboard_by_uri(session, uri) -> models.Dashboard: - dashboard: models.Dashboard = session.query(models.Dashboard).get(uri) - if not dashboard: - raise exceptions.ObjectNotFound('Dashboard', uri) - return dashboard - - @staticmethod - def query_user_dashboards(session, username, groups, filter) -> Query: - query = ( - session.query(models.Dashboard) - .outerjoin( - models.DashboardShare, - models.Dashboard.dashboardUri == models.DashboardShare.dashboardUri, - ) - .filter( - or_( - models.Dashboard.owner == username, - models.Dashboard.SamlGroupName.in_(groups), - and_( - models.DashboardShare.SamlGroupName.in_(groups), - models.DashboardShare.status - == models.DashboardShareStatus.APPROVED.value, - ), - ) - ) - ) - if filter and filter.get('term'): - query = query.filter( - or_( - models.Dashboard.description.ilike(filter.get('term') + '%%'), - models.Dashboard.label.ilike(filter.get('term') + '%%'), - ) - ) - return query - - @staticmethod - def paginated_user_dashboards( - session, username, groups, uri, data=None, check_perm=None - ) -> dict: - return paginate( - query=Dashboard.query_user_dashboards(session, username, groups, data), - page=data.get('page', 1), - page_size=data.get('pageSize', 10), - ).to_dict() - - @staticmethod - def query_dashboard_shares(session, username, groups, uri, filter) -> Query: - query = ( - session.query(models.DashboardShare) - .join( - models.Dashboard, - models.Dashboard.dashboardUri == models.DashboardShare.dashboardUri, - ) - .filter( - and_( - models.DashboardShare.dashboardUri == uri, - or_( - models.Dashboard.owner == username, - models.Dashboard.SamlGroupName.in_(groups), - ), - ) - ) - ) - if filter and filter.get('term'): - query = query.filter( - or_( - models.DashboardShare.SamlGroupName.ilike( - filter.get('term') + '%%' - ), - models.Dashboard.label.ilike(filter.get('term') + '%%'), - ) - ) - return query - - @staticmethod - def query_all_user_groups_shareddashboard(session, username, groups, uri) -> Query: - query = ( - session.query(models.DashboardShare) - .filter( - and_( - models.DashboardShare.dashboardUri == uri, - models.DashboardShare.SamlGroupName.in_(groups), - ) - ) - ) - - return [ - share.SamlGroupName - for share in query.all() - ] - - @staticmethod - @has_tenant_perm(permissions.MANAGE_DASHBOARDS) - @has_resource_perm(permissions.SHARE_DASHBOARD) - def paginated_dashboard_shares( - session, username, groups, uri, data=None, check_perm=None - ) -> dict: - return paginate( - query=Dashboard.query_dashboard_shares( - session, username, groups, uri, data - ), - page=data.get('page', 1), - page_size=data.get('pageSize', 10), - ).to_dict() - - @staticmethod - @has_tenant_perm(permissions.MANAGE_DASHBOARDS) - @has_resource_perm(permissions.UPDATE_DASHBOARD) - def update_dashboard( - session, - username: str, - groups: [str], - uri: str, - data: dict = None, - check_perm: bool = False, - ) -> models.Dashboard: - - dashboard = data.get( - 'dashboard', - Dashboard.get_dashboard_by_uri(session, data['dashboardUri']), - ) - - for k in data.keys(): - setattr(dashboard, k, data.get(k)) - - if 'terms' in data.keys(): - Glossary.set_glossary_terms_links( - session, - username, - dashboard.dashboardUri, - 'Dashboard', - data.get('terms', []), - ) - environment: models.Environment = Environment.get_environment_by_uri( - session, dashboard.environmentUri - ) - Dashboard.set_dashboard_resource_policy( - session, environment, dashboard, dashboard.SamlGroupName - ) - return dashboard - - @staticmethod - def delete_dashboard( - session, username, groups, uri, data=None, check_perm=None - ) -> bool: - dashboard = Dashboard.get_dashboard_by_uri(session, uri) - session.delete(dashboard) - ResourcePolicy.delete_resource_policy( - session=session, resource_uri=uri, group=dashboard.SamlGroupName - ) - Glossary.delete_glossary_terms_links( - session, target_uri=dashboard.dashboardUri, target_type='Dashboard' - ) - Vote.delete_votes(session, dashboard.dashboardUri, 'dashboard') - session.commit() - return True - - @staticmethod - @has_tenant_perm(permissions.MANAGE_DASHBOARDS) - def request_dashboard_share( - session, - username: str, - groups: [str], - uri: str, - data: dict = None, - check_perm: bool = False, - ) -> models.DashboardShare: - dashboard = Dashboard.get_dashboard_by_uri(session, uri) - if dashboard.SamlGroupName == data['principalId']: - raise exceptions.UnauthorizedOperation( - action=permissions.CREATE_DASHBOARD, - message=f'Team {dashboard.SamlGroupName} is the owner of the dashboard {dashboard.label}', - ) - share: models.DashboardShare = ( - session.query(models.DashboardShare) - .filter( - models.DashboardShare.dashboardUri == uri, - models.DashboardShare.SamlGroupName == data['principalId'], - ) - .first() - ) - if not share: - share = models.DashboardShare( - owner=username, - dashboardUri=dashboard.dashboardUri, - SamlGroupName=data['principalId'], - status=models.DashboardShareStatus.REQUESTED.value, - ) - session.add(share) - else: - if share.status not in models.DashboardShareStatus.__members__: - raise exceptions.InvalidInput( - 'Share status', - share.status, - str(models.DashboardShareStatus.__members__), - ) - if share.status == 'REJECTED': - share.status = 'REQUESTED' - - return share - - @staticmethod - @has_tenant_perm(permissions.MANAGE_DASHBOARDS) - @has_resource_perm(permissions.SHARE_DASHBOARD) - def approve_dashboard_share( - session, - username: str, - groups: [str], - uri: str, - data: dict = None, - check_perm: bool = False, - ) -> models.DashboardShare: - - share: models.DashboardShare = data.get( - 'share', session.query(models.DashboardShare).get(data['shareUri']) - ) - - if share.status not in models.DashboardShareStatus.__members__: - raise exceptions.InvalidInput( - 'Share status', - share.status, - str(models.DashboardShareStatus.__members__), - ) - if share.status == models.DashboardShareStatus.APPROVED.value: - return share - - share.status = models.DashboardShareStatus.APPROVED.value - - ResourcePolicy.attach_resource_policy( - session=session, - group=share.SamlGroupName, - permissions=[permissions.GET_DASHBOARD], - resource_uri=share.dashboardUri, - resource_type=models.Dashboard.__name__, - ) - - return share - - @staticmethod - @has_tenant_perm(permissions.MANAGE_DASHBOARDS) - @has_resource_perm(permissions.SHARE_DASHBOARD) - def reject_dashboard_share( - session, - username: str, - groups: [str], - uri: str, - data: dict = None, - check_perm: bool = False, - ) -> models.DashboardShare: - - share: models.DashboardShare = data.get( - 'share', session.query(models.DashboardShare).get(data['shareUri']) - ) - - if share.status not in models.DashboardShareStatus.__members__: - raise exceptions.InvalidInput( - 'Share status', - share.status, - str(models.DashboardShareStatus.__members__), - ) - if share.status == models.DashboardShareStatus.REJECTED.value: - return share - - share.status = models.DashboardShareStatus.REJECTED.value - - ResourcePolicy.delete_resource_policy( - session=session, - group=share.SamlGroupName, - resource_uri=share.dashboardUri, - resource_type=models.Dashboard.__name__, - ) - - return share - - @staticmethod - @has_tenant_perm(permissions.MANAGE_DASHBOARDS) - @has_resource_perm(permissions.SHARE_DASHBOARD) - def share_dashboard( - session, - username: str, - groups: [str], - uri: str, - data: dict = None, - check_perm: bool = False, - ) -> models.DashboardShare: - - dashboard = Dashboard.get_dashboard_by_uri(session, uri) - share = models.DashboardShare( - owner=username, - dashboardUri=dashboard.dashboardUri, - SamlGroupName=data['principalId'], - status=models.DashboardShareStatus.APPROVED.value, - ) - session.add(share) - ResourcePolicy.attach_resource_policy( - session=session, - group=data['principalId'], - permissions=[permissions.GET_DASHBOARD], - resource_uri=dashboard.dashboardUri, - resource_type=models.Dashboard.__name__, - ) - return share - - @staticmethod - def get_dashboard_share_by_uri(session, uri) -> models.DashboardShare: - share: models.DashboardShare = session.query(models.DashboardShare).get(uri) - if not share: - raise exceptions.ObjectNotFound('DashboardShare', uri) - return share diff --git a/backend/dataall/db/api/environment.py b/backend/dataall/db/api/environment.py index adf35326a..c65d8cab8 100644 --- a/backend/dataall/db/api/environment.py +++ b/backend/dataall/db/api/environment.py @@ -18,8 +18,6 @@ from ..models.Enums import ( EnvironmentType, EnvironmentPermission, - PrincipalType - ) from ..models.Permission import PermissionType from ..paginator import paginate @@ -59,7 +57,6 @@ def create_environment(session, username, groups, uri, data=None, check_perm=Non ), EnvironmentDefaultIAMRoleArn=f'arn:aws:iam::{data.get("AwsAccountId")}:role/{data.get("EnvironmentDefaultIAMRoleName")}', CDKRoleArn=f"arn:aws:iam::{data.get('AwsAccountId')}:role/{data['cdk_role_name']}", - dashboardsEnabled=data.get('dashboardsEnabled', False), warehousesEnabled=data.get('warehousesEnabled', True), resourcePrefix=data.get('resourcePrefix'), ) @@ -187,8 +184,6 @@ def update_environment(session, username, groups, uri, data=None, check_perm=Non environment.description = data.get('description', 'No description provided') if data.get('tags'): environment.tags = data.get('tags') - if 'dashboardsEnabled' in data.keys(): - environment.dashboardsEnabled = data.get('dashboardsEnabled') if 'warehousesEnabled' in data.keys(): environment.warehousesEnabled = data.get('warehousesEnabled') if data.get('resourcePrefix'): @@ -345,16 +340,11 @@ def remove_group(session, username, groups, uri, data=None, check_perm=None): models.RedshiftCluster.environmentUri == models.Environment.environmentUri, ) - .outerjoin( - models.Dashboard, - models.Dashboard.environmentUri == models.Environment.environmentUri, - ) .filter( and_( models.Environment.environmentUri == environment.environmentUri, or_( models.RedshiftCluster.SamlGroupName == group, - models.Dashboard.SamlGroupName == group, ), ) ) @@ -1011,3 +1001,8 @@ def check_group_environment_permission( @staticmethod def get_environment_parameters(session, env_uri): return EnvironmentParameterRepository(session).get_params(env_uri) + + @staticmethod + def get_boolean_env_param(session, env: models.Environment, param: str) -> bool: + param = EnvironmentParameterRepository(session).get_param(env.environmentUri, param) + return param is not None and param.value.lower() == "true" diff --git a/backend/dataall/db/models/DashboardShare.py b/backend/dataall/db/models/DashboardShare.py deleted file mode 100644 index a8d25dca0..000000000 --- a/backend/dataall/db/models/DashboardShare.py +++ /dev/null @@ -1,24 +0,0 @@ -from enum import Enum - -from sqlalchemy import Column, String - -from .. import Base, utils - - -class DashboardShareStatus(Enum): - REQUESTED = 'REQUESTED' - APPROVED = 'APPROVED' - REJECTED = 'REJECTED' - - -class DashboardShare(Base): - __tablename__ = 'dashboardshare' - shareUri = Column( - String, nullable=False, primary_key=True, default=utils.uuid('shareddashboard') - ) - dashboardUri = Column(String, nullable=False, default=utils.uuid('dashboard')) - SamlGroupName = Column(String, nullable=False) - owner = Column(String, nullable=True) - status = Column( - String, nullable=False, default=DashboardShareStatus.REQUESTED.value - ) diff --git a/backend/dataall/db/models/Enums.py b/backend/dataall/db/models/Enums.py index 10cee7bd1..3e11b6489 100644 --- a/backend/dataall/db/models/Enums.py +++ b/backend/dataall/db/models/Enums.py @@ -36,13 +36,6 @@ class ProjectMemberRole(Enum): NotContributor = '000' -class DashboardRole(Enum): - Creator = '999' - Admin = '900' - Shared = '800' - NoPermission = '000' - - class RedshiftClusterRole(Enum): Creator = '950' Admin = '900' diff --git a/backend/dataall/db/models/Environment.py b/backend/dataall/db/models/Environment.py index 5d5e5801a..9701ec70d 100644 --- a/backend/dataall/db/models/Environment.py +++ b/backend/dataall/db/models/Environment.py @@ -24,7 +24,6 @@ class Environment(Resource, Base): EnvironmentDefaultAthenaWorkGroup = Column(String) roleCreated = Column(Boolean, nullable=False, default=False) - dashboardsEnabled = Column(Boolean, default=False) warehousesEnabled = Column(Boolean, default=True) userRoleInEnvironment = query_expression() diff --git a/backend/dataall/db/models/__init__.py b/backend/dataall/db/models/__init__.py index 97bca8095..22ecda1fe 100644 --- a/backend/dataall/db/models/__init__.py +++ b/backend/dataall/db/models/__init__.py @@ -1,9 +1,6 @@ from .Enums import * from .Activity import Activity from .KeyValueTag import KeyValueTag -from .Dashboard import Dashboard -from .DashboardShare import DashboardShare -from .DashboardShare import DashboardShareStatus from .Environment import Environment from .EnvironmentGroup import EnvironmentGroup from .FeedMessage import FeedMessage diff --git a/backend/dataall/db/permissions.py b/backend/dataall/db/permissions.py index bd3cc851b..ad6145ec1 100644 --- a/backend/dataall/db/permissions.py +++ b/backend/dataall/db/permissions.py @@ -23,7 +23,6 @@ TENANT PERMISSIONS """ MANAGE_REDSHIFT_CLUSTERS = 'MANAGE_REDSHIFT_CLUSTERS' -MANAGE_DASHBOARDS = 'MANAGE_DASHBOARDS' MANAGE_GROUPS = 'MANAGE_GROUPS' MANAGE_ENVIRONMENT = 'MANAGE_ENVIRONMENT' MANAGE_GLOSSARIES = 'MANAGE_GLOSSARIES' @@ -49,8 +48,6 @@ DISABLE_ENVIRONMENT_SUBSCRIPTIONS = 'DISABLE_ENVIRONMENT_SUBSCRIPTIONS' CREATE_REDSHIFT_CLUSTER = 'CREATE_REDSHIFT_CLUSTER' LIST_ENVIRONMENT_REDSHIFT_CLUSTERS = 'LIST_ENVIRONMENT_REDSHIFT_CLUSTERS' -CREATE_DASHBOARD = 'CREATE_DASHBOARD' -LIST_ENVIRONMENT_DASHBOARDS = 'LIST_ENVIRONMENT_DASHBOARDS' CREATE_NETWORK = 'CREATE_NETWORK' LIST_ENVIRONMENT_NETWORKS = 'LIST_ENVIRONMENT_NETWORKS' @@ -62,8 +59,6 @@ LIST_ENVIRONMENT_CONSUMPTION_ROLES, CREATE_REDSHIFT_CLUSTER, LIST_ENVIRONMENT_REDSHIFT_CLUSTERS, - CREATE_DASHBOARD, - LIST_ENVIRONMENT_DASHBOARDS, INVITE_ENVIRONMENT_GROUP, ADD_ENVIRONMENT_CONSUMPTION_ROLES, CREATE_NETWORK, @@ -73,7 +68,6 @@ INVITE_ENVIRONMENT_GROUP, ADD_ENVIRONMENT_CONSUMPTION_ROLES, CREATE_REDSHIFT_CLUSTER, - CREATE_DASHBOARD, CREATE_NETWORK, ] ENVIRONMENT_ALL = [ @@ -92,8 +86,6 @@ DISABLE_ENVIRONMENT_SUBSCRIPTIONS, CREATE_REDSHIFT_CLUSTER, LIST_ENVIRONMENT_REDSHIFT_CLUSTERS, - CREATE_DASHBOARD, - LIST_ENVIRONMENT_DASHBOARDS, CREATE_NETWORK, LIST_ENVIRONMENT_NETWORKS, ] @@ -130,7 +122,6 @@ TENANT_ALL = [ MANAGE_REDSHIFT_CLUSTERS, - MANAGE_DASHBOARDS, MANAGE_GLOSSARIES, MANAGE_GROUPS, MANAGE_ENVIRONMENTS, @@ -139,7 +130,6 @@ ] TENANT_ALL_WITH_DESC = {k: k for k in TENANT_ALL} -TENANT_ALL_WITH_DESC[MANAGE_DASHBOARDS] = 'Manage dashboards' TENANT_ALL_WITH_DESC[MANAGE_REDSHIFT_CLUSTERS] = 'Manage Redshift clusters' TENANT_ALL_WITH_DESC[MANAGE_GLOSSARIES] = 'Manage glossaries' TENANT_ALL_WITH_DESC[MANAGE_ENVIRONMENTS] = 'Manage environments' @@ -177,23 +167,6 @@ GET_REDSHIFT_CLUSTER_CREDENTIALS, ] - -""" -DASHBOARDS -""" -GET_DASHBOARD = 'GET_DASHBOARD' -UPDATE_DASHBOARD = 'UPDATE_DASHBOARD' -DELETE_DASHBOARD = 'DELETE_DASHBOARD' -DASHBOARD_URL = 'DASHBOARD_URL' -SHARE_DASHBOARD = 'SHARE_DASHBOARD' -DASHBOARD_ALL = [ - GET_DASHBOARD, - UPDATE_DASHBOARD, - DELETE_DASHBOARD, - DASHBOARD_URL, - SHARE_DASHBOARD, -] - """ NETWORKS """ @@ -211,12 +184,10 @@ + CONSUMPTION_ROLE_ALL + REDSHIFT_CLUSTER_ALL + GLOSSARY_ALL - + DASHBOARD_ALL + NETWORK_ALL ) RESOURCES_ALL_WITH_DESC = {k: k for k in RESOURCES_ALL} -RESOURCES_ALL_WITH_DESC[CREATE_DASHBOARD] = 'Create dashboards on this environment' RESOURCES_ALL_WITH_DESC[CREATE_REDSHIFT_CLUSTER] = 'Create Redshift clusters on this environment' RESOURCES_ALL_WITH_DESC[INVITE_ENVIRONMENT_GROUP] = 'Invite other teams to this environment' RESOURCES_ALL_WITH_DESC[ADD_ENVIRONMENT_CONSUMPTION_ROLES] = 'Add IAM consumption roles to this environment' diff --git a/backend/dataall/modules/dashboards/__init__.py b/backend/dataall/modules/dashboards/__init__.py new file mode 100644 index 000000000..9c2f4f10e --- /dev/null +++ b/backend/dataall/modules/dashboards/__init__.py @@ -0,0 +1,63 @@ +"""Contains the code related to dashboards""" +import logging +from typing import Set + +from dataall.core.group.services.environment_resource_manager import EnvironmentResourceManager +from dataall.modules.dashboards.db.dashboard_repository import DashboardRepository +from dataall.modules.dashboards.db.models import Dashboard +from dataall.modules.loader import ImportMode, ModuleInterface + +log = logging.getLogger(__name__) + + +class DashboardApiModuleInterface(ModuleInterface): + """Implements ModuleInterface for dashboard GraphQl lambda""" + + @staticmethod + def is_supported(modes: Set[ImportMode]) -> bool: + return ImportMode.API in modes + + def __init__(self): + import dataall.modules.dashboards.api + from dataall.api.Objects.Feed.registry import FeedRegistry, FeedDefinition + from dataall.api.Objects.Glossary.registry import GlossaryRegistry, GlossaryDefinition + from dataall.api.Objects.Vote.resolvers import add_vote_type + from dataall.modules.dashboards.indexers.dashboard_indexer import DashboardIndexer + + FeedRegistry.register(FeedDefinition("Dashboard", Dashboard)) + + GlossaryRegistry.register(GlossaryDefinition( + target_type="Dashboard", + object_type="Dashboard", + model=Dashboard, + reindexer=DashboardIndexer + )) + + add_vote_type("dashboard", DashboardIndexer) + + EnvironmentResourceManager.register(DashboardRepository()) + log.info("Dashboard API has been loaded") + + +class DashboardCdkModuleInterface(ModuleInterface): + + @staticmethod + def is_supported(modes: Set[ImportMode]) -> bool: + return ImportMode.CDK in modes + + def __init__(self): + import dataall.modules.dashboards.cdk + log.info("Dashboard CDK code has been loaded") + + +class DashboardCatalogIndexerModuleInterface(ModuleInterface): + + @staticmethod + def is_supported(modes: Set[ImportMode]) -> bool: + return ImportMode.CATALOG_INDEXER_TASK in modes + + def __init__(self): + from dataall.modules.dashboards.indexers.dashboard_catalog_indexer import DashboardCatalogIndexer + + DashboardCatalogIndexer() + log.info("Dashboard catalog indexer task has been loaded") diff --git a/backend/dataall/api/Objects/Dashboard/__init__.py b/backend/dataall/modules/dashboards/api/__init__.py similarity index 76% rename from backend/dataall/api/Objects/Dashboard/__init__.py rename to backend/dataall/modules/dashboards/api/__init__.py index dfa46b264..e92cd6d66 100644 --- a/backend/dataall/api/Objects/Dashboard/__init__.py +++ b/backend/dataall/modules/dashboards/api/__init__.py @@ -1,4 +1,4 @@ -from . import ( +from dataall.modules.dashboards.api import ( input_types, mutations, queries, diff --git a/backend/dataall/modules/dashboards/api/enums.py b/backend/dataall/modules/dashboards/api/enums.py new file mode 100644 index 000000000..00b72d961 --- /dev/null +++ b/backend/dataall/modules/dashboards/api/enums.py @@ -0,0 +1,8 @@ +from dataall.api.constants import GraphQLEnumMapper + + +class DashboardRole(GraphQLEnumMapper): + Creator = '999' + Admin = '900' + Shared = '800' + NoPermission = '000' diff --git a/backend/dataall/api/Objects/Dashboard/input_types.py b/backend/dataall/modules/dashboards/api/input_types.py similarity index 98% rename from backend/dataall/api/Objects/Dashboard/input_types.py rename to backend/dataall/modules/dashboards/api/input_types.py index 1686c31e3..93e6cb5c1 100644 --- a/backend/dataall/api/Objects/Dashboard/input_types.py +++ b/backend/dataall/modules/dashboards/api/input_types.py @@ -1,4 +1,4 @@ -from ... import gql +from dataall.api import gql ImportDashboardInput = gql.InputType( name='ImportDashboardInput', diff --git a/backend/dataall/api/Objects/Dashboard/mutations.py b/backend/dataall/modules/dashboards/api/mutations.py similarity index 84% rename from backend/dataall/api/Objects/Dashboard/mutations.py rename to backend/dataall/modules/dashboards/api/mutations.py index 7af472838..3e92b1995 100644 --- a/backend/dataall/api/Objects/Dashboard/mutations.py +++ b/backend/dataall/modules/dashboards/api/mutations.py @@ -1,5 +1,5 @@ -from ... import gql -from .resolvers import * +from dataall.api import gql +from dataall.modules.dashboards.api.resolvers import * importDashboard = gql.MutationField( @@ -70,3 +70,12 @@ ], resolver=reject_dashboard_share, ) + +createQuicksightDataSourceSet = gql.MutationField( + name='createQuicksightDataSourceSet', + args=[ + gql.Argument(name='vpcConnectionId', type=gql.NonNullableType(gql.String)) + ], + type=gql.String, + resolver=create_quicksight_data_source_set, +) diff --git a/backend/dataall/api/Objects/Dashboard/queries.py b/backend/dataall/modules/dashboards/api/queries.py similarity index 59% rename from backend/dataall/api/Objects/Dashboard/queries.py rename to backend/dataall/modules/dashboards/api/queries.py index d8d3b9982..9dae022bb 100644 --- a/backend/dataall/api/Objects/Dashboard/queries.py +++ b/backend/dataall/modules/dashboards/api/queries.py @@ -1,5 +1,5 @@ -from ... import gql -from .resolvers import * +from dataall.api import gql +from dataall.modules.dashboards.api.resolvers import * searchDashboards = gql.QueryField( name='searchDashboards', @@ -15,6 +15,35 @@ resolver=get_dashboard, ) +getMonitoringDashboardId = gql.QueryField( + name='getMonitoringDashboardId', + type=gql.String, + resolver=get_monitoring_dashboard_id, +) + +getMonitoringVpcConnectionId = gql.QueryField( + name='getMonitoringVPCConnectionId', + type=gql.String, + resolver=get_monitoring_vpc_connection_id, +) + +getPlatformAuthorSession = gql.QueryField( + name='getPlatformAuthorSession', + args=[ + gql.Argument(name='awsAccount', type=gql.NonNullableType(gql.String)), + ], + type=gql.String, + resolver=get_quicksight_author_session, +) + +getPlatformReaderSession = gql.QueryField( + name='getPlatformReaderSession', + args=[ + gql.Argument(name='dashboardId', type=gql.NonNullableType(gql.String)), + ], + type=gql.String, + resolver=get_quicksight_reader_session, +) getAuthorSession = gql.QueryField( name='getAuthorSession', diff --git a/backend/dataall/modules/dashboards/api/resolvers.py b/backend/dataall/modules/dashboards/api/resolvers.py new file mode 100644 index 000000000..8f071db52 --- /dev/null +++ b/backend/dataall/modules/dashboards/api/resolvers.py @@ -0,0 +1,147 @@ +from dataall.api.context import Context +from dataall.db import models +from dataall.db.api import Glossary, Vote, Organization +from dataall.db.exceptions import RequiredParameter +from dataall.modules.dashboards.api.enums import DashboardRole +from dataall.modules.dashboards.db.dashboard_repository import DashboardRepository +from dataall.modules.dashboards.db.models import Dashboard +from dataall.modules.dashboards.services.dashboard_quicksight_service import DashboardQuicksightService +from dataall.modules.dashboards.services.dashboard_service import DashboardService +from dataall.modules.dashboards.services.dashboard_share_service import DashboardShareService + + +def import_dashboard(context: Context, source, input: dict = None): + if not input: + raise RequiredParameter(input) + if not input.get('environmentUri'): + raise RequiredParameter('environmentUri') + if not input.get('SamlGroupName'): + raise RequiredParameter('group') + if not input.get('dashboardId'): + raise RequiredParameter('dashboardId') + if not input.get('label'): + raise RequiredParameter('label') + + return DashboardService.import_dashboard( + uri=input['environmentUri'], + admin_group=input['SamlGroupName'], + data=input + ) + + +def update_dashboard(context, source, input: dict = None): + return DashboardService.update_dashboard(uri=input['dashboardUri'], data=input) + + +def list_dashboards(context: Context, source, filter: dict = None): + if not filter: + filter = {} + with context.engine.scoped_session() as session: + return DashboardRepository.paginated_user_dashboards( + session=session, + username=context.username, + groups=context.groups, + data=filter, + ) + + +def get_dashboard(context: Context, source, dashboardUri: str = None): + return DashboardService.get_dashboard(uri=dashboardUri) + + +def resolve_user_role(context: Context, source: Dashboard): + if context.username and source.owner == context.username: + return DashboardRole.Creator.value + elif context.groups and source.SamlGroupName in context.groups: + return DashboardRole.Admin.value + return DashboardRole.Shared.value + + +def get_dashboard_organization(context: Context, source: Dashboard, **kwargs): + with context.engine.scoped_session() as session: + return Organization.get_organization_by_uri(session, source.organizationUri) + + +def request_dashboard_share( + context: Context, + source: Dashboard, + principalId: str = None, + dashboardUri: str = None, +): + return DashboardShareService.request_dashboard_share(uri=dashboardUri, principal_id=principalId) + + +def approve_dashboard_share(context: Context, source: Dashboard, shareUri: str = None): + return DashboardShareService.approve_dashboard_share(uri=shareUri) + + +def reject_dashboard_share(context: Context, source: Dashboard, shareUri: str = None): + return DashboardShareService.reject_dashboard_share(uri=shareUri) + + +def list_dashboard_shares( + context: Context, + source: Dashboard, + dashboardUri: str = None, + filter: dict = None, +): + if not filter: + filter = {} + return DashboardShareService.list_dashboard_shares(uri=dashboardUri, data=filter) + + +def share_dashboard( + context: Context, + source: Dashboard, + principalId: str = None, + dashboardUri: str = None, +): + return DashboardShareService.share_dashboard(uri=dashboardUri, principal_id=principalId) + + +def delete_dashboard(context: Context, source, dashboardUri: str = None): + return DashboardService.delete_dashboard(uri=dashboardUri) + + +def resolve_glossary_terms(context: Context, source: Dashboard, **kwargs): + with context.engine.scoped_session() as session: + return Glossary.get_glossary_terms_links( + session, source.dashboardUri, 'Dashboard' + ) + + +def resolve_upvotes(context: Context, source: Dashboard, **kwargs): + with context.engine.scoped_session() as session: + return Vote.count_upvotes( + session, None, None, source.dashboardUri, data={'targetType': 'dashboard'} + ) + + +def get_monitoring_dashboard_id(context, source): + return DashboardQuicksightService.get_monitoring_dashboard_id() + + +def get_monitoring_vpc_connection_id(context, source): + return DashboardQuicksightService.get_monitoring_vpc_connection_id() + + +def create_quicksight_data_source_set(context, source, vpcConnectionId: str = None): + return DashboardQuicksightService.create_quicksight_data_source_set(vpcConnectionId) + + +def get_quicksight_author_session(context, source, awsAccount: str = None): + return DashboardQuicksightService.get_quicksight_author_session(awsAccount) + + +def get_quicksight_reader_session(context, source, dashboardId: str = None): + return DashboardQuicksightService.get_quicksight_reader_session(dashboardId) + + +def get_quicksight_reader_url(context, source, dashboardUri: str = None): + return DashboardQuicksightService.get_quicksight_reader_url(uri=dashboardUri) + + +def get_quicksight_designer_url( + context, source, environmentUri: str = None, dashboardUri: str = None +): + return DashboardQuicksightService.get_quicksight_designer_url(uri=environmentUri) diff --git a/backend/dataall/api/Objects/Dashboard/schema.py b/backend/dataall/modules/dashboards/api/schema.py similarity index 97% rename from backend/dataall/api/Objects/Dashboard/schema.py rename to backend/dataall/modules/dashboards/api/schema.py index 58b6a30cb..429696ab8 100644 --- a/backend/dataall/api/Objects/Dashboard/schema.py +++ b/backend/dataall/modules/dashboards/api/schema.py @@ -1,6 +1,5 @@ -from ... import gql -from .resolvers import * -from ...constants import DashboardRole +from dataall.api import gql +from dataall.modules.dashboards.api.resolvers import * from dataall.api.Objects.Environment.resolvers import resolve_environment diff --git a/backend/dataall/modules/dashboards/aws/__init__.py b/backend/dataall/modules/dashboards/aws/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/dataall/modules/dashboards/aws/dashboard_quicksight_client.py b/backend/dataall/modules/dashboards/aws/dashboard_quicksight_client.py new file mode 100644 index 000000000..0c424eb68 --- /dev/null +++ b/backend/dataall/modules/dashboards/aws/dashboard_quicksight_client.py @@ -0,0 +1,265 @@ +import logging +import re +import os +import ast + +from botocore.exceptions import ClientError + +from dataall.aws.handlers.parameter_store import ParameterStoreManager +from dataall.aws.handlers.quicksight import QuicksightClient +from dataall.aws.handlers.secrets_manager import SecretsManager +from dataall.aws.handlers.sts import SessionHelper + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + + +class DashboardQuicksightClient: + _DEFAULT_GROUP_NAME = QuicksightClient.DEFAULT_GROUP_NAME + + def __init__(self, username, aws_account_id, region='eu-west-1'): + session = SessionHelper.remote_session(accountid=aws_account_id) + self._client = session.client('quicksight', region_name=region) + self._account_id = aws_account_id + self._region = region + self._username = username + + def register_user_in_group(self, group_name, user_role='READER'): + QuicksightClient.create_quicksight_group(self._account_id, group_name) + user = self._describe_user() + + if user is not None: + self._client.update_user( + UserName=self._username, + AwsAccountId=self._account_id, + Namespace='default', + Email=self._username, + Role=user_role, + ) + else: + self._client.register_user( + UserName=self._username, + Email=self._username, + AwsAccountId=self._account_id, + Namespace='default', + IdentityType='QUICKSIGHT', + UserRole=user_role, + ) + + response = self._client.list_user_groups( + UserName=self._username, AwsAccountId=self._account_id, Namespace='default' + ) + log.info(f'list_user_groups for {self._username}: {response})') + if group_name not in [g['GroupName'] for g in response['GroupList']]: + log.warning(f'Adding {self._username} to Quicksight group {group_name} on {self._account_id}') + self._client.create_group_membership( + MemberName=self._username, + GroupName=group_name, + AwsAccountId=self._account_id, + Namespace='default', + ) + return self._describe_user() + + def get_reader_session(self, user_role="READER", dashboard_id=None, domain_name: str = None): + user = self._describe_user() + if user is None: + user = self.register_user_in_group(self._DEFAULT_GROUP_NAME, user_role) + + response = self._client.generate_embed_url_for_registered_user( + AwsAccountId=self._account_id, + SessionLifetimeInMinutes=120, + UserArn=user.get("Arn"), + ExperienceConfiguration={ + "Dashboard": { + "InitialDashboardId": dashboard_id, + }, + }, + AllowedDomains=[domain_name], + ) + return response.get('EmbedUrl') + + def get_shared_reader_session(self, group_name, user_role='READER', dashboard_id=None): + aws_account_id = self._account_id + identity_region = QuicksightClient.get_identity_region(aws_account_id) + group_principal = f"arn:aws:quicksight:{identity_region}:{aws_account_id}:group/default/{group_name}" + + user = self.register_user_in_group(group_name, user_role) + + read_principals, write_principals = self._check_dashboard_permissions(dashboard_id) + + if group_principal not in read_principals: + permissions = self._client.update_dashboard_permissions( + AwsAccountId=aws_account_id, + DashboardId=dashboard_id, + GrantPermissions=[ + { + 'Principal': group_principal, + 'Actions': [ + "quicksight:DescribeDashboard", + "quicksight:ListDashboardVersions", + "quicksight:QueryDashboard", + ] + }, + ] + ) + log.info(f"Permissions granted: {permissions}") + + response = self._client.get_dashboard_embed_url( + AwsAccountId=aws_account_id, + DashboardId=dashboard_id, + IdentityType='QUICKSIGHT', + SessionLifetimeInMinutes=120, + UserArn=user.get('Arn'), + ) + return response.get('EmbedUrl') + + def get_anonymous_session(self, dashboard_id=None): + response = self._client.generate_embed_url_for_anonymous_user( + AwsAccountId=self._account_id, + SessionLifetimeInMinutes=120, + Namespace='default', + SessionTags=[{'Key': self._DEFAULT_GROUP_NAME, 'Value': self._username}], + AuthorizedResourceArns=[ + f'arn:aws:quicksight:{self._region}:{self._account_id}:dashboard/{dashboard_id}', + ], + ExperienceConfiguration={'Dashboard': {'InitialDashboardId': dashboard_id}}, + ) + return response.get('EmbedUrl') + + def get_author_session(self): + user = self._describe_user() + if user is None or user.get("Role", None) not in ["AUTHOR", "ADMIN"]: + user = self.register_user_in_group(self._DEFAULT_GROUP_NAME, "AUTHOR") + + response = self._client.get_session_embed_url( + AwsAccountId=self._account_id, + EntryPoint='/start/dashboards', + SessionLifetimeInMinutes=120, + UserArn=user['Arn'], + ) + return response['EmbedUrl'] + + def can_import_dashboard(self, dashboard_id): + user = self._describe_user() + if not user: + return False + + groups = self._list_user_groups() + grouparns = [g['Arn'] for g in groups] + try: + response = self._client.describe_dashboard_permissions( + AwsAccountId=self._account_id, DashboardId=dashboard_id + ) + except ClientError as e: + raise e + + permissions = response.get('Permissions', []) + for p in permissions: + if p['Principal'] == user.get('Arn') or p['Principal'] in grouparns: + for a in p['Actions']: + if a in [ + 'quicksight:UpdateDashboard', + 'UpdateDashboardPermissions', + ]: + return True + + return False + + def create_data_source_vpc(self, vpc_connection_id): + client = self._client + aws_account_id = self._account_id + region = self._region + + self.register_user_in_group(self._DEFAULT_GROUP_NAME, 'AUTHOR') + try: + client.describe_data_source( + AwsAccountId=aws_account_id, DataSourceId="dataall-metadata-db" + ) + + except client.exceptions.ResourceNotFoundException: + aurora_secret_arn = ParameterStoreManager.get_parameter_value( + AwsAccountId=aws_account_id, + region=region, + parameter_path=f'/dataall/{os.getenv("envname", "local")}/aurora/secret_arn' + ) + + aurora_params = SecretsManager.get_secret_value( + AwsAccountId=aws_account_id, region=region, secretId=aurora_secret_arn + ) + aurora_params_dict = ast.literal_eval(aurora_params) + client.create_data_source( + AwsAccountId=aws_account_id, + DataSourceId="dataall-metadata-db", + Name="dataall-metadata-db", + Type="AURORA_POSTGRESQL", + DataSourceParameters={ + 'AuroraPostgreSqlParameters': { + 'Host': aurora_params_dict["host"], + 'Port': aurora_params_dict["port"], + 'Database': aurora_params_dict["dbname"] + } + }, + Credentials={ + "CredentialPair": { + "Username": aurora_params_dict["username"], + "Password": aurora_params_dict["password"], + } + }, + Permissions=[ + { + "Principal": f"arn:aws:quicksight:{region}:{aws_account_id}:group/default/dataall", + "Actions": [ + "quicksight:UpdateDataSourcePermissions", + "quicksight:DescribeDataSource", + "quicksight:DescribeDataSourcePermissions", + "quicksight:PassDataSource", + "quicksight:UpdateDataSource", + "quicksight:DeleteDataSource" + ] + } + ], + VpcConnectionProperties={ + 'VpcConnectionArn': f"arn:aws:quicksight:{region}:{aws_account_id}:vpcConnection/" + f"{vpc_connection_id}" + } + ) + + return "dataall-metadata-db" + + def _check_dashboard_permissions(self, dashboard_id): + response = self._client.describe_dashboard_permissions( + AwsAccountId=self._account_id, + DashboardId=dashboard_id + )['Permissions'] + log.info(f"Dashboard initial permissions: {response}") + read_principals = [] + write_principals = [] + + for a, p in zip([p["Actions"] for p in response], [p["Principal"] for p in response]): + write_principals.append(p) if "Update" in str(a) else read_principals.append(p) + + log.info(f"Dashboard updated permissions, Read principals: {read_principals}") + log.info(f"Dashboard updated permissions, Write principals: {write_principals}") + + return read_principals, write_principals + + def _list_user_groups(self): + client = QuicksightClient.get_quicksight_client_in_identity_region(self._account_id) + user = self._describe_user() + if not user: + return [] + response = client.list_user_groups( + UserName=self._username, AwsAccountId=self._account_id, Namespace='default' + ) + return response['GroupList'] + + def _describe_user(self): + """Describes a QS user, returns None if not found""" + client = QuicksightClient.get_quicksight_client_in_identity_region(self._account_id) + try: + response = client.describe_user( + UserName=self._username, AwsAccountId=self._account_id, Namespace='default' + ) + except ClientError: + return None + return response.get('User') diff --git a/backend/dataall/modules/dashboards/cdk/__init__.py b/backend/dataall/modules/dashboards/cdk/__init__.py new file mode 100644 index 000000000..50217cecc --- /dev/null +++ b/backend/dataall/modules/dashboards/cdk/__init__.py @@ -0,0 +1,3 @@ +from dataall.modules.dashboards.cdk import dashboard_quicksight_policy + +__all__ = ['dashboard_quicksight_policy'] diff --git a/backend/dataall/cdkproxy/stacks/policies/quicksight.py b/backend/dataall/modules/dashboards/cdk/dashboard_quicksight_policy.py similarity index 88% rename from backend/dataall/cdkproxy/stacks/policies/quicksight.py rename to backend/dataall/modules/dashboards/cdk/dashboard_quicksight_policy.py index e67b3436c..fb237a22d 100644 --- a/backend/dataall/cdkproxy/stacks/policies/quicksight.py +++ b/backend/dataall/modules/dashboards/cdk/dashboard_quicksight_policy.py @@ -1,12 +1,12 @@ from aws_cdk import aws_iam as iam -from dataall.db import permissions -from .service_policy import ServicePolicy +from dataall.cdkproxy.stacks.policies.service_policy import ServicePolicy +from dataall.modules.dashboards.services.dashboard_permissions import CREATE_DASHBOARD -class QuickSight(ServicePolicy): +class QuickSightPolicy(ServicePolicy): def get_statements(self, group_permissions, **kwargs): - if permissions.CREATE_DASHBOARD not in group_permissions: + if CREATE_DASHBOARD not in group_permissions: return [] return [ diff --git a/backend/dataall/modules/dashboards/db/__init__.py b/backend/dataall/modules/dashboards/db/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/dataall/modules/dashboards/db/dashboard_repository.py b/backend/dataall/modules/dashboards/db/dashboard_repository.py new file mode 100644 index 000000000..0a3059f0f --- /dev/null +++ b/backend/dataall/modules/dashboards/db/dashboard_repository.py @@ -0,0 +1,189 @@ +import logging + +from sqlalchemy import or_, and_ +from sqlalchemy.orm import Query + +from dataall.core.group.services.environment_resource_manager import EnvironmentResource +from dataall.db import exceptions, paginate +from dataall.db.api import Environment +from dataall.modules.dashboards.db.models import DashboardShare, DashboardShareStatus, Dashboard + +logger = logging.getLogger(__name__) + + +class DashboardRepository(EnvironmentResource): + + @staticmethod + def count_resources(session, environment, group_uri) -> int: + return ( + session.query(Dashboard) + .filter( + and_( + Dashboard.environmentUri == environment.environmentUri, + Dashboard.SamlGroupName == group_uri + )) + .count() + ) + + @staticmethod + def update_env(session, environment): + return Environment.get_boolean_env_param(session, environment, "dashboardsEnabled") + + @staticmethod + def create_dashboard(session, env, username: str, data: dict = None) -> Dashboard: + dashboard: Dashboard = Dashboard( + label=data.get('label', 'untitled'), + environmentUri=data.get('environmentUri'), + organizationUri=env.organizationUri, + region=env.region, + DashboardId=data.get('dashboardId'), + AwsAccountId=env.AwsAccountId, + owner=username, + namespace='test', + tags=data.get('tags', []), + SamlGroupName=data['SamlGroupName'], + ) + session.add(dashboard) + session.commit() + return dashboard + + @staticmethod + def get_dashboard_by_uri(session, uri) -> Dashboard: + dashboard: Dashboard = session.query(Dashboard).get(uri) + if not dashboard: + raise exceptions.ObjectNotFound('Dashboard', uri) + return dashboard + + @staticmethod + def _query_user_dashboards(session, username, groups, filter) -> Query: + query = ( + session.query(Dashboard) + .outerjoin( + DashboardShare, + Dashboard.dashboardUri == DashboardShare.dashboardUri, + ) + .filter( + or_( + Dashboard.owner == username, + Dashboard.SamlGroupName.in_(groups), + and_( + DashboardShare.SamlGroupName.in_(groups), + DashboardShare.status + == DashboardShareStatus.APPROVED.value, + ), + ) + ) + ) + if filter and filter.get('term'): + query = query.filter( + or_( + Dashboard.description.ilike(filter.get('term') + '%%'), + Dashboard.label.ilike(filter.get('term') + '%%'), + ) + ) + return query + + @staticmethod + def paginated_user_dashboards( + session, username, groups, data=None + ) -> dict: + return paginate( + query=DashboardRepository._query_user_dashboards(session, username, groups, data), + page=data.get('page', 1), + page_size=data.get('pageSize', 10), + ).to_dict() + + @staticmethod + def _query_dashboard_shares(session, username, groups, uri, filter) -> Query: + query = ( + session.query(DashboardShare) + .join( + Dashboard, + Dashboard.dashboardUri == DashboardShare.dashboardUri, + ) + .filter( + and_( + DashboardShare.dashboardUri == uri, + or_( + Dashboard.owner == username, + Dashboard.SamlGroupName.in_(groups), + ), + ) + ) + ) + if filter and filter.get('term'): + query = query.filter( + or_( + DashboardShare.SamlGroupName.ilike( + filter.get('term') + '%%' + ), + Dashboard.label.ilike(filter.get('term') + '%%'), + ) + ) + return query + + @staticmethod + def query_all_user_groups_shareddashboard(session, groups, uri) -> [str]: + query = ( + session.query(DashboardShare) + .filter( + and_( + DashboardShare.dashboardUri == uri, + DashboardShare.SamlGroupName.in_(groups), + ) + ) + ) + + return [share.SamlGroupName for share in query.all()] + + @staticmethod + def paginated_dashboard_shares( + session, username, groups, uri, data=None + ) -> dict: + return paginate( + query=DashboardRepository._query_dashboard_shares( + session, username, groups, uri, data + ), + page=data.get('page', 1), + page_size=data.get('pageSize', 10), + ).to_dict() + + @staticmethod + def delete_dashboard(session, dashboard) -> bool: + session.delete(dashboard) + return True + + @staticmethod + def create_share( + session, + username: str, + dashboard: Dashboard, + principal_id: str, + init_status: DashboardShareStatus = DashboardShareStatus.REQUESTED + ) -> DashboardShare: + share = DashboardShare( + owner=username, + dashboardUri=dashboard.dashboardUri, + SamlGroupName=principal_id, + status=init_status.value, + ) + session.add(share) + return share + + @staticmethod + def get_dashboard_share_by_uri(session, uri) -> DashboardShare: + share: DashboardShare = session.query(DashboardShare).get(uri) + if not share: + raise exceptions.ObjectNotFound('DashboardShare', uri) + return share + + @staticmethod + def find_share_for_group(session, dashboard_uri, group) -> DashboardShare: + return ( + session.query(DashboardShare) + .filter( + DashboardShare.dashboardUri == dashboard_uri, + DashboardShare.SamlGroupName == group, + ) + .first() + ) diff --git a/backend/dataall/db/models/Dashboard.py b/backend/dataall/modules/dashboards/db/models.py similarity index 54% rename from backend/dataall/db/models/Dashboard.py rename to backend/dataall/modules/dashboards/db/models.py index 61c4d1400..d4cbe6f5b 100644 --- a/backend/dataall/db/models/Dashboard.py +++ b/backend/dataall/modules/dashboards/db/models.py @@ -1,7 +1,28 @@ +from enum import Enum + from sqlalchemy import Column, String, ForeignKey from sqlalchemy.orm import query_expression -from .. import Base, Resource, utils +from dataall.db import Base, Resource, utils + + +class DashboardShareStatus(Enum): + REQUESTED = 'REQUESTED' + APPROVED = 'APPROVED' + REJECTED = 'REJECTED' + + +class DashboardShare(Base): + __tablename__ = 'dashboardshare' + shareUri = Column( + String, nullable=False, primary_key=True, default=utils.uuid('shareddashboard') + ) + dashboardUri = Column(String, nullable=False, default=utils.uuid('dashboard')) + SamlGroupName = Column(String, nullable=False) + owner = Column(String, nullable=True) + status = Column( + String, nullable=False, default=DashboardShareStatus.REQUESTED.value + ) class Dashboard(Resource, Base): diff --git a/backend/dataall/modules/dashboards/indexers/__init__.py b/backend/dataall/modules/dashboards/indexers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/dataall/modules/dashboards/indexers/dashboard_catalog_indexer.py b/backend/dataall/modules/dashboards/indexers/dashboard_catalog_indexer.py new file mode 100644 index 000000000..44633e592 --- /dev/null +++ b/backend/dataall/modules/dashboards/indexers/dashboard_catalog_indexer.py @@ -0,0 +1,19 @@ +import logging + +from dataall.core.catalog.catalog_indexer import CatalogIndexer +from dataall.modules.dashboards import Dashboard +from dataall.modules.dashboards.indexers.dashboard_indexer import DashboardIndexer + +log = logging.getLogger(__name__) + + +class DashboardCatalogIndexer(CatalogIndexer): + + def index(self, session) -> int: + all_dashboards: [Dashboard] = session.query(Dashboard).all() + log.info(f'Found {len(all_dashboards)} dashboards') + dashboard: Dashboard + for dashboard in all_dashboards: + DashboardIndexer.upsert(session=session, dashboard_uri=dashboard.dashboardUri) + + return len(all_dashboards) diff --git a/backend/dataall/modules/dashboards/indexers/dashboard_indexer.py b/backend/dataall/modules/dashboards/indexers/dashboard_indexer.py new file mode 100644 index 000000000..6988696ae --- /dev/null +++ b/backend/dataall/modules/dashboards/indexers/dashboard_indexer.py @@ -0,0 +1,48 @@ +import logging + +from dataall import db +from dataall.db.api import Environment, Organization +from dataall.modules.dashboards import DashboardRepository +from dataall.searchproxy.base_indexer import BaseIndexer +from dataall.modules.dashboards.db.models import Dashboard + +log = logging.getLogger(__name__) + + +class DashboardIndexer(BaseIndexer): + @classmethod + def upsert(cls, session, dashboard_uri: str): + dashboard: Dashboard = DashboardRepository.get_dashboard_by_uri(session, dashboard_uri) + + if dashboard: + env = Environment.get_environment_by_uri(session, dashboard.environmentUri) + org = Organization.get_organization_by_uri(session, env.organizationUri) + + glossary = BaseIndexer._get_target_glossary_terms(session, dashboard_uri) + count_upvotes = db.api.Vote.count_upvotes( + session, None, None, dashboard_uri, {'targetType': 'dashboard'} + ) + BaseIndexer._index( + doc_id=dashboard_uri, + doc={ + 'name': dashboard.name, + 'admins': dashboard.SamlGroupName, + 'owner': dashboard.owner, + 'label': dashboard.label, + 'resourceKind': 'dashboard', + 'description': dashboard.description, + 'tags': [f.replace('-', '') for f in dashboard.tags or []], + 'topics': [], + 'region': dashboard.region.replace('-', ''), + 'environmentUri': env.environmentUri, + 'environmentName': env.name, + 'organizationUri': org.organizationUri, + 'organizationName': org.name, + 'created': dashboard.created, + 'updated': dashboard.updated, + 'deleted': dashboard.deleted, + 'glossary': glossary, + 'upvotes': count_upvotes, + }, + ) + return dashboard diff --git a/backend/dataall/modules/dashboards/services/__init__.py b/backend/dataall/modules/dashboards/services/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/dataall/modules/dashboards/services/dashboard_permissions.py b/backend/dataall/modules/dashboards/services/dashboard_permissions.py new file mode 100644 index 000000000..f5c55b89f --- /dev/null +++ b/backend/dataall/modules/dashboards/services/dashboard_permissions.py @@ -0,0 +1,41 @@ +from dataall.db.permissions import ENVIRONMENT_INVITED, ENVIRONMENT_INVITATION_REQUEST, ENVIRONMENT_ALL, TENANT_ALL, \ + TENANT_ALL_WITH_DESC, RESOURCES_ALL, RESOURCES_ALL_WITH_DESC + +""" +DASHBOARDS +""" +GET_DASHBOARD = 'GET_DASHBOARD' +UPDATE_DASHBOARD = 'UPDATE_DASHBOARD' +DELETE_DASHBOARD = 'DELETE_DASHBOARD' +SHARE_DASHBOARD = 'SHARE_DASHBOARD' +DASHBOARD_ALL = [ + GET_DASHBOARD, + UPDATE_DASHBOARD, + DELETE_DASHBOARD, + SHARE_DASHBOARD, +] + +RESOURCES_ALL.extend(DASHBOARD_ALL) +for perm in DASHBOARD_ALL: + RESOURCES_ALL_WITH_DESC[perm] = perm + +""" +TENANT PERMISSIONS +""" +MANAGE_DASHBOARDS = 'MANAGE_DASHBOARDS' + +TENANT_ALL.append(MANAGE_DASHBOARDS) +TENANT_ALL_WITH_DESC[MANAGE_DASHBOARDS] = 'Manage dashboards' + + +""" +ENVIRONMENT PERMISSIONS +""" +CREATE_DASHBOARD = 'CREATE_DASHBOARD' + + +ENVIRONMENT_INVITED.append(CREATE_DASHBOARD) +ENVIRONMENT_INVITATION_REQUEST.append(CREATE_DASHBOARD) +ENVIRONMENT_ALL.append(CREATE_DASHBOARD) +RESOURCES_ALL.append(CREATE_DASHBOARD) +RESOURCES_ALL_WITH_DESC[CREATE_DASHBOARD] = 'Create dashboards on this environment' diff --git a/backend/dataall/modules/dashboards/services/dashboard_quicksight_service.py b/backend/dataall/modules/dashboards/services/dashboard_quicksight_service.py new file mode 100644 index 000000000..1f9a4548c --- /dev/null +++ b/backend/dataall/modules/dashboards/services/dashboard_quicksight_service.py @@ -0,0 +1,169 @@ +import os + +from dataall.aws.handlers.parameter_store import ParameterStoreManager +from dataall.aws.handlers.sts import SessionHelper +from dataall.core.context import get_context +from dataall.core.permission_checker import has_resource_permission +from dataall.db.api import Environment, TenantPolicy +from dataall.db.exceptions import UnauthorizedOperation, TenantUnauthorized, AWSResourceNotFound +from dataall.db.permissions import TENANT_ALL +from dataall.modules.dashboards import DashboardRepository, Dashboard +from dataall.modules.dashboards.aws.dashboard_quicksight_client import DashboardQuicksightClient +from dataall.modules.dashboards.services.dashboard_permissions import GET_DASHBOARD, CREATE_DASHBOARD +from dataall.utils import Parameter + + +class DashboardQuicksightService: + _PARAM_STORE = Parameter() + _REGION = os.getenv('AWS_REGION', 'eu-west-1') + + @classmethod + @has_resource_permission(GET_DASHBOARD) + def get_quicksight_reader_url(cls, uri): + context = get_context() + with context.db_engine.scoped_session() as session: + dash: Dashboard = DashboardRepository.get_dashboard_by_uri(session, uri) + env = Environment.get_environment_by_uri(session, dash.environmentUri) + cls._check_dashboards_enabled(session, env, GET_DASHBOARD) + client = cls._client(env.AwsAccountId, env.region) + + if dash.SamlGroupName in context.groups: + return client.get_reader_session( + dashboard_id=dash.DashboardId, + domain_name=DashboardQuicksightService._get_domain_url(), + ) + + else: + shared_groups = DashboardRepository.query_all_user_groups_shareddashboard( + session=session, + groups=context.groups, + uri=uri + ) + if not shared_groups: + raise UnauthorizedOperation( + action=GET_DASHBOARD, + message='Dashboard has not been shared with your Teams', + ) + + session_type = ParameterStoreManager.get_parameter_value( + parameter_path=f"/dataall/{os.getenv('envname', 'local')}/quicksight/sharedDashboardsSessions" + ) + + if session_type == 'reader': + return client.get_shared_reader_session( + group_name=shared_groups[0], + dashboard_id=dash.DashboardId, + ) + else: + return client.get_anonymous_session(dashboard_id=dash.DashboardId) + + @classmethod + @has_resource_permission(CREATE_DASHBOARD) + def get_quicksight_designer_url(cls, uri: str): + context = get_context() + with context.db_engine.scoped_session() as session: + env = Environment.get_environment_by_uri(session, uri) + cls._check_dashboards_enabled(session, env, CREATE_DASHBOARD) + + return cls._client(env.AwsAccountId, env.region).get_author_session() + + @staticmethod + def get_monitoring_dashboard_id(): + current_account = SessionHelper.get_account() + dashboard_id = ParameterStoreManager.get_parameter_value( + AwsAccountId=current_account, + region=DashboardQuicksightService._REGION, + parameter_path=f'/dataall/{os.getenv("envname", "local")}/quicksightmonitoring/DashboardId' + ) + + if not dashboard_id: + raise AWSResourceNotFound( + action='GET_DASHBOARD_ID', + message='Dashboard Id could not be found on AWS Parameter Store', + ) + return dashboard_id + + @staticmethod + def get_monitoring_vpc_connection_id(): + current_account = SessionHelper.get_account() + vpc_connection_id = ParameterStoreManager.get_parameter_value( + AwsAccountId=current_account, + region=DashboardQuicksightService._REGION, + parameter_path=f'/dataall/{os.getenv("envname", "local")}/quicksightmonitoring/VPCConnectionId' + ) + + if not vpc_connection_id: + raise AWSResourceNotFound( + action='GET_VPC_CONNECTION_ID', + message='VPC Connection Id could not be found on AWS Parameter Store', + ) + return vpc_connection_id + + @classmethod + def create_quicksight_data_source_set(cls, vpc_connection_id): + client = cls._client() + client.register_user_in_group(group_name='dataall', user_role='AUTHOR') + + datasource_id = client.create_data_source_vpc(vpc_connection_id=vpc_connection_id) + # Data sets are not created programmatically. Too much overhead for the value added. + # However, an example API is provided: datasets = Quicksight.create_data_set_from_source( + # AwsAccountId=current_account, region=region, UserName='dataallTenantUser', + # dataSourceId=datasourceId, tablesToImport=['organization', + # 'environment', 'dataset', 'datapipeline', 'dashboard', 'share_object'] + # ) + + return datasource_id + + @classmethod + def get_quicksight_author_session(cls, aws_account): + DashboardQuicksightService._check_user_must_be_admin() + return cls._client(aws_account).get_author_session() + + @classmethod + def get_quicksight_reader_session(cls, dashboard_uri): + cls._check_user_must_be_admin() + client = cls._client() + return client.get_reader_session(user_role='READER', dashboard_id=dashboard_uri) + + @staticmethod + def _check_user_must_be_admin(): + context = get_context() + admin = TenantPolicy.is_tenant_admin(context.groups) + + if not admin: + raise TenantUnauthorized( + username=context.username, + action=TENANT_ALL, + tenant_name=context.username, + ) + + @staticmethod + def _get_domain_url(): + envname = os.getenv("envname", "local") + if envname in ["local", "dkrcompose"]: + return "http://localhost:8080" + + domain_name = DashboardQuicksightService._PARAM_STORE.get_parameter( + env=envname, + path="frontend/custom_domain_name" + ) + + return f"https://{domain_name}" + + @staticmethod + def _check_dashboards_enabled(session, env, action): + enabled = Environment.get_boolean_env_param(session, env, "dashboardsEnabled") + if not enabled: + raise UnauthorizedOperation( + action=action, + message=f'Dashboards feature is disabled for the environment {env.label}', + ) + + @classmethod + def _client(cls, account_id: str = None, region: str = None): + if not account_id: + account_id = SessionHelper.get_account() + + if not region: + region = cls._REGION + return DashboardQuicksightClient(get_context().username, account_id, region) diff --git a/backend/dataall/modules/dashboards/services/dashboard_service.py b/backend/dataall/modules/dashboards/services/dashboard_service.py new file mode 100644 index 000000000..156dddf5e --- /dev/null +++ b/backend/dataall/modules/dashboards/services/dashboard_service.py @@ -0,0 +1,133 @@ +from dataall.core.context import get_context +from dataall.core.permission_checker import has_tenant_permission, has_resource_permission, has_group_permission +from dataall.db.api import ResourcePolicy, Glossary, Vote, Environment +from dataall.db.exceptions import UnauthorizedOperation +from dataall.db.models import Activity +from dataall.modules.dashboards import DashboardRepository, Dashboard +from dataall.modules.dashboards.aws.dashboard_quicksight_client import DashboardQuicksightClient +from dataall.modules.dashboards.indexers.dashboard_indexer import DashboardIndexer +from dataall.modules.dashboards.services.dashboard_permissions import MANAGE_DASHBOARDS, GET_DASHBOARD, \ + UPDATE_DASHBOARD, CREATE_DASHBOARD, DASHBOARD_ALL + + +class DashboardService: + """Service that serves request related to dashboard""" + @staticmethod + @has_tenant_permission(MANAGE_DASHBOARDS) + @has_resource_permission(GET_DASHBOARD) + def get_dashboard(uri: str) -> Dashboard: + with get_context().db_engine.scoped_session() as session: + return DashboardRepository.get_dashboard_by_uri(session, uri) + + @staticmethod + @has_tenant_permission(MANAGE_DASHBOARDS) + @has_resource_permission(CREATE_DASHBOARD) + @has_group_permission(CREATE_DASHBOARD) + def import_dashboard(uri: str, admin_group: str, data: dict = None) -> Dashboard: + context = get_context() + with context.db_engine.scoped_session() as session: + env = Environment.get_environment_by_uri(session, data['environmentUri']) + enabled = Environment.get_boolean_env_param(session, env, "dashboardsEnabled") + + if not enabled: + raise UnauthorizedOperation( + action=CREATE_DASHBOARD, + message=f'Dashboards feature is disabled for the environment {env.label}', + ) + + aws_client = DashboardQuicksightClient(context.username, env.AwsAccountId, env.region) + can_import = aws_client.can_import_dashboard(data.get('dashboardId')) + + if not can_import: + raise UnauthorizedOperation( + action=CREATE_DASHBOARD, + message=f'User: {context.username} has not AUTHOR rights on quicksight for the environment {env.label}', + ) + + env = data.get( + 'environment', Environment.get_environment_by_uri(session, uri) + ) + + dashboard = DashboardRepository.create_dashboard(session, env, context.username, data) + + activity = Activity( + action='DASHBOARD:CREATE', + label='DASHBOARD:CREATE', + owner=context.username, + summary=f'{context.username} created dashboard {dashboard.label} in {env.label}', + targetUri=dashboard.dashboardUri, + targetType='dashboard', + ) + session.add(activity) + + DashboardService._set_dashboard_resource_policy( + session, env, dashboard, data['SamlGroupName'] + ) + + DashboardService._update_glossary(session, dashboard, data) + DashboardIndexer.upsert(session, dashboard_uri=dashboard.dashboardUri) + return dashboard + + @staticmethod + @has_tenant_permission(MANAGE_DASHBOARDS) + @has_resource_permission(UPDATE_DASHBOARD) + def update_dashboard(uri: str, data: dict = None) -> Dashboard: + with get_context().db_engine.scoped_session() as session: + dashboard = DashboardRepository.get_dashboard_by_uri(session, uri) + for k in data.keys(): + setattr(dashboard, k, data.get(k)) + + DashboardService._update_glossary(session, dashboard, data) + environment = Environment.get_environment_by_uri(session, dashboard.environmentUri) + DashboardService._set_dashboard_resource_policy( + session, environment, dashboard, dashboard.SamlGroupName + ) + + DashboardIndexer.upsert(session, dashboard_uri=dashboard.dashboardUri) + return dashboard + + @staticmethod + def delete_dashboard(uri) -> bool: + # TODO THERE WAS NO PERMISSION CHECK + with get_context().db_engine.scoped_session() as session: + dashboard = DashboardRepository.get_dashboard_by_uri(session, uri) + DashboardRepository.delete_dashboard(session, dashboard) + + ResourcePolicy.delete_resource_policy( + session=session, resource_uri=uri, group=dashboard.SamlGroupName + ) + Glossary.delete_glossary_terms_links( + session, target_uri=dashboard.dashboardUri, target_type='Dashboard' + ) + Vote.delete_votes(session, dashboard.dashboardUri, 'dashboard') + + DashboardIndexer.delete_doc(doc_id=uri) + return True + + @staticmethod + def _set_dashboard_resource_policy(session, environment, dashboard, group): + DashboardService._attach_dashboard_policy(session, group, dashboard) + if environment.SamlGroupName != dashboard.SamlGroupName: + DashboardService._attach_dashboard_policy(session, environment.SamlGroupName, dashboard) + + @staticmethod + def _attach_dashboard_policy(session, group: str, dashboard: Dashboard): + ResourcePolicy.attach_resource_policy( + session=session, + group=group, + permissions=DASHBOARD_ALL, + resource_uri=dashboard.dashboardUri, + resource_type=Dashboard.__name__, + ) + + @staticmethod + def _update_glossary(session, dashboard, data): + context = get_context() + if 'terms' in data: + Glossary.set_glossary_terms_links( + session, + context.username, + dashboard.dashboardUri, + 'Dashboard', + data['terms'], + ) diff --git a/backend/dataall/modules/dashboards/services/dashboard_share_service.py b/backend/dataall/modules/dashboards/services/dashboard_share_service.py new file mode 100644 index 000000000..2431b653a --- /dev/null +++ b/backend/dataall/modules/dashboards/services/dashboard_share_service.py @@ -0,0 +1,123 @@ +from dataall.core.context import get_context +from dataall.core.permission_checker import has_tenant_permission, has_resource_permission +from dataall.db.api import ResourcePolicy +from dataall.db.exceptions import InvalidInput, UnauthorizedOperation +from dataall.modules.dashboards import DashboardRepository +from dataall.modules.dashboards.db.models import DashboardShareStatus, Dashboard +from dataall.modules.dashboards.services.dashboard_permissions import SHARE_DASHBOARD, MANAGE_DASHBOARDS, GET_DASHBOARD, \ + CREATE_DASHBOARD + + +class DashboardShareService: + @staticmethod + def _get_dashboard_uri_by_share_uri(session, uri): + share = DashboardRepository.get_dashboard_share_by_uri(session, uri) + dashboard = DashboardRepository.get_dashboard_by_uri(session, share.dashboardUri) + return dashboard.dashboardUri + + @staticmethod + @has_tenant_permission(MANAGE_DASHBOARDS) + def request_dashboard_share(uri: str, principal_id: str): + context = get_context() + with context.db_engine.scoped_session() as session: + dashboard = DashboardRepository.get_dashboard_by_uri(session, uri) + if dashboard.SamlGroupName == principal_id: + raise UnauthorizedOperation( + action=CREATE_DASHBOARD, + message=f'Team {dashboard.SamlGroupName} is the owner of the dashboard {dashboard.label}', + ) + + share = DashboardRepository.find_share_for_group(session, dashboard.dashboardUri, principal_id) + if not share: + share = DashboardRepository.create_share(session, context.username, dashboard, principal_id) + else: + DashboardShareService._check_share_status(share) + + if share.status == DashboardShareStatus.REJECTED.value: + share.status = DashboardShareStatus.REQUESTED.value + + return share + + @staticmethod + @has_tenant_permission(MANAGE_DASHBOARDS) + @has_resource_permission(SHARE_DASHBOARD, parent_resource=_get_dashboard_uri_by_share_uri) + def approve_dashboard_share(uri: str): + with get_context().db_engine.scoped_session() as session: + share = DashboardRepository.get_dashboard_share_by_uri(session, uri) + DashboardShareService._change_share_status(share, DashboardShareStatus.APPROVED) + DashboardShareService._create_share_policy(session, share.SamlGroupName, share.dashboardUri) + return share + + @staticmethod + @has_tenant_permission(MANAGE_DASHBOARDS) + @has_resource_permission(SHARE_DASHBOARD, parent_resource=_get_dashboard_uri_by_share_uri) + def reject_dashboard_share(uri: str): + with get_context().db_engine.scoped_session() as session: + share = DashboardRepository.get_dashboard_share_by_uri(session, uri) + DashboardShareService._change_share_status(share, DashboardShareStatus.REJECTED) + + ResourcePolicy.delete_resource_policy( + session=session, + group=share.SamlGroupName, + resource_uri=share.dashboardUri, + resource_type=Dashboard.__name__, + ) + + return share + + @staticmethod + def list_dashboard_shares(uri: str, data: dict): + context = get_context() + with context.db_engine.scoped_session() as session: + return DashboardRepository.paginated_dashboard_shares( + session=session, + username=context.username, + groups=context.groups, + uri=uri, + data=data, + ) + + @staticmethod + @has_tenant_permission(MANAGE_DASHBOARDS) + @has_resource_permission(SHARE_DASHBOARD) + def share_dashboard(uri: str, principal_id: str): + context = get_context() + with context.db_engine.scoped_session() as session: + dashboard = DashboardRepository.get_dashboard_by_uri(session, uri) + share = DashboardRepository.create_share( + session=session, + username=context.username, + dashboard=dashboard, + principal_id=principal_id, + init_status=DashboardShareStatus.APPROVED + ) + + DashboardShareService._create_share_policy(session, principal_id, dashboard.dashboardUri) + return share + + @staticmethod + def _change_share_status(share, status): + DashboardShareService._check_share_status(share) + if share.status == status.value: + return share + + share.status = status.value + + @staticmethod + def _check_share_status(share): + if share.status not in DashboardShareStatus.__members__: + raise InvalidInput( + 'Share status', + share.status, + str(DashboardShareStatus.__members__), + ) + + @staticmethod + def _create_share_policy(session, principal_id, dashboard_uri): + ResourcePolicy.attach_resource_policy( + session=session, + group=principal_id, + permissions=[GET_DASHBOARD], + resource_uri=dashboard_uri, + resource_type=Dashboard.__name__, + ) diff --git a/backend/dataall/modules/datapipelines/services/datapipelines_service.py b/backend/dataall/modules/datapipelines/services/datapipelines_service.py index 35a9d48ba..c5ae7a269 100644 --- a/backend/dataall/modules/datapipelines/services/datapipelines_service.py +++ b/backend/dataall/modules/datapipelines/services/datapipelines_service.py @@ -2,7 +2,6 @@ import logging from dataall.aws.handlers.sts import SessionHelper -from dataall.core.environment.db.repositories import EnvironmentParameterRepository from dataall.core.permission_checker import has_resource_permission, has_tenant_permission, \ has_group_permission from dataall.db.api import ( @@ -41,9 +40,9 @@ def create_pipeline( ) -> DataPipeline: environment = Environment.get_environment_by_uri(session, uri) - enabled = EnvironmentParameterRepository(session).get_param(uri, "pipelinesEnabled") + enabled = Environment.get_boolean_env_param(session, environment, "pipelinesEnabled") - if not enabled and enabled.lower() != "true": + if not enabled: raise exceptions.UnauthorizedOperation( action=CREATE_PIPELINE, message=f'Pipelines feature is disabled for the environment {environment.label}', @@ -116,9 +115,9 @@ def create_pipeline_environment( ) -> DataPipelineEnvironment: environment = Environment.get_environment_by_uri(session, data['environmentUri']) - enabled = EnvironmentParameterRepository(session).get_param(uri, "pipelinesEnabled") + enabled = Environment.get_boolean_env_param(session, environment, "pipelinesEnabled") - if not enabled and enabled.lower() != "true": + if not enabled: raise exceptions.UnauthorizedOperation( action=CREATE_PIPELINE, message=f'Pipelines feature is disabled for the environment {environment.label}', diff --git a/backend/dataall/modules/dataset_sharing/services/share_managers/lf_share_manager.py b/backend/dataall/modules/dataset_sharing/services/share_managers/lf_share_manager.py index cdf164cd9..96940f518 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_managers/lf_share_manager.py +++ b/backend/dataall/modules/dataset_sharing/services/share_managers/lf_share_manager.py @@ -5,9 +5,10 @@ from botocore.exceptions import ClientError +from dataall.db.api import Environment from dataall.modules.dataset_sharing.aws.glue_client import GlueClient from dataall.modules.dataset_sharing.aws.lakeformation_client import LakeFormationClient -from dataall.aws.handlers.quicksight import Quicksight +from dataall.aws.handlers.quicksight import QuicksightClient from dataall.aws.handlers.sts import SessionHelper from dataall.aws.handlers.ram import Ram from dataall.db import exceptions, models @@ -41,12 +42,6 @@ def __init__( self.shared_db_name = self.build_shared_db_name() self.principals = self.get_share_principals() - self.glue_client = GlueClient( - account_id=self.target_environment.AwsAccountId, - region=self.target_environment.region, - database=self.shared_db_name, - ) - @abc.abstractmethod def process_approved_shares(self) -> [str]: return NotImplementedError @@ -67,8 +62,10 @@ def get_share_principals(self) -> [str]: List of principals """ principals = [f"arn:aws:iam::{self.target_environment.AwsAccountId}:role/{self.share.principalIAMRoleName}"] - if self.target_environment.dashboardsEnabled: - group = Quicksight.create_quicksight_group(AwsAccountId=self.target_environment.AwsAccountId) + dashboard_enabled = Environment.get_boolean_env_param(self.session, self.target_environment, "dashboardsEnabled") + + if dashboard_enabled: + group = QuicksightClient.create_quicksight_group(AwsAccountId=self.target_environment.AwsAccountId) if group and group.get('Group'): group_arn = group.get('Group').get('Arn') if group_arn: @@ -130,7 +127,7 @@ def check_share_item_exists_on_glue_catalog( ------- exceptions.AWSResourceNotFound """ - if not self.glue_client.table_exists(table.GlueTableName): + if not self.glue_client().table_exists(table.GlueTableName): raise exceptions.AWSResourceNotFound( action='ProcessShare', message=( @@ -211,7 +208,7 @@ def delete_shared_database(self) -> bool: bool """ logger.info(f'Deleting shared database {self.shared_db_name}') - return self.glue_client.delete_database() + return self.glue_client().delete_database() @classmethod def create_resource_link(cls, **data) -> dict: @@ -276,7 +273,7 @@ def revoke_table_resource_link_access(self, table: DatasetTable, principals: [st ------- True if revoke is successful """ - glue_client = self.glue_client + glue_client = self.glue_client() if not glue_client.table_exists(table.GlueTableName): logger.info( f'Resource link could not be found ' @@ -327,7 +324,7 @@ def revoke_source_table_access(self, table, principals: [str]): ------- True if revoke is successful """ - glue_client = self.glue_client + glue_client = self.glue_client() if not glue_client.table_exists(table.GlueTableName): logger.info( f'Source table could not be found ' @@ -353,7 +350,7 @@ def revoke_source_table_access(self, table, principals: [str]): def delete_resource_link_table(self, table: DatasetTable): logger.info(f'Deleting shared table {table.GlueTableName}') - glue_client = self.glue_client + glue_client = self.glue_client() if not glue_client.table_exists(table.GlueTableName): return True @@ -529,3 +526,10 @@ def handle_revoke_failure( table, self.share, self.target_environment ) return True + + def glue_client(self): + return GlueClient( + account_id=self.target_environment.AwsAccountId, + region=self.target_environment.region, + database=self.shared_db_name, + ) diff --git a/backend/dataall/modules/datasets/api/table/mutations.py b/backend/dataall/modules/datasets/api/table/mutations.py index 16c4e09ac..c6e111fc3 100644 --- a/backend/dataall/modules/datasets/api/table/mutations.py +++ b/backend/dataall/modules/datasets/api/table/mutations.py @@ -27,4 +27,3 @@ type=gql.Ref('DatasetTableSearchResult'), resolver=sync_tables, ) - diff --git a/backend/dataall/modules/datasets/cdk/dataset_stack.py b/backend/dataall/modules/datasets/cdk/dataset_stack.py index 34df41906..9637f69fb 100644 --- a/backend/dataall/modules/datasets/cdk/dataset_stack.py +++ b/backend/dataall/modules/datasets/cdk/dataset_stack.py @@ -19,7 +19,7 @@ from dataall.cdkproxy.stacks.manager import stack from dataall import db -from dataall.aws.handlers.quicksight import Quicksight +from dataall.aws.handlers.quicksight import QuicksightClient from dataall.aws.handlers.sts import SessionHelper from dataall.db import models from dataall.db.api import Environment @@ -78,6 +78,10 @@ def get_target(self) -> Dataset: raise Exception('ObjectNotFound') return dataset + def has_quicksight_enabled(self, env) -> bool: + with self.get_engine().scoped_session() as session: + return Environment.get_boolean_env_param(session, env, "dashboardsEnabled") + def __init__(self, scope, id, target_uri: str = None, **kwargs): super().__init__( scope, @@ -97,8 +101,8 @@ def __init__(self, scope, id, target_uri: str = None, **kwargs): env_group = self.get_env_group(dataset) quicksight_default_group_arn = None - if env.dashboardsEnabled: - quicksight_default_group = Quicksight.create_quicksight_group(AwsAccountId=env.AwsAccountId) + if self.has_quicksight_enabled(env): + quicksight_default_group = QuicksightClient.create_quicksight_group(AwsAccountId=env.AwsAccountId) quicksight_default_group_arn = quicksight_default_group['Group']['Arn'] # Dataset S3 Bucket and KMS key diff --git a/backend/dataall/modules/datasets/services/dataset_column_service.py b/backend/dataall/modules/datasets/services/dataset_column_service.py index e6b16a790..08d299ae4 100644 --- a/backend/dataall/modules/datasets/services/dataset_column_service.py +++ b/backend/dataall/modules/datasets/services/dataset_column_service.py @@ -58,4 +58,3 @@ def update_table_column_description(column_uri: str, description) -> DatasetTabl Worker.queue(engine=get_context().db_engine, task_ids=[task.taskUri]) return column - diff --git a/backend/dataall/modules/datasets/services/dataset_service.py b/backend/dataall/modules/datasets/services/dataset_service.py index b0193ec77..0f93d4d64 100644 --- a/backend/dataall/modules/datasets/services/dataset_service.py +++ b/backend/dataall/modules/datasets/services/dataset_service.py @@ -2,7 +2,7 @@ import logging from dataall.api.Objects.Stack import stack_helper -from dataall.aws.handlers.quicksight import Quicksight +from dataall.aws.handlers.quicksight import QuicksightClient from dataall.aws.handlers.service_handlers import Worker from dataall.aws.handlers.sts import SessionHelper from dataall.core.context import get_context @@ -32,12 +32,13 @@ class DatasetService: @staticmethod - def check_dataset_account(environment): - if environment.dashboardsEnabled: - quicksight_subscription = Quicksight.check_quicksight_enterprise_subscription( + def check_dataset_account(session, environment): + dashboards_enabled = Environment.get_boolean_env_param(session, environment, "dashboardsEnabled") + if dashboards_enabled: + quicksight_subscription = QuicksightClient.check_quicksight_enterprise_subscription( AwsAccountId=environment.AwsAccountId) if quicksight_subscription: - group = Quicksight.create_quicksight_group(AwsAccountId=environment.AwsAccountId) + group = QuicksightClient.create_quicksight_group(AwsAccountId=environment.AwsAccountId) return True if group else False return True @@ -49,7 +50,7 @@ def create_dataset(uri, admin_group, data: dict): context = get_context() with context.db_engine.scoped_session() as session: environment = Environment.get_environment_by_uri(session, uri) - DatasetService.check_dataset_account(environment=environment) + DatasetService.check_dataset_account(session=session, environment=environment) dataset = DatasetRepository.create_dataset( session=session, @@ -149,7 +150,7 @@ def update_dataset(uri: str, data: dict): with get_context().db_engine.scoped_session() as session: dataset = DatasetRepository.get_dataset_by_uri(session, uri) environment = Environment.get_environment_by_uri(session, dataset.environmentUri) - DatasetService.check_dataset_account(environment=environment) + DatasetService.check_dataset_account(session=session, environment=environment) username = get_context().username dataset: Dataset = DatasetRepository.get_dataset_by_uri(session, uri) diff --git a/backend/dataall/modules/mlstudio/services/mlstudio_service.py b/backend/dataall/modules/mlstudio/services/mlstudio_service.py index a890d4627..3492d11f3 100644 --- a/backend/dataall/modules/mlstudio/services/mlstudio_service.py +++ b/backend/dataall/modules/mlstudio/services/mlstudio_service.py @@ -9,7 +9,6 @@ from dataall.api.Objects.Stack import stack_helper from dataall.core.context import get_context -from dataall.core.environment.db.repositories import EnvironmentParameterRepository from dataall.db.api import ( ResourcePolicy, Environment, KeyValueTag, Stack, @@ -72,9 +71,9 @@ def create_sagemaker_studio_user(*, uri: str, admin_group: str, request: Sagemak """ with _session() as session: env = Environment.get_environment_by_uri(session, uri) - enabled = EnvironmentParameterRepository(session).get_param(uri, "mlStudiosEnabled") + enabled = Environment.get_boolean_env_param(session, env, "mlStudiosEnabled") - if not enabled and enabled.lower() != "true": + if not enabled: raise exceptions.UnauthorizedOperation( action=CREATE_SGMSTUDIO_USER, message=f'ML Studio feature is disabled for the environment {env.label}', diff --git a/backend/dataall/modules/notebooks/services/notebook_service.py b/backend/dataall/modules/notebooks/services/notebook_service.py index 3049ca870..f40bb4395 100644 --- a/backend/dataall/modules/notebooks/services/notebook_service.py +++ b/backend/dataall/modules/notebooks/services/notebook_service.py @@ -10,7 +10,6 @@ from dataall.api.Objects.Stack import stack_helper from dataall.core.context import get_context as context -from dataall.core.environment.db.repositories import EnvironmentParameterRepository from dataall.db.api import ( ResourcePolicy, Environment, KeyValueTag, Stack, @@ -73,9 +72,9 @@ def create_notebook(*, uri: str, admin_group: str, request: NotebookCreationRequ with _session() as session: env = Environment.get_environment_by_uri(session, uri) - enabled = EnvironmentParameterRepository(session).get_param(uri, "notebooksEnabled") + enabled = Environment.get_boolean_env_param(session, env, "notebooksEnabled") - if not enabled and enabled.lower() != "true": + if not enabled: raise exceptions.UnauthorizedOperation( action=CREATE_NOTEBOOK, message=f'Notebooks feature is disabled for the environment {env.label}', diff --git a/backend/dataall/searchproxy/indexers.py b/backend/dataall/searchproxy/indexers.py deleted file mode 100644 index 4655de65a..000000000 --- a/backend/dataall/searchproxy/indexers.py +++ /dev/null @@ -1,71 +0,0 @@ -import logging - -from .. import db -from ..db import models -from dataall.searchproxy.base_indexer import BaseIndexer - -log = logging.getLogger(__name__) - - -# TODO Should be moved to dashboard module -class DashboardIndexer(BaseIndexer): - @classmethod - def upsert(cls, session, dashboard_uri: str): - dashboard = ( - session.query( - models.Dashboard.dashboardUri.label('uri'), - models.Dashboard.name.label('name'), - models.Dashboard.owner.label('owner'), - models.Dashboard.label.label('label'), - models.Dashboard.description.label('description'), - models.Dashboard.tags.label('tags'), - models.Dashboard.region.label('region'), - models.Organization.organizationUri.label('orgUri'), - models.Organization.name.label('orgName'), - models.Environment.environmentUri.label('envUri'), - models.Environment.name.label('envName'), - models.Dashboard.SamlGroupName.label('admins'), - models.Dashboard.created, - models.Dashboard.updated, - models.Dashboard.deleted, - ) - .join( - models.Organization, - models.Dashboard.organizationUri == models.Dashboard.organizationUri, - ) - .join( - models.Environment, - models.Dashboard.environmentUri == models.Environment.environmentUri, - ) - .filter(models.Dashboard.dashboardUri == dashboard_uri) - .first() - ) - if dashboard: - glossary = BaseIndexer._get_target_glossary_terms(session, dashboard_uri) - count_upvotes = db.api.Vote.count_upvotes( - session, None, None, dashboard_uri, {'targetType': 'dashboard'} - ) - BaseIndexer._index( - doc_id=dashboard_uri, - doc={ - 'name': dashboard.name, - 'admins': dashboard.admins, - 'owner': dashboard.owner, - 'label': dashboard.label, - 'resourceKind': 'dashboard', - 'description': dashboard.description, - 'tags': [f.replace('-', '') for f in dashboard.tags or []], - 'topics': [], - 'region': dashboard.region.replace('-', ''), - 'environmentUri': dashboard.envUri, - 'environmentName': dashboard.envName, - 'organizationUri': dashboard.orgUri, - 'organizationName': dashboard.orgName, - 'created': dashboard.created, - 'updated': dashboard.updated, - 'deleted': dashboard.deleted, - 'glossary': glossary, - 'upvotes': count_upvotes, - }, - ) - return dashboard diff --git a/backend/dataall/tasks/catalog_indexer_task.py b/backend/dataall/tasks/catalog_indexer_task.py index e25f0902d..a4fa3169e 100644 --- a/backend/dataall/tasks/catalog_indexer_task.py +++ b/backend/dataall/tasks/catalog_indexer_task.py @@ -3,9 +3,8 @@ import sys from dataall.core.catalog.catalog_indexer import CatalogIndexer -from dataall.db import get_engine, models +from dataall.db import get_engine from dataall.modules.loader import load_modules, ImportMode -from dataall.searchproxy.indexers import DashboardIndexer from dataall.utils.alarm_service import AlarmService root = logging.getLogger() @@ -22,13 +21,6 @@ def index_objects(engine): for indexer in CatalogIndexer.all(): indexed_objects_counter += indexer.index(session) - all_dashboards: [models.Dashboard] = session.query(models.Dashboard).all() - log.info(f'Found {len(all_dashboards)} dashboards') - dashboard: models.Dashboard - for dashboard in all_dashboards: - DashboardIndexer.upsert(session=session, dashboard_uri=dashboard.dashboardUri) - indexed_objects_counter = indexed_objects_counter + 1 - log.info(f'Successfully indexed {indexed_objects_counter} objects') return indexed_objects_counter except Exception as e: diff --git a/backend/migrations/versions/5fc49baecea4_add_enviromental_parameters.py b/backend/migrations/versions/5fc49baecea4_add_enviromental_parameters.py index c440d135e..871a533e9 100644 --- a/backend/migrations/versions/5fc49baecea4_add_enviromental_parameters.py +++ b/backend/migrations/versions/5fc49baecea4_add_enviromental_parameters.py @@ -25,9 +25,9 @@ Base = declarative_base() -UNUSED_PERMISSIONS = ['LIST_DATASETS', 'LIST_DATASET_TABLES', 'LIST_DATASET_SHARES', 'SUMMARY_DATASET', +UNUSED_PERMISSIONS = ['LIST_DATASETS', 'LIST_DATASET_TABLES', 'LIST_DATASET_SHARES', 'SUMMARY_DATASET', 'IMPORT_DATASET', 'UPLOAD_DATASET', 'URL_DATASET', 'STACK_DATASET', 'SUBSCRIPTIONS_DATASET', - 'CREATE_DATASET_TABLE', 'LIST_PIPELINES'] + 'CREATE_DATASET_TABLE', 'LIST_PIPELINES', 'DASHBOARD_URL'] class Environment(Resource, Base): @@ -36,6 +36,7 @@ class Environment(Resource, Base): notebooksEnabled = Column(Boolean) mlStudiosEnabled = Column(Boolean) pipelinesEnabled = Column(Boolean) + dashboardsEnabled = Column(Boolean) class EnvironmentParameter(Base): @@ -84,6 +85,9 @@ def upgrade(): _add_param_if_exists( params, env, "pipelinesEnabled", str(env.pipelinesEnabled).lower() # for frontend ) + _add_param_if_exists( + params, env, "dashboardsEnabled", str(env.dashboardsEnabled).lower() # for frontend + ) session.add_all(params) print("Migration of the environmental parameters has been complete") @@ -91,6 +95,7 @@ def upgrade(): op.drop_column("environment", "notebooksEnabled") op.drop_column("environment", "mlStudiosEnabled") op.drop_column("environment", "pipelinesEnabled") + op.drop_column("environment", "dashboardsEnabled") print("Dropped the columns from the environment table ") create_foreign_key_to_env(op, 'sagemaker_notebook') @@ -125,6 +130,7 @@ def downgrade(): op.add_column("environment", Column("notebooksEnabled", Boolean, default=True)) op.add_column("environment", Column("mlStudiosEnabled", Boolean, default=True)) op.add_column("environment", Column("pipelinesEnabled", Boolean, default=True)) + op.add_column("environment", Column("dashboardsEnabled", Boolean, default=True)) print("Filling environment table with parameters rows...") params = session.query(EnvironmentParameter).all() @@ -135,7 +141,8 @@ def downgrade(): environmentUri=param.environmentUri, notebooksEnabled=params["notebooksEnabled"] == "true", mlStudiosEnabled=params["mlStudiosEnabled"] == "true", - pipelinesEnabled=params["pipelinesEnabled"] == "true" + pipelinesEnabled=params["pipelinesEnabled"] == "true", + dashboardsEnabled=params["dashboardsEnabled"] == "true" )) for name in UNUSED_PERMISSIONS: @@ -145,7 +152,6 @@ def downgrade(): print("Dropping environment_parameter table...") op.drop_table("environment_parameters") - except Exception as ex: print(f"Failed to execute the rollback script due to: {ex}") diff --git a/config.json b/config.json index a77719964..96fafbd7d 100644 --- a/config.json +++ b/config.json @@ -14,6 +14,9 @@ }, "worksheets": { "active": true + }, + "dashboards": { + "active": true } } } \ No newline at end of file diff --git a/frontend/src/api/Dashboard/deleteDashboard.js b/frontend/src/api/Dashboard/deleteDashboard.js index 7b9d71eac..e9dd2e1bd 100644 --- a/frontend/src/api/Dashboard/deleteDashboard.js +++ b/frontend/src/api/Dashboard/deleteDashboard.js @@ -5,7 +5,7 @@ const deleteDashboard = (dashboardUri) => ({ dashboardUri }, mutation: gql` - mutation importDashboard($dashboardUri: String!) { + mutation deleteDashboard($dashboardUri: String!) { deleteDashboard(dashboardUri: $dashboardUri) } ` diff --git a/frontend/src/api/DatasetTable/previewTable2.js b/frontend/src/api/DatasetTable/previewTable.js similarity index 100% rename from frontend/src/api/DatasetTable/previewTable2.js rename to frontend/src/api/DatasetTable/previewTable.js diff --git a/frontend/src/api/Environment/createEnvironment.js b/frontend/src/api/Environment/createEnvironment.js index 937d40d0f..6eb2c3b1c 100644 --- a/frontend/src/api/Environment/createEnvironment.js +++ b/frontend/src/api/Environment/createEnvironment.js @@ -13,7 +13,6 @@ const createEnvironment = (input) => ({ SamlGroupName AwsAccountId created - dashboardsEnabled warehousesEnabled parameters { key diff --git a/frontend/src/api/Environment/getEnvironment.js b/frontend/src/api/Environment/getEnvironment.js index 619fb64e8..7a995b51f 100644 --- a/frontend/src/api/Environment/getEnvironment.js +++ b/frontend/src/api/Environment/getEnvironment.js @@ -14,7 +14,6 @@ const getEnvironment = ({ environmentUri }) => ({ name label AwsAccountId - dashboardsEnabled warehousesEnabled region owner diff --git a/frontend/src/api/Environment/listOrganizationEnvironments.js b/frontend/src/api/Environment/listOrganizationEnvironments.js index 2871d1b4b..fe89eb5a9 100644 --- a/frontend/src/api/Environment/listOrganizationEnvironments.js +++ b/frontend/src/api/Environment/listOrganizationEnvironments.js @@ -32,7 +32,6 @@ const listOrganizationEnvironments = ({ organizationUri, filter }) => ({ tags environmentType AwsAccountId - dashboardsEnabled warehousesEnabled userRoleInEnvironment stack { diff --git a/frontend/src/api/Environment/updateEnvironment.js b/frontend/src/api/Environment/updateEnvironment.js index 48a33dacb..e08871e05 100644 --- a/frontend/src/api/Environment/updateEnvironment.js +++ b/frontend/src/api/Environment/updateEnvironment.js @@ -16,7 +16,6 @@ const updateEnvironment = ({ environmentUri, input }) => ({ userRoleInEnvironment SamlGroupName AwsAccountId - dashboardsEnabled warehousesEnabled created parameters { diff --git a/frontend/src/views/Environments/EnvironmentCreateForm.js b/frontend/src/views/Environments/EnvironmentCreateForm.js index df5727742..90e74bbe7 100644 --- a/frontend/src/views/Environments/EnvironmentCreateForm.js +++ b/frontend/src/views/Environments/EnvironmentCreateForm.js @@ -153,7 +153,6 @@ const EnvironmentCreateForm = (props) => { tags: values.tags, description: values.description, region: values.region, - dashboardsEnabled: values.dashboardsEnabled, warehousesEnabled: values.warehousesEnabled, EnvironmentDefaultIAMRoleName: values.EnvironmentDefaultIAMRoleName, resourcePrefix: values.resourcePrefix, @@ -162,6 +161,10 @@ const EnvironmentCreateForm = (props) => { key: 'notebooksEnabled', value: String(values.notebooksEnabled) }, + { + key: 'dashboardsEnabled', + value: String(values.dashboardsEnabled) + }, { key: 'mlStudiosEnabled', value: String(values.mlStudiosEnabled) diff --git a/frontend/src/views/Environments/EnvironmentEditForm.js b/frontend/src/views/Environments/EnvironmentEditForm.js index 0349c4ca1..9a3789985 100644 --- a/frontend/src/views/Environments/EnvironmentEditForm.js +++ b/frontend/src/views/Environments/EnvironmentEditForm.js @@ -75,7 +75,6 @@ const EnvironmentEditForm = (props) => { label: values.label, tags: values.tags, description: values.description, - dashboardsEnabled: values.dashboardsEnabled, warehousesEnabled: values.warehousesEnabled, resourcePrefix: values.resourcePrefix, parameters: [ @@ -90,6 +89,10 @@ const EnvironmentEditForm = (props) => { { key: 'pipelinesEnabled', value: String(values.pipelinesEnabled) + }, + { + key: 'dashboardsEnabled', + value: String(values.dashboardsEnabled) } ] } @@ -210,6 +213,8 @@ const EnvironmentEditForm = (props) => { notebooksEnabled: env.parameters['notebooksEnabled'] === 'true', mlStudiosEnabled: env.parameters['mlStudiosEnabled'] === 'true', pipelinesEnabled: env.parameters['pipelinesEnabled'] === 'true', + dashboardsEnabled: + env.parameters['dashboardsEnabled'] === 'true', warehousesEnabled: env.warehousesEnabled, resourcePrefix: env.resourcePrefix }} diff --git a/frontend/src/views/Environments/EnvironmentFeatures.js b/frontend/src/views/Environments/EnvironmentFeatures.js index 1bfae31f9..d8d8ebf66 100644 --- a/frontend/src/views/Environments/EnvironmentFeatures.js +++ b/frontend/src/views/Environments/EnvironmentFeatures.js @@ -33,9 +33,15 @@ const EnvironmentFeatures = (props) => { diff --git a/tests/api/conftest.py b/tests/api/conftest.py index b7bf5aaf8..ba31b3a87 100644 --- a/tests/api/conftest.py +++ b/tests/api/conftest.py @@ -1,4 +1,3 @@ -import dataall.searchproxy.indexers from .client import * from dataall.db import models @@ -24,7 +23,6 @@ def patch_check_env(module_mocker): def patch_es(module_mocker): module_mocker.patch('dataall.searchproxy.connect', return_value={}) module_mocker.patch('dataall.searchproxy.search', return_value={}) - module_mocker.patch('dataall.searchproxy.indexers.DashboardIndexer.upsert', return_value={}) module_mocker.patch('dataall.searchproxy.base_indexer.BaseIndexer.delete_doc', return_value={}) @@ -170,8 +168,8 @@ def env(client): cache = {} def factory(org, envname, owner, group, account, region, desc='test', parameters=None): - if parameters == None: - parameters = {} + if not parameters: + parameters = {"dashboardsEnabled": "true"} key = f"{org.organizationUri}{envname}{owner}{''.join(group or '-')}{account}{region}" if cache.get(key): @@ -205,7 +203,6 @@ def factory(org, envname, owner, group, account, region, desc='test', parameters 'tags': ['a', 'b', 'c'], 'region': f'{region}', 'SamlGroupName': f'{group}', - 'dashboardsEnabled': True, 'vpcId': 'vpc-123456', 'parameters': [{'key': k, 'value': v} for k, v in parameters.items()] }, @@ -225,7 +222,6 @@ def factory( owner: str, samlGroupName: str, environmentDefaultIAMRoleName: str, - dashboardsEnabled: bool = False, ) -> models.Environment: with db.scoped_session() as session: env = models.Environment( @@ -240,7 +236,6 @@ def factory( EnvironmentDefaultIAMRoleName=environmentDefaultIAMRoleName, EnvironmentDefaultIAMRoleArn=f"arn:aws:iam::{awsAccountId}:role/{environmentDefaultIAMRoleName}", CDKRoleArn=f"arn:aws::{awsAccountId}:role/EnvRole", - dashboardsEnabled=dashboardsEnabled, ) session.add(env) session.commit() diff --git a/tests/api/test_environment.py b/tests/api/test_environment.py index 55d2c4e90..c781c51e3 100644 --- a/tests/api/test_environment.py +++ b/tests/api/test_environment.py @@ -29,7 +29,6 @@ def get_env(client, env1, group): region SamlGroupName owner - dashboardsEnabled warehousesEnabled stack{ EcsTaskArn @@ -54,10 +53,13 @@ def test_get_environment(client, org1, env1, group): response.data.getEnvironment.organization.organizationUri == org1.organizationUri ) - assert response.data.getEnvironment.owner == 'alice' - assert response.data.getEnvironment.AwsAccountId == env1.AwsAccountId - assert response.data.getEnvironment.dashboardsEnabled - assert response.data.getEnvironment.warehousesEnabled + body = response.data.getEnvironment + assert body.owner == 'alice' + assert body.AwsAccountId == env1.AwsAccountId + assert body.warehousesEnabled + + params = {p.key: p.value for p in body.parameters} + assert params["dashboardsEnabled"] == "true" def test_get_environment_object_not_found(client, org1, env1, group): @@ -85,7 +87,7 @@ def test_get_environment_object_not_found(client, org1, env1, group): def test_update_env(client, org1, env1, group): - query = """ + query = """ mutation UpdateEnv($environmentUri:String!,$input:ModifyEnvironmentInput){ updateEnvironment(environmentUri:$environmentUri,input:$input){ organization{ @@ -98,7 +100,6 @@ def test_update_env(client, org1, env1, group): owner tags resourcePrefix - dashboardsEnabled warehousesEnabled parameters { key @@ -114,7 +115,6 @@ def test_update_env(client, org1, env1, group): input={ 'label': 'DEV', 'tags': ['test', 'env'], - 'dashboardsEnabled': False, 'warehousesEnabled': False, 'parameters': [ { @@ -134,7 +134,6 @@ def test_update_env(client, org1, env1, group): input={ 'label': 'DEV', 'tags': ['test', 'env'], - 'dashboardsEnabled': False, 'warehousesEnabled': False, 'parameters': [ { @@ -674,7 +673,6 @@ def test_create_environment(db, client, org1, env1, user, group): owner EnvironmentDefaultIAMRoleName EnvironmentDefaultIAMRoleImported - dashboardsEnabled resourcePrefix networks{ VpcId @@ -699,18 +697,17 @@ def test_create_environment(db, client, org1, env1, user, group): 'vpcId': 'vpc-1234567', 'privateSubnetIds': 'subnet-1', 'publicSubnetIds': 'subnet-21', - 'dashboardsEnabled': True, 'resourcePrefix': 'customer-prefix', }, ) - assert response.data.createEnvironment.dashboardsEnabled - assert response.data.createEnvironment.networks - assert ( - response.data.createEnvironment.EnvironmentDefaultIAMRoleName == 'myOwnIamRole' - ) - assert response.data.createEnvironment.EnvironmentDefaultIAMRoleImported - assert response.data.createEnvironment.resourcePrefix == 'customer-prefix' - for vpc in response.data.createEnvironment.networks: + + body = response.data.createEnvironment + + assert body.networks + assert body.EnvironmentDefaultIAMRoleName == 'myOwnIamRole' + assert body.EnvironmentDefaultIAMRoleImported + assert body.resourcePrefix == 'customer-prefix' + for vpc in body.networks: assert vpc.privateSubnetIds assert vpc.publicSubnetIds assert vpc.default diff --git a/tests/api/test_organization.py b/tests/api/test_organization.py index 9b74e52a2..8930f014b 100644 --- a/tests/api/test_organization.py +++ b/tests/api/test_organization.py @@ -1,6 +1,8 @@ import dataall import pytest +from dataall.core.environment.models import EnvironmentParameter + @pytest.fixture(scope='module', autouse=True) def org1(org, user, group, tenant): @@ -280,6 +282,7 @@ def test_group_invitation(db, client, org1, group2, user, group3, group, env): assert 'OrganizationResourcesFound' in response.errors[0].message with db.scoped_session() as session: + session.query(EnvironmentParameter).filter(EnvironmentParameter.environmentUri == env2.environmentUri).delete() env = session.query(dataall.db.models.Environment).get(env2.environmentUri) session.delete(env) session.commit() diff --git a/tests/api/test_vote.py b/tests/api/test_vote.py index 513f0b903..69183f150 100644 --- a/tests/api/test_vote.py +++ b/tests/api/test_vote.py @@ -1,65 +1,3 @@ -import pytest - -from dataall.db import models - - -@pytest.fixture(scope='module') -def org1(db, org, tenant, user, group) -> models.Organization: - org = org('testorg', user.userName, group.name) - yield org - - -@pytest.fixture(scope='module') -def env1( - db, org1: models.Organization, user, group, env -) -> models.Environment: - env1 = env(org1, 'dev', user.userName, group.name, '111111111111', 'eu-west-1') - yield env1 - - -@pytest.fixture(scope='module') -def dashboard(client, env1, org1, group, module_mocker, patch_es): - module_mocker.patch( - 'dataall.aws.handlers.quicksight.Quicksight.can_import_dashboard', - return_value=True, - ) - response = client.query( - """ - mutation importDashboard( - $input:ImportDashboardInput, - ){ - importDashboard(input:$input){ - dashboardUri - name - label - DashboardId - created - owner - SamlGroupName - } - } - """, - input={ - 'dashboardId': f'1234', - 'label': f'1234', - 'environmentUri': env1.environmentUri, - 'SamlGroupName': group.name, - 'terms': ['term'], - }, - username='alice', - groups=[group.name], - ) - assert response.data.importDashboard.owner == 'alice' - assert response.data.importDashboard.SamlGroupName == group.name - yield response.data.importDashboard - - -def test_count_votes(client, dashboard, env1): - response = count_votes_query( - client, dashboard.dashboardUri, 'dashboard', env1.SamlGroupName - ) - assert response.data.countUpVotes == 0 - def count_votes_query(client, target_uri, target_type, group): response = client.query( @@ -93,38 +31,6 @@ def get_vote_query(client, target_uri, target_type, group): return response -def test_upvote(patch_es, client, env1, dashboard): - - response = upvote_mutation( - client, dashboard.dashboardUri, 'dashboard', True, env1.SamlGroupName - ) - assert response.data.upVote.upvote - response = count_votes_query( - client, dashboard.dashboardUri, 'dashboard', env1.SamlGroupName - ) - assert response.data.countUpVotes == 1 - response = get_vote_query( - client, dashboard.dashboardUri, 'dashboard', env1.SamlGroupName - ) - assert response.data.getVote.upvote - - response = upvote_mutation( - client, dashboard.dashboardUri, 'dashboard', False, env1.SamlGroupName - ) - - assert not response.data.upVote.upvote - - response = get_vote_query( - client, dashboard.dashboardUri, 'dashboard', env1.SamlGroupName - ) - assert not response.data.getVote.upvote - - response = count_votes_query( - client, dashboard.dashboardUri, 'dashboard', env1.SamlGroupName - ) - assert response.data.countUpVotes == 0 - - def upvote_mutation(client, target_uri, target_type, upvote, group): response = client.query( """ diff --git a/tests/modules/dashboards/__init__.py b/tests/modules/dashboards/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/modules/dashboards/conftest.py b/tests/modules/dashboards/conftest.py new file mode 100644 index 000000000..6d9b0de97 --- /dev/null +++ b/tests/modules/dashboards/conftest.py @@ -0,0 +1,63 @@ +from unittest.mock import MagicMock + +import pytest + +from tests.api.conftest import * + +@pytest.fixture(scope='module', autouse=True) +def org1(org, user, group, tenant): + org1 = org('testorg', user.userName, group.name) + yield org1 + + +@pytest.fixture(scope='module', autouse=True) +def env1(env, org1, user, group, tenant, module_mocker): + module_mocker.patch('requests.post', return_value=True) + module_mocker.patch( + 'dataall.api.Objects.Environment.resolvers.check_environment', return_value=True + ) + module_mocker.patch( + 'dataall.api.Objects.Environment.resolvers.get_pivot_role_as_part_of_environment', return_value=False + ) + env1 = env(org1, 'dev', user.userName, group.name, '111111111111', 'eu-west-1') + yield env1 + + +@pytest.fixture(scope='module') +def dashboard(client, env1, org1, group, module_mocker, patch_es): + mock_client = MagicMock() + module_mocker.patch( + 'dataall.modules.dashboards.services.dashboard_service.DashboardQuicksightClient', + mock_client + ) + response = client.query( + """ + mutation importDashboard( + $input:ImportDashboardInput, + ){ + importDashboard(input:$input){ + dashboardUri + name + label + DashboardId + created + owner + SamlGroupName + upvotes + userRoleForDashboard + } + } + """, + input={ + 'dashboardId': f'1234', + 'label': f'1234', + 'environmentUri': env1.environmentUri, + 'SamlGroupName': group.name, + 'terms': ['term'], + }, + username='alice', + groups=[group.name], + ) + assert response.data.importDashboard.owner == 'alice' + assert response.data.importDashboard.SamlGroupName == group.name + yield response.data.importDashboard \ No newline at end of file diff --git a/tests/modules/dashboards/test_dashboard_votes.py b/tests/modules/dashboards/test_dashboard_votes.py new file mode 100644 index 000000000..e8842fce9 --- /dev/null +++ b/tests/modules/dashboards/test_dashboard_votes.py @@ -0,0 +1,40 @@ +from tests.api.test_vote import upvote_mutation, count_votes_query, get_vote_query + + +def test_dashboard_count_votes(client, dashboard, env1): + response = count_votes_query( + client, dashboard.dashboardUri, 'dashboard', env1.SamlGroupName + ) + assert response.data.countUpVotes == 0 + + +def test_dashboard_upvote(patch_es, client, env1, dashboard): + + response = upvote_mutation( + client, dashboard.dashboardUri, 'dashboard', True, env1.SamlGroupName + ) + assert response.data.upVote.upvote + response = count_votes_query( + client, dashboard.dashboardUri, 'dashboard', env1.SamlGroupName + ) + assert response.data.countUpVotes == 1 + response = get_vote_query( + client, dashboard.dashboardUri, 'dashboard', env1.SamlGroupName + ) + assert response.data.getVote.upvote + + response = upvote_mutation( + client, dashboard.dashboardUri, 'dashboard', False, env1.SamlGroupName + ) + + assert not response.data.upVote.upvote + + response = get_vote_query( + client, dashboard.dashboardUri, 'dashboard', env1.SamlGroupName + ) + assert not response.data.getVote.upvote + + response = count_votes_query( + client, dashboard.dashboardUri, 'dashboard', env1.SamlGroupName + ) + assert response.data.countUpVotes == 0 \ No newline at end of file diff --git a/tests/api/test_dashboards.py b/tests/modules/dashboards/test_dashboards.py similarity index 82% rename from tests/api/test_dashboards.py rename to tests/modules/dashboards/test_dashboards.py index cd0da17bd..aa4629fcb 100644 --- a/tests/api/test_dashboards.py +++ b/tests/modules/dashboards/test_dashboards.py @@ -4,71 +4,9 @@ import dataall -@pytest.fixture(scope='module', autouse=True) -def org1(org, user, group, tenant): - org1 = org('testorg', user.userName, group.name) - yield org1 - - -@pytest.fixture(scope='module', autouse=True) -def env1(env, org1, user, group, tenant, module_mocker): - module_mocker.patch('requests.post', return_value=True) - module_mocker.patch( - 'dataall.api.Objects.Environment.resolvers.check_environment', return_value=True - ) - module_mocker.patch( - 'dataall.api.Objects.Environment.resolvers.get_pivot_role_as_part_of_environment', return_value=False - ) - env1 = env(org1, 'dev', user.userName, group.name, '111111111111', 'eu-west-1') - yield env1 - - -@pytest.fixture(scope='module') -def dashboard(client, env1, org1, group, module_mocker, patch_es): - module_mocker.patch( - 'dataall.aws.handlers.quicksight.Quicksight.can_import_dashboard', - return_value=True, - ) - response = client.query( - """ - mutation importDashboard( - $input:ImportDashboardInput, - ){ - importDashboard(input:$input){ - dashboardUri - name - label - DashboardId - created - owner - SamlGroupName - upvotes - userRoleForDashboard - } - } - """, - input={ - 'dashboardId': f'1234', - 'label': f'1234', - 'environmentUri': env1.environmentUri, - 'SamlGroupName': group.name, - 'terms': ['term'], - }, - username='alice', - groups=[group.name], - ) - assert response.data.importDashboard.owner == 'alice' - assert response.data.importDashboard.SamlGroupName == group.name - yield response.data.importDashboard - - def test_update_dashboard( client, env1, org1, group, module_mocker, patch_es, dashboard ): - module_mocker.patch( - 'dataall.aws.handlers.quicksight.Quicksight.can_import_dashboard', - return_value=True, - ) response = client.query( """ mutation updateDashboard( diff --git a/tests/modules/datasets/tasks/conftest.py b/tests/modules/datasets/tasks/conftest.py index e3556fed6..8b4e241ac 100644 --- a/tests/modules/datasets/tasks/conftest.py +++ b/tests/modules/datasets/tasks/conftest.py @@ -50,7 +50,6 @@ def factory( owner: str, samlGroupName: str, environmentDefaultIAMRoleName: str, - dashboardsEnabled: bool = False, ) -> models.Environment: with db.scoped_session() as session: env = models.Environment( @@ -65,7 +64,6 @@ def factory( EnvironmentDefaultIAMRoleName=environmentDefaultIAMRoleName, EnvironmentDefaultIAMRoleArn=f"arn:aws:iam::{awsAccountId}:role/{environmentDefaultIAMRoleName}", CDKRoleArn=f"arn:aws::{awsAccountId}:role/EnvRole", - dashboardsEnabled=dashboardsEnabled, ) session.add(env) session.commit() diff --git a/tests/modules/datasets/test_dataset_count_votes.py b/tests/modules/datasets/test_dataset_count_votes.py index 2212d8ad9..60af2e524 100644 --- a/tests/modules/datasets/test_dataset_count_votes.py +++ b/tests/modules/datasets/test_dataset_count_votes.py @@ -1,48 +1,40 @@ import pytest -from dataall.modules.datasets import Dataset from tests.api.test_vote import * -@pytest.fixture(scope='module', autouse=True) -def dataset1(db, env1, org1, group, user, dataset) -> Dataset: - yield dataset( - org=org1, env=env1, name='dataset1', owner=user.userName, group=group.name - ) - - -def test_count_votes(client, dataset1, dashboard): +def test_count_votes(client, dataset_fixture): response = count_votes_query( - client, dataset1.datasetUri, 'dataset', dataset1.SamlAdminGroupName + client, dataset_fixture.datasetUri, 'dataset', dataset_fixture.SamlAdminGroupName ) assert response.data.countUpVotes == 0 -def test_upvote(patch_es, client, dataset1): +def test_upvote(patch_es, client, dataset_fixture): response = upvote_mutation( - client, dataset1.datasetUri, 'dataset', True, dataset1.SamlAdminGroupName + client, dataset_fixture.datasetUri, 'dataset', True, dataset_fixture.SamlAdminGroupName ) assert response.data.upVote.upvote response = count_votes_query( - client, dataset1.datasetUri, 'dataset', dataset1.SamlAdminGroupName + client, dataset_fixture.datasetUri, 'dataset', dataset_fixture.SamlAdminGroupName ) assert response.data.countUpVotes == 1 response = get_vote_query( - client, dataset1.datasetUri, 'dataset', dataset1.SamlAdminGroupName + client, dataset_fixture.datasetUri, 'dataset', dataset_fixture.SamlAdminGroupName ) assert response.data.getVote.upvote response = upvote_mutation( - client, dataset1.datasetUri, 'dataset', False, dataset1.SamlAdminGroupName + client, dataset_fixture.datasetUri, 'dataset', False, dataset_fixture.SamlAdminGroupName ) assert not response.data.upVote.upvote response = get_vote_query( - client, dataset1.datasetUri, 'dataset', dataset1.SamlAdminGroupName + client, dataset_fixture.datasetUri, 'dataset', dataset_fixture.SamlAdminGroupName ) assert not response.data.getVote.upvote response = count_votes_query( - client, dataset1.datasetUri, 'dataset', dataset1.SamlAdminGroupName + client, dataset_fixture.datasetUri, 'dataset', dataset_fixture.SamlAdminGroupName ) - assert response.data.countUpVotes == 0 \ No newline at end of file + assert response.data.countUpVotes == 0 diff --git a/tests/modules/datasets/test_dataset_resource_found.py b/tests/modules/datasets/test_dataset_resource_found.py index e8f056a61..3536c1702 100644 --- a/tests/modules/datasets/test_dataset_resource_found.py +++ b/tests/modules/datasets/test_dataset_resource_found.py @@ -30,9 +30,6 @@ def get_env(client, env1, group): region SamlGroupName owner - dashboardsEnabled - mlStudiosEnabled - pipelinesEnabled warehousesEnabled stack{ EcsTaskArn diff --git a/tests/modules/datasets/test_share.py b/tests/modules/datasets/test_share.py index e1f6983c0..80d1fcf2b 100644 --- a/tests/modules/datasets/test_share.py +++ b/tests/modules/datasets/test_share.py @@ -40,7 +40,6 @@ def env1(environment: typing.Callable, org1: dataall.db.models.Organization, use owner=user.userName, samlGroupName=group.name, environmentDefaultIAMRoleName=f"source-{group.name}", - dashboardsEnabled=False, ) @@ -96,7 +95,6 @@ def env2( owner=user2.userName, samlGroupName=group2.name, environmentDefaultIAMRoleName=f"source-{group2.name}", - dashboardsEnabled=False, )