quicksight dashboard connected to rds metadata database for platform monitoring

backend/dataall/api/Objects/Tenant/mutations.py

@@ -1,6 +1,6 @@
 from ... import gql
 from .input_types import UpdateGroupTenantPermissionsInput
-from .resolvers import update_group_permissions
+from .resolvers import *
 
 updateGroupPermission = gql.MutationField(
     name='updateGroupTenantPermissions',
@@ -12,3 +12,22 @@
     type=gql.Boolean,
     resolver=update_group_permissions,
 )
+
+createQuicksightDataSourceSet = gql.MutationField(
+    name='createQuicksightDataSourceSet',
+    args=[
+        gql.Argument(name='vpcConnectionId', type=gql.NonNullableType(gql.String))
+    ],
+    type=gql.String,
+    resolver=create_quicksight_data_source_set,
+)
+
+updateSSMParameter = gql.MutationField(
+    name='updateSSMParameter',
+    args=[
+        gql.Argument(name='name', type=gql.NonNullableType(gql.String)),
+        gql.Argument(name='value', type=gql.NonNullableType(gql.String))
+    ],
+    type=gql.String,
+    resolver=update_ssm_parameter,
+)

backend/dataall/api/Objects/Tenant/queries.py

@@ -16,3 +16,33 @@
     type=gql.Ref('GroupSearchResult'),
     resolver=list_tenant_groups,
 )
+
+getMonitoringDashboardId = gql.QueryField(
+    name='getMonitoringDashboardId',
+    type=gql.String,
+    resolver=get_monitoring_dashboard_id,
+)
+
+getMonitoringVpcConnectionId = gql.QueryField(
+    name='getMonitoringVPCConnectionId',
+    type=gql.String,
+    resolver=get_monitoring_vpc_connection_id,
+)
+
+getPlatformAuthorSession = gql.QueryField(
+    name='getPlatformAuthorSession',
+    args=[
+        gql.Argument(name='awsAccount', type=gql.NonNullableType(gql.String)),
+    ],
+    type=gql.String,
+    resolver=get_quicksight_author_session,
+)
+
+getPlatformReaderSession = gql.QueryField(
+    name='getPlatformReaderSession',
+    args=[
+        gql.Argument(name='dashboardId', type=gql.NonNullableType(gql.String)),
+    ],
+    type=gql.String,
+    resolver=get_quicksight_reader_session,
+)

backend/dataall/api/Objects/Tenant/resolvers.py

@@ -1,4 +1,10 @@
+import os
+
 from .... import db
+from ....aws.handlers.sts import SessionHelper
+from ....aws.handlers.parameter_store import ParameterStoreManager
+from ....aws.handlers.quicksight import Quicksight
+from ....db import exceptions
 
 
 def update_group_permissions(context, source, input=None):
@@ -32,3 +38,95 @@ def list_tenant_groups(context, source, filter=None):
             data=filter,
             check_perm=True,
         )
+
+
+def update_ssm_parameter(context, source, name: str = None, value: str = None):
+    current_account = SessionHelper.get_account()
+    region = os.getenv('AWS_REGION', 'eu-west-1')
+    print(value)
+    print(name)
+    response = ParameterStoreManager.update_parameter(AwsAccountId=current_account, region=region, parameter_name=f'/dataall/{os.getenv("envname", "local")}/quicksightmonitoring/{name}', parameter_value=value)
+    return response
+
+
+def get_monitoring_dashboard_id(context, source):
+    current_account = SessionHelper.get_account()
+    region = os.getenv('AWS_REGION', 'eu-west-1')
+    dashboard_id = ParameterStoreManager.get_parameter_value(AwsAccountId=current_account, region=region, parameter_path=f'/dataall/{os.getenv("envname", "local")}/quicksightmonitoring/DashboardId')
+    if not dashboard_id:
+        raise exceptions.AWSResourceNotFound(
+            action='GET_DASHBOARD_ID',
+            message='Dashboard Id could not be found on AWS Parameter Store',
+        )
+    return dashboard_id
+
+
+def get_monitoring_vpc_connection_id(context, source):
+    current_account = SessionHelper.get_account()
+    region = os.getenv('AWS_REGION', 'eu-west-1')
+    vpc_connection_id = ParameterStoreManager.get_parameter_value(AwsAccountId=current_account, region=region, parameter_path=f'/dataall/{os.getenv("envname", "local")}/quicksightmonitoring/VPCConnectionId')
+    if not vpc_connection_id:
+        raise exceptions.AWSResourceNotFound(
+            action='GET_VPC_CONNECTION_ID',
+            message='Dashboard Id could not be found on AWS Parameter Store',
+        )
+    return vpc_connection_id
+
+
+def create_quicksight_data_source_set(context, source, vpcConnectionId: str = None):
+    current_account = SessionHelper.get_account()
+    region = os.getenv('AWS_REGION', 'eu-west-1')
+    user = Quicksight.register_user(AwsAccountId=current_account, UserName=context.username, UserRole='AUTHOR')
+
+    datasourceId = Quicksight.create_data_source_vpc(AwsAccountId=current_account, region=region, UserName=context.username, vpcConnectionId=vpcConnectionId)
+    # Data sets are not created programmatically. Too much overhead for the value added. However, an example API is provided:
+    # datasets = Quicksight.create_data_set_from_source(AwsAccountId=current_account, region=region, UserName='dataallTenantUser', dataSourceId=datasourceId, tablesToImport=['organization', 'environment', 'dataset', 'datapipeline', 'dashboard', 'share_object'])
+
+    return datasourceId
+
+
+def get_quicksight_author_session(context, source, awsAccount: str = None):
+    with context.engine.scoped_session() as session:
+        admin = db.api.TenantPolicy.is_tenant_admin(context.groups)
+
+        if not admin:
+            raise db.exceptions.TenantUnauthorized(
+                username=context.username,
+                action=db.permissions.TENANT_ALL,
+                tenant_name=context.username,
+            )
+        region = os.getenv('AWS_REGION', 'eu-west-1')
+
+        url = Quicksight.get_author_session(
+            AwsAccountId=awsAccount,
+            region=region,
+            UserName=context.username,
+            UserRole='AUTHOR',
+        )
+
+    return url
+
+
+def get_quicksight_reader_session(context, source, dashboardId: str = None):
+    with context.engine.scoped_session() as session:
+        admin = db.api.TenantPolicy.is_tenant_admin(context.groups)
+
+        if not admin:
+            raise db.exceptions.TenantUnauthorized(
+                username=context.username,
+                action=db.permissions.TENANT_ALL,
+                tenant_name=context.username,
+            )
+
+        region = os.getenv('AWS_REGION', 'eu-west-1')
+        current_account = SessionHelper.get_account()
+
+        url = Quicksight.get_reader_session(
+            AwsAccountId=current_account,
+            region=region,
+            UserName=context.username,
+            UserRole='READER',
+            DashboardId=dashboardId
+        )
+
+    return url

backend/dataall/aws/handlers/parameter_store.py

@@ -23,11 +23,26 @@ def client(AwsAccountId, region):
     @staticmethod
     def get_parameter_value(AwsAccountId, region, parameter_path):
         if not parameter_path:
-            raise Exception('Secret name is None')
+            raise Exception('Parameter name is None')
         try:
             parameter_value = ParameterStoreManager.client(
                 AwsAccountId, region
             ).get_parameter(Name=parameter_path)['Parameter']['Value']
         except ClientError as e:
             raise Exception(e)
         return parameter_value
+
+    @staticmethod
+    def update_parameter(AwsAccountId, region, parameter_name, parameter_value):
+        if not parameter_name:
+            raise Exception('Parameter name is None')
+        if not parameter_value:
+            raise Exception('Parameter value is None')
+        try:
+            response = ParameterStoreManager.client(
+                AwsAccountId, region
+            ).put_parameter(Name=parameter_name, Value=parameter_value, Overwrite=True)['Version']
+        except ClientError as e:
+            raise Exception(e)
+        else:
+            return str(response)

backend/dataall/aws/handlers/quicksight.py

@@ -1,9 +1,13 @@
 import logging
 import re
+import os
+import ast
 
 from botocore.exceptions import ClientError
 
 from .sts import SessionHelper
+from .secrets_manager import SecretsManager
+from .parameter_store import ParameterStoreManager
 
 logger = logging.getLogger('QuicksightHandler')
 logger.setLevel(logging.DEBUG)
@@ -262,3 +266,154 @@ def can_import_dashboard(AwsAccountId, region, UserName, DashboardId):
                         return True
 
         return False
+
+    @staticmethod
+    def create_data_source_vpc(AwsAccountId, region, UserName, vpcConnectionId):
+        client = Quicksight.get_quicksight_client(AwsAccountId, region)
+        identity_region = 'us-east-1'
+
+        user = Quicksight.register_user(AwsAccountId, UserName, UserRole='AUTHOR')
+        try:
+            response = client.describe_data_source(
+                AwsAccountId=AwsAccountId, DataSourceId="dataall-metadata-db"
+            )
+
+        except client.exceptions.ResourceNotFoundException:
+            aurora_secret_arn = ParameterStoreManager.get_parameter_value(AwsAccountId=AwsAccountId, region=region, parameter_path=f'/dataall/{os.getenv("envname", "local")}/aurora/secret_arn')
+            aurora_params = SecretsManager.get_secret_value(
+                AwsAccountId=AwsAccountId, region=region, secretId=aurora_secret_arn
+            )
+            aurora_params_dict = ast.literal_eval(aurora_params)
+            response = client.create_data_source(
+                AwsAccountId=AwsAccountId,
+                DataSourceId="dataall-metadata-db",
+                Name="dataall-metadata-db",
+                Type="AURORA_POSTGRESQL",
+                DataSourceParameters={
+                    'AuroraPostgreSqlParameters': {
+                        'Host': aurora_params_dict["host"],
+                        'Port': aurora_params_dict["port"],
+                        'Database': aurora_params_dict["dbname"]
+                    }
+                },
+                Credentials={
+                    "CredentialPair": {
+                        "Username": aurora_params_dict["username"],
+                        "Password": aurora_params_dict["password"],
+                    }
+                },
+                Permissions=[
+                    {
+                        "Principal": f"arn:aws:quicksight:{region}:{AwsAccountId}:group/default/dataall",
+                        "Actions": [
+                            "quicksight:UpdateDataSourcePermissions",
+                            "quicksight:DescribeDataSource",
+                            "quicksight:DescribeDataSourcePermissions",
+                            "quicksight:PassDataSource",
+                            "quicksight:UpdateDataSource",
+                            "quicksight:DeleteDataSource"
+                        ]
+                    }
+                ],
+                VpcConnectionProperties={
+                    'VpcConnectionArn': f"arn:aws:quicksight:{region}:{AwsAccountId}:vpcConnection/{vpcConnectionId}"
+                }
+            )
+
+        return "dataall-metadata-db"
+
+    @staticmethod
+    def create_data_set_from_source(AwsAccountId, region, UserName, dataSourceId, tablesToImport):
+        client = Quicksight.get_quicksight_client(AwsAccountId, region)
+        user = Quicksight.describe_user(AwsAccountId, UserName)
+        if not user:
+            return False
+
+        data_source = client.describe_data_source(
+            AwsAccountId=AwsAccountId,
+            DataSourceId=dataSourceId
+        )
+
+        if not data_source:
+            return False
+
+        for table in tablesToImport:
+
+            response = client.create_data_set(
+                AwsAccountId=AwsAccountId,
+                DataSetId=f"dataall-imported-{table}",
+                Name=f"dataall-imported-{table}",
+                PhysicalTableMap={
+                    'string': {
+                        'RelationalTable': {
+                            'DataSourceArn': data_source.get('DataSource').get('Arn'),
+                            'Catalog': 'string',
+                            'Schema': 'dev',
+                            'Name': table,
+                            'InputColumns': [
+                                {
+                                    'Name': 'string',
+                                    'Type': 'STRING'
+                                },
+                            ]
+                        }
+                    }},
+                ImportMode='DIRECT_QUERY',
+                Permissions=[
+                    {
+                        'Principal': user.get('Arn'),
+                        'Actions': [
+                            "quicksight:DescribeDataSet",
+                            "quicksight:DescribeDataSetPermissions",
+                            "quicksight:PassDataSet",
+                            "quicksight:DescribeIngestion",
+                            "quicksight:ListIngestions"
+                        ]
+                    },
+                ],
+            )
+
+        return True
+
+    @staticmethod
+    def create_analysis(AwsAccountId, region, UserName):
+        client = Quicksight.get_quicksight_client(AwsAccountId, region)
+        user = Quicksight.describe_user(AwsAccountId, UserName)
+        if not user:
+            return False
+
+        response = client.create_analysis(
+            AwsAccountId=AwsAccountId,
+            AnalysisId='dataallMonitoringAnalysis',
+            Name='dataallMonitoringAnalysis',
+            Permissions=[
+                {
+                    'Principal': user.get('Arn'),
+                    'Actions': [
+                        'quicksight:DescribeAnalysis',
+                        'quicksight:DescribeAnalysisPermissions',
+                        'quicksight:UpdateAnalysisPermissions',
+                        'quicksight:UpdateAnalysis'
+                    ]
+                },
+            ],
+            SourceEntity={
+                'SourceTemplate': {
+                    'DataSetReferences': [
+                        {
+                            'DataSetPlaceholder': 'environment',
+                            'DataSetArn': f"arn:aws:quicksight:{region}:{AwsAccountId}:dataset/<DATASET-ID>"
+                        },
+                    ],
+                    'Arn': '<TEMPLATE-THAT-WE-WANT-TO-MIGRATE'
+                }
+            },
+            Tags=[
+                {
+                    'Key': 'application',
+                    'Value': 'dataall'
+                },
+            ]
+        )
+
+        return True