From 276a184006ec873fc5613cb5fd34b3f1c57b90c3 Mon Sep 17 00:00:00 2001 From: harshsoni2024 Date: Tue, 19 Mar 2024 20:07:11 +0530 Subject: [PATCH] Fix #14089: Add QlikCloud connector support --- .../examples/workflows/qlikcloud.yaml | 29 ++ .../source/dashboard/qlikcloud/__init__.py | 0 .../source/dashboard/qlikcloud/client.py | 171 +++++++++ .../source/dashboard/qlikcloud/connection.py | 56 +++ .../source/dashboard/qlikcloud/constants.py | 56 +++ .../source/dashboard/qlikcloud/metadata.py | 340 ++++++++++++++++++ .../source/dashboard/qlikcloud/models.py | 110 ++++++ .../testConnections/dashboard/qlikcloud.json | 16 + .../entity/data/dashboardDataModel.json | 6 +- .../dashboard/qlikCloudConnection.json | 42 +++ .../entity/services/dashboardService.json | 9 +- 11 files changed, 833 insertions(+), 2 deletions(-) create mode 100644 ingestion/src/metadata/examples/workflows/qlikcloud.yaml create mode 100644 ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/__init__.py create mode 100644 ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py create mode 100644 ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py create mode 100644 ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/constants.py create mode 100644 ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/metadata.py create mode 100644 ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/models.py create mode 100644 openmetadata-service/src/main/resources/json/data/testConnections/dashboard/qlikcloud.json create mode 100644 openmetadata-spec/src/main/resources/json/schema/entity/services/connections/dashboard/qlikCloudConnection.json diff --git a/ingestion/src/metadata/examples/workflows/qlikcloud.yaml b/ingestion/src/metadata/examples/workflows/qlikcloud.yaml new file mode 100644 index 000000000000..8bf4d186f42a --- /dev/null +++ b/ingestion/src/metadata/examples/workflows/qlikcloud.yaml @@ -0,0 +1,29 @@ +source: + type: qlikcloud + serviceName: qlikcloud_connector + serviceConnection: + config: + type: QlikCloud + token: + hostPort: + + sourceConfig: + config: + type: DashboardMetadata + # dashboardFilterPattern: {} + # chartFilterPattern: {} + # projectFilterPattern: {} + lineageInformation: + dbServiceNames: + - postgres + + +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/__init__.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py new file mode 100644 index 000000000000..3d5ab45197e8 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py @@ -0,0 +1,171 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +REST Auth & Client for QlikCloud +""" +import json +import traceback +from typing import Dict, List, Optional + +from metadata.generated.schema.entity.services.connections.dashboard.qlikCloudConnection import ( + QlikCloudConnection, +) +from metadata.ingestion.ometa.client import REST, ClientConfig +from metadata.ingestion.source.dashboard.qlikcloud.models import ( + QlikAppDetails, + QlikAppList, + QlikDataModelResult, + QlikSheet, + QlikSheetResult, + QlikTable, +) +from metadata.ingestion.source.dashboard.qlikcloud.constants import ( + OPEN_DOC_REQ, + CREATE_SHEET_SESSION, + GET_SHEET_LAYOUT, + APP_LOADMODEL_REQ, + GET_LOADMODEL_LAYOUT +) +from metadata.utils.constants import AUTHORIZATION_HEADER +from metadata.utils.helpers import clean_uri +from metadata.utils.logger import ingestion_logger + +logger = ingestion_logger() + +API_VERSION = "api" + + +class QlikCloudClient: + """ + Client Handling API communication with QlikCloud + """ + + def __init__( + self, + config: QlikCloudConnection, + ): + self.config = config + self.socket_connection = None + + # self.config.token = f"{self.config.token}" + client_config: ClientConfig = ClientConfig( + base_url=self.config.hostPort, + api_version=API_VERSION, + auth_header=AUTHORIZATION_HEADER, + auth_token=lambda: (self.config.token, 0), + ) + self.client = REST(client_config) + + def connect_websocket(self, dashboard_id: str = None) -> None: + """ + Method to initialise websocket connection + """ + # pylint: disable=import-outside-toplevel + import ssl + + from websocket import create_connection + + if self.socket_connection: + self.socket_connection.close() + self.socket_connection = create_connection( + f"wss{clean_uri(self.config.hostPort)[5:]}/app/{dashboard_id or ''}", + sslopt={"cert_reqs": ssl.CERT_NONE}, + header={"Authorization": f"Bearer {self.config.token}"}, + ) + self.socket_connection.recv() + + def _websocket_send_request( + self, request: dict, response: bool = False + ) -> Optional[Dict]: + """ + Method to send request to websocket + + request: data required to be sent to websocket + response: is json response required? + """ + self.socket_connection.send(json.dumps(request)) + resp = self.socket_connection.recv() + if response: + return json.loads(resp) + return None + + def get_dashboard_charts(self, dashboard_id: str) -> List[QlikSheet]: + """ + Get dahsboard chart list + """ + try: + self.connect_websocket(dashboard_id) + # self._websocket_send_request(request=None) + OPEN_DOC_REQ.update({"params": [dashboard_id]}) + self._websocket_send_request(OPEN_DOC_REQ) + self._websocket_send_request(CREATE_SHEET_SESSION) + sheets = self._websocket_send_request(GET_SHEET_LAYOUT, response=True) + data = QlikSheetResult(**sheets) + return data.result.qLayout.qAppObjectList.qItems + except Exception: + logger.debug(traceback.format_exc()) + logger.warning("Failed to fetch the dashboard charts") + return [] + + def get_dashboards_list(self) -> List[QlikAppList]: + """ + Get List of all apps + """ + try: + resp_apps = self.client.get("/v1/items?resourceType=app") + if resp_apps: + app_list = QlikAppList(apps=resp_apps.get("data", [])) + return app_list.apps + except Exception: + logger.debug(traceback.format_exc()) + logger.warning("Failed to fetch the app list") + return [] + + def get_dashboard_details(self, dashboard_id: str): + """ + Get App Details + """ + if not dashboard_id: + return None # don't call api if dashboard_id is None + try: + resp_dashboard = self.client.get(f"/v1/apps/{dashboard_id}") + if resp_dashboard: + return QlikAppDetails(**resp_dashboard.get("attributes")) + except Exception: + logger.debug(traceback.format_exc()) + logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}") + return None + + def get_dashboard_models(self) -> List[QlikTable]: + """ + Get dahsboard chart list + """ + try: + self._websocket_send_request(APP_LOADMODEL_REQ) + models = self._websocket_send_request(GET_LOADMODEL_LAYOUT, response=True) + data_models = QlikDataModelResult(**models) + layout = data_models.result.qLayout + if isinstance(layout, list): + tables = [] + for layout in data_models.result.qLayout: + tables.extend(layout.value.tables) + return tables + return layout.tables + except Exception: + logger.debug(traceback.format_exc()) + logger.warning("Failed to fetch the dashboard datamodels") + return [] + + def get_collections_list(self): + """ + Get List of all collections + """ + return [] diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py new file mode 100644 index 000000000000..928645178b3a --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py @@ -0,0 +1,56 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Source connection handler +""" +from typing import Optional + +from metadata.generated.schema.entity.automations.workflow import ( + Workflow as AutomationWorkflow, +) +from metadata.generated.schema.entity.services.connections.dashboard.qlikCloudConnection import ( + QlikCloudConnection, +) +from metadata.ingestion.connections.test_connections import test_connection_steps +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.dashboard.qlikcloud.client import QlikCloudClient + + +def get_connection(connection: QlikCloudConnection) -> QlikCloudClient: + """ + Create connection + """ + return QlikCloudClient(connection) + + +def test_connection( + metadata: OpenMetadata, + client: QlikCloudClient, + service_connection: QlikCloudConnection, + automation_workflow: Optional[AutomationWorkflow] = None, +) -> None: + """ + Test connection. This can be executed either as part + of a metadata workflow or during an Automation Workflow + """ + + def custom_executor(): + return client.get_dashboards_list() + + test_fn = {"GetDashboards": custom_executor} + + test_connection_steps( + metadata=metadata, + test_fn=test_fn, + service_type=service_connection.type.value, + automation_workflow=automation_workflow, + ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/constants.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/constants.py new file mode 100644 index 000000000000..cbc847a75f1d --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/constants.py @@ -0,0 +1,56 @@ +""" +QlikCloud Constants +""" + +OPEN_DOC_REQ = { + "method": "OpenDoc", + "handle": -1, + "outKey": -1, + "id": 1, +} +CREATE_SHEET_SESSION = { + "method": "CreateSessionObject", + "handle": 1, + "params": [ + { + "qInfo": {"qType": "SheetList"}, + "qAppObjectListDef": { + "qType": "sheet", + "qData": { + "title": "/qMetaDef/title", + "description": "/qMetaDef/description", + "thumbnail": "/thumbnail", + "cells": "/cells", + "rank": "/rank", + "columns": "/columns", + "rows": "/rows", + }, + }, + } + ], + "outKey": -1, + "id": 2, +} +GET_SHEET_LAYOUT = { + "method": "GetLayout", + "handle": 2, + "params": [], + "outKey": -1, + "id": 3, +} +APP_LOADMODEL_REQ = { + "delta": True, + "handle": 1, + "method": "GetObject", + "params": ["LoadModel"], + "id": 4, + "jsonrpc": "2.0", +} +GET_LOADMODEL_LAYOUT = { + "delta": True, + "handle": 3, + "method": "GetLayout", + "params": [], + "id": 5, + "jsonrpc": "2.0", +} diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/metadata.py new file mode 100644 index 000000000000..dfd0e5ee0208 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/metadata.py @@ -0,0 +1,340 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""QlikCloud source module""" + +import traceback +from typing import Iterable, List, Optional + +from metadata.generated.schema.api.data.createChart import CreateChartRequest +from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest +from metadata.generated.schema.api.data.createDashboardDataModel import ( + CreateDashboardDataModelRequest, +) +from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest +from metadata.generated.schema.entity.data.chart import Chart, ChartType +from metadata.generated.schema.entity.data.dashboardDataModel import ( + DashboardDataModel, + DataModelType, +) +from metadata.generated.schema.entity.data.table import Column, DataType, Table +from metadata.generated.schema.entity.services.connections.dashboard.qlikCloudConnection import ( + QlikCloudConnection, +) +from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( + OpenMetadataConnection, +) +from metadata.generated.schema.entity.services.dashboardService import ( + DashboardServiceType, +) +from metadata.generated.schema.entity.services.databaseService import DatabaseService +from metadata.generated.schema.entity.services.ingestionPipelines.status import ( + StackTraceError, +) +from metadata.generated.schema.metadataIngestion.workflow import ( + Source as WorkflowSource, +) +from metadata.ingestion.api.models import Either +from metadata.ingestion.api.steps import InvalidSourceException +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource +from metadata.ingestion.source.dashboard.qlikcloud.models import ( + QlikApp, + QlikAppDetails, + QlikAppList, + QlikTable, +) +from metadata.utils import fqn +from metadata.utils.filters import filter_by_chart, filter_by_datamodel +from metadata.utils.helpers import clean_uri, replace_special_with +from metadata.utils.logger import ingestion_logger + +logger = ingestion_logger() + + +class QlikcloudSource(DashboardServiceSource): + """ + QlikCloud Source Class + """ + + config: WorkflowSource + metadata_config: OpenMetadataConnection + + @classmethod + def create( + cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None + ): + config = WorkflowSource.parse_obj(config_dict) + connection: QlikCloudConnection = config.serviceConnection.__root__.config + if not isinstance(connection, QlikCloudConnection): + raise InvalidSourceException( + f"Expected QlikCloudConnection, but got {connection}" + ) + return cls(config, metadata) + + def __init__( + self, + config: WorkflowSource, + metadata: OpenMetadata, + ): + super().__init__(config, metadata) + self.collections: List[QlikAppList] = [] + self.data_models: List[QlikTable] = [] + + + def prepare(self): + self.collections = self.client.get_collections_list() + return super().prepare() + + def get_dashboards_list(self) -> Optional[List[QlikApp]]: + """ + Get List of all apps + """ + return self.client.get_dashboards_list() + + def get_dashboard_name(self, dashboard: QlikApp) -> str: + """ + Get app Name + """ + return dashboard.name + + def get_dashboard_details(self, dashboard: QlikApp) -> dict: + """ + Get app Details + """ + return self.client.get_dashboard_details(dashboard.resourceId) + + def yield_dashboard( + self, dashboard_details: QlikAppDetails + ) -> Iterable[Either[CreateDashboardRequest]]: + """ + Method to Get Dashboard Entity + """ + try: + dashboard_url = ( + f"{clean_uri(self.service_connection.hostPort)}/sense/app/{dashboard_details.id}/overview" + f"{replace_special_with(raw=dashboard_details.name.lower(), replacement='-')}" + ) + + dashboard_request = CreateDashboardRequest( + name=dashboard_details.id, + sourceUrl=dashboard_url, + displayName=dashboard_details.name, + description=dashboard_details.description, + project=self.context.project_name, + charts=[ + fqn.build( + self.metadata, + entity_type=Chart, + service_name=self.context.dashboard_service, + chart_name=chart, + ) + for chart in self.context.charts or [] + ], + service=self.context.dashboard_service, + owner=self.get_owner_ref(dashboard_details=dashboard_details), + ) + yield Either(right=dashboard_request) + self.register_record(dashboard_request=dashboard_request) + except Exception as exc: # pylint: disable=broad-except + yield Either( + left=StackTraceError( + name=dashboard_details.name, + error=f"Error creating dashboard [{dashboard_details.name}]: {exc}", + stackTrace=traceback.format_exc(), + ) + ) + + def _get_datamodel(self, datamodel: QlikTable): + datamodel_fqn = fqn.build( + self.metadata, + entity_type=DashboardDataModel, + service_name=self.context.dashboard_service, + data_model_name=datamodel.id, + ) + if datamodel_fqn: + return self.metadata.get_by_name( + entity=DashboardDataModel, + fqn=datamodel_fqn, + ) + return None + + def _get_database_table( + self, db_service_entity: DatabaseService, datamodel: QlikTable + ) -> Optional[Table]: + """ + Get the table entity for lineage + """ + # table.name in tableau can come as db.schema.table_name. Hence the logic to split it + if datamodel.tableName and db_service_entity: + try: + if len(datamodel.connectorProperties.tableQualifiers) > 1: + ( + database_name, + schema_name, + ) = datamodel.connectorProperties.tableQualifiers[-2:] + elif len(datamodel.connectorProperties.tableQualifiers) == 1: + schema_name = datamodel.connectorProperties.tableQualifiers[-1] + database_name = None + else: + schema_name, database_name = None, None + + table_fqn = fqn.build( + self.metadata, + entity_type=Table, + service_name=db_service_entity.name.__root__, + schema_name=schema_name, + table_name=datamodel.tableName, + database_name=database_name, + ) + if table_fqn: + return self.metadata.get_by_name( + entity=Table, + fqn=table_fqn, + ) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Error occured while finding table fqn: {exc}") + return None + + def yield_dashboard_lineage_details( + self, + dashboard_details: QlikApp, + db_service_name: Optional[str], + ) -> Iterable[Either[AddLineageRequest]]: + """Get lineage method""" + db_service_entity = self.metadata.get_by_name( + entity=DatabaseService, fqn=db_service_name + ) + for datamodel in self.data_models or []: + try: + data_model_entity = self._get_datamodel(datamodel=datamodel) + if data_model_entity: + om_table = self._get_database_table( + db_service_entity, datamodel=datamodel + ) + if om_table: + yield self._get_add_lineage_request( + to_entity=data_model_entity, from_entity=om_table + ) + except Exception as err: + yield Either( + left=StackTraceError( + name=f"{dashboard_details.qDocName} Lineage", + error=( + "Error to yield dashboard lineage details for DB " + f"service name [{db_service_name}]: {err}" + ), + stackTrace=traceback.format_exc(), + ) + ) + + def _get_database_service(self, db_service_name: str): + return self.metadata.get_by_name(DatabaseService, db_service_name) + + def _yield_lineage_from_query(self) -> Iterable[Either[AddLineageRequest]]: + pass + + def _yield_lineage_from_api(self) -> Iterable[Either[AddLineageRequest]]: + pass + + def yield_dashboard_chart( + self, dashboard_details + ) -> Iterable[Either[CreateChartRequest]]: + """Get chart method""" + charts = self.client.get_dashboard_charts(dashboard_id=dashboard_details.id) + for chart in charts: + try: + if not chart.qInfo.qId: + continue + # https://oq5biajesqmyhuh.sg.qlikcloud.com/sense/app/8b67aac5-15ba-45fc-98c3-7f65dcdfb956/sheet/SXqqZ/state/analysis/hubUrl/%2Fcatalog + if self.service_connection.hostPort: + chart_url = ( + f"{clean_uri(self.service_connection.hostPort)}/sense/app/{dashboard_details.id}" + f"/sheet/{chart.qInfo.qId}" + ) + else: + chart_url = None + if chart.qMeta.title and filter_by_chart( + self.source_config.chartFilterPattern, chart.qMeta.title + ): + self.status.filter(chart.qMeta.title, "Chart Pattern not allowed") + continue + yield Either( + right=CreateChartRequest( + name=chart.qInfo.qId, + displayName=chart.qMeta.title, + description=chart.qMeta.description, + chartType=ChartType.Other, + sourceUrl=chart_url, + service=self.context.dashboard_service, + ) + ) + except Exception as exc: # pylint: disable=broad-except + yield Either( + left=StackTraceError( + name=dashboard_details.qDocName, + error=f"Error creating chart [{chart}]: {exc}", + stackTrace=traceback.format_exc(), + ) + ) + + def yield_datamodel(self, _: QlikApp) -> Iterable[Either[DashboardDataModel]]: + if self.source_config.includeDataModels: + self.data_models = self.client.get_dashboard_models() + for data_model in self.data_models or []: + try: + data_model_name = ( + data_model.tableName if data_model.tableName else data_model.id + ) + if filter_by_datamodel( + self.source_config.dataModelFilterPattern, data_model_name + ): + self.status.filter(data_model_name, "Data model filtered out.") + continue + + data_model_request = CreateDashboardDataModelRequest( + name=data_model.id, + displayName=data_model_name, + service=self.context.dashboard_service, + dataModelType=DataModelType.QlikCloudDataModel.value, + serviceType=DashboardServiceType.QlikCloud.value, + columns=self.get_column_info(data_model), + ) + yield Either(right=data_model_request) + self.register_record_datamodel(datamodel_request=data_model_request) + except Exception as exc: + name = ( + data_model.tableName if data_model.tableName else data_model.id + ) + yield Either( + left=StackTraceError( + name=name, + error=f"Error yielding Data Model [{name}]: {exc}", + stackTrace=traceback.format_exc(), + ) + ) + + def get_column_info(self, data_source: QlikApp) -> Optional[List[Column]]: + """Build data model columns""" + datasource_columns = [] + for field in data_source.fields or []: + try: + parsed_fields = { + "dataTypeDisplay": "Qlik Field", + "dataType": DataType.UNKNOWN, + "name": field.id, + "displayName": field.name if field.name else field.id, + } + datasource_columns.append(Column(**parsed_fields)) + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.warning(f"Error to yield datamodel column: {exc}") + return datasource_columns diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/models.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/models.py new file mode 100644 index 000000000000..95ac87ce22f6 --- /dev/null +++ b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/models.py @@ -0,0 +1,110 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Metabase Models +""" +from typing import List, Optional, Union + +from pydantic import BaseModel + + +# App +class QlikApp(BaseModel): + """ + Metabase dashboard model + """ + + description: Optional[str] + name: str + id: str + resourceId: str + + +class QlikAppList(BaseModel): + apps: Optional[List[QlikApp]] + + +class QlikAppDetails(BaseModel): + """ + Metabase app details model + """ + + description: Optional[str] + + name: Optional[str] + id: str + collection_id: Optional[str] + + +# Sheet +class QlikSheetInfo(BaseModel): + qId: str + + +class QlikSheetMeta(BaseModel): + title: Optional[str] + description: Optional[str] + + +class QlikSheet(BaseModel): + qInfo: QlikSheetInfo + qMeta: Optional[QlikSheetMeta] = QlikSheetMeta() + + +class QlikSheetItems(BaseModel): + qItems: Optional[List[QlikSheet]] = [] + + +class QlikSheetAppObject(BaseModel): + qAppObjectList: Optional[QlikSheetItems] = QlikSheetItems() + + +class QlikSheetLayout(BaseModel): + qLayout: Optional[QlikSheetAppObject] = QlikSheetAppObject() + + +class QlikSheetResult(BaseModel): + result: Optional[QlikSheetLayout] = QlikSheetLayout() + + +# datamodel models +class QlikFields(BaseModel): + name: Optional[str] + id: Optional[str] + + +class QlikTableConnectionProp(BaseModel): + tableQualifiers: Optional[List[str]] = [] + + +class QlikTable(BaseModel): + tableName: Optional[str] + id: Optional[str] + connectorProperties: Optional[QlikTableConnectionProp] = QlikTableConnectionProp() + fields: Optional[List[QlikFields]] = [] + + +class QlikTablesList(BaseModel): + tables: Optional[List[QlikTable]] = [] + + +class QlikDataModelValue(BaseModel): + value: Optional[QlikTablesList] = QlikTablesList() + + +class QlikDataModelLayout(BaseModel): + qLayout: Optional[ + Union[QlikTablesList, List[QlikDataModelValue]] + ] = QlikTablesList() + + +class QlikDataModelResult(BaseModel): + result: Optional[QlikDataModelLayout] = QlikDataModelLayout() diff --git a/openmetadata-service/src/main/resources/json/data/testConnections/dashboard/qlikcloud.json b/openmetadata-service/src/main/resources/json/data/testConnections/dashboard/qlikcloud.json new file mode 100644 index 000000000000..e39a80bb3df1 --- /dev/null +++ b/openmetadata-service/src/main/resources/json/data/testConnections/dashboard/qlikcloud.json @@ -0,0 +1,16 @@ +{ + "name": "QlikCloud", + "displayName": "Qlik Cloud Test Connection", + "description": "This Test Connection validates the access against the server and basic metadata extraction of dashboards and charts.", + "steps": [ + { + "name": "GetDashboards", + "description": "List all the dashboards available to the user", + "errorMessage": "Failed to fetch dashboards, please validate the credentials or validate if user has access to fetch dashboards", + "shortCircuit": true, + "mandatory": true + } + ] + } + + \ No newline at end of file diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json index b600656c1198..79dc3ca43f25 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json @@ -20,7 +20,8 @@ "LookMlView", "LookMlExplore", "PowerBIDataModel", - "QlikSenseDataModel" + "QlikSenseDataModel", + "QlikCloudDataModel" ], "javaEnums": [ { @@ -43,6 +44,9 @@ }, { "name": "QlikSenseDataModel" + }, + { + "name": "QlikCloudDataModel" } ] } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/dashboard/qlikCloudConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/dashboard/qlikCloudConnection.json new file mode 100644 index 000000000000..276142a3bbeb --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/dashboard/qlikCloudConnection.json @@ -0,0 +1,42 @@ +{ + "$id": "https://open-metadata.org/schema/entity/services/connections/dashboard/qlikCloudConnection.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "QlikCloudConnection", + "description": "Qlik Cloud Connection Config", + "type": "object", + "javaType": "org.openmetadata.schema.services.connections.dashboard.QlikCloudConnection", + "definitions": { + "qlikCloudType": { + "description": "Qlik Cloud service type", + "type": "string", + "enum": ["QlikCloud"], + "default": "QlikCloud" + } + }, + "properties": { + "type": { + "title": "Service Type", + "description": "Service Type", + "$ref": "#/definitions/qlikCloudType", + "default": "QlikCloud" + }, + "token": { + "title": "token", + "description": "token to connect to Qlik Cloud.", + "type": "string" + }, + "hostPort": { + "expose": true, + "title": "Host and Port", + "description": "Host and Port of the Qlik Cloud instance.", + "type": "string", + "format": "uri" + }, + "supportsMetadataExtraction": { + "title": "Supports Metadata Extraction", + "$ref": "../connectionBasicType.json#/definitions/supportsMetadataExtraction" + } + }, + "additionalProperties": false, + "required": ["hostPort", "token"] + } diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json index 09d06fc9bf1f..a76bcbfeaf6b 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/dashboardService.json @@ -27,7 +27,8 @@ "QuickSight", "QlikSense", "Lightdash", - "Mstr" + "Mstr", + "QlikCloud" ], "javaEnums": [ { @@ -68,6 +69,9 @@ }, { "name": "Mstr" + }, + { + "name": "QlikCloud" } ] }, @@ -120,6 +124,9 @@ }, { "$ref": "./connections/dashboard/mstrConnection.json" + }, + { + "$ref": "./connections/dashboard/qlikCloudConnection.json" } ] }