Fix open-metadata#14089: Add QlikCloud connector support

ingestion/src/metadata/examples/workflows/qlikcloud.yaml

@@ -0,0 +1,29 @@
+source:
+  type: qlikcloud
+  serviceName: qlikcloud_connector
+  serviceConnection:
+    config:
+      type: QlikCloud
+      token:
+      hostPort:
+
+  sourceConfig:
+    config:
+      type: DashboardMetadata
+      # dashboardFilterPattern: {}
+      # chartFilterPattern: {}
+      # projectFilterPattern: {}
+      lineageInformation:
+        dbServiceNames:
+        - postgres
+
+
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"      

ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py

@@ -0,0 +1,171 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+REST Auth & Client for QlikCloud
+"""
+import json
+import traceback
+from typing import Dict, List, Optional
+
+from metadata.generated.schema.entity.services.connections.dashboard.qlikCloudConnection import (
+    QlikCloudConnection,
+)
+from metadata.ingestion.ometa.client import REST, ClientConfig
+from metadata.ingestion.source.dashboard.qlikcloud.models import (
+    QlikAppDetails,
+    QlikAppList,
+    QlikDataModelResult,
+    QlikSheet,
+    QlikSheetResult,
+    QlikTable,
+)
+from metadata.ingestion.source.dashboard.qlikcloud.constants import (
+    OPEN_DOC_REQ,
+    CREATE_SHEET_SESSION,
+    GET_SHEET_LAYOUT,
+    APP_LOADMODEL_REQ,
+    GET_LOADMODEL_LAYOUT
+)
+from metadata.utils.constants import AUTHORIZATION_HEADER
+from metadata.utils.helpers import clean_uri
+from metadata.utils.logger import ingestion_logger
+
+logger = ingestion_logger()
+
+API_VERSION = "api"
+
+
+class QlikCloudClient:
+    """
+    Client Handling API communication with QlikCloud
+    """
+
+    def __init__(
+        self,
+        config: QlikCloudConnection,
+    ):
+        self.config = config
+        self.socket_connection = None
+
+        # self.config.token = f"{self.config.token}"
+        client_config: ClientConfig = ClientConfig(
+            base_url=self.config.hostPort,
+            api_version=API_VERSION,
+            auth_header=AUTHORIZATION_HEADER,
+            auth_token=lambda: (self.config.token, 0),
+        )
+        self.client = REST(client_config)
+
+    def connect_websocket(self, dashboard_id: str = None) -> None:
+        """
+        Method to initialise websocket connection
+        """
+        # pylint: disable=import-outside-toplevel
+        import ssl
+
+        from websocket import create_connection
+
+        if self.socket_connection:
+            self.socket_connection.close()
+        self.socket_connection = create_connection(
+            f"wss{clean_uri(self.config.hostPort)[5:]}/app/{dashboard_id or ''}",
+            sslopt={"cert_reqs": ssl.CERT_NONE},
+            header={"Authorization": f"Bearer {self.config.token}"},
+        )
+        self.socket_connection.recv()
+
+    def _websocket_send_request(
+        self, request: dict, response: bool = False
+    ) -> Optional[Dict]:
+        """
+        Method to send request to websocket
+
+        request: data required to be sent to websocket
+        response: is json response required?
+        """
+        self.socket_connection.send(json.dumps(request))
+        resp = self.socket_connection.recv()
+        if response:
+            return json.loads(resp)
+        return None
+
+    def get_dashboard_charts(self, dashboard_id: str) -> List[QlikSheet]:
+        """
+        Get dahsboard chart list
+        """
+        try:
+            self.connect_websocket(dashboard_id)
+            # self._websocket_send_request(request=None)
+            OPEN_DOC_REQ.update({"params": [dashboard_id]})
+            self._websocket_send_request(OPEN_DOC_REQ)
+            self._websocket_send_request(CREATE_SHEET_SESSION)
+            sheets = self._websocket_send_request(GET_SHEET_LAYOUT, response=True)
+            data = QlikSheetResult(**sheets)
+            return data.result.qLayout.qAppObjectList.qItems
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the dashboard charts")
+        return []
+
+    def get_dashboards_list(self) -> List[QlikAppList]:
+        """
+        Get List of all apps
+        """
+        try:
+            resp_apps = self.client.get("/v1/items?resourceType=app")
+            if resp_apps:
+                app_list = QlikAppList(apps=resp_apps.get("data", []))
+                return app_list.apps
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the app list")
+        return []
+
+    def get_dashboard_details(self, dashboard_id: str):
+        """
+        Get App Details
+        """
+        if not dashboard_id:
+            return None  # don't call api if dashboard_id is None
+        try:
+            resp_dashboard = self.client.get(f"/v1/apps/{dashboard_id}")
+            if resp_dashboard:
+                return QlikAppDetails(**resp_dashboard.get("attributes"))
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}")
+        return None
+
+    def get_dashboard_models(self) -> List[QlikTable]:
+        """
+        Get dahsboard chart list
+        """
+        try:
+            self._websocket_send_request(APP_LOADMODEL_REQ)
+            models = self._websocket_send_request(GET_LOADMODEL_LAYOUT, response=True)
+            data_models = QlikDataModelResult(**models)
+            layout = data_models.result.qLayout
+            if isinstance(layout, list):
+                tables = []
+                for layout in data_models.result.qLayout:
+                    tables.extend(layout.value.tables)
+                return tables
+            return layout.tables
+        except Exception:
+            logger.debug(traceback.format_exc())
+            logger.warning("Failed to fetch the dashboard datamodels")
+        return []
+
+    def get_collections_list(self):
+        """
+        Get List of all collections
+        """
+        return []

ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py

@@ -0,0 +1,56 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Source connection handler
+"""
+from typing import Optional
+
+from metadata.generated.schema.entity.automations.workflow import (
+    Workflow as AutomationWorkflow,
+)
+from metadata.generated.schema.entity.services.connections.dashboard.qlikCloudConnection import (
+    QlikCloudConnection,
+)
+from metadata.ingestion.connections.test_connections import test_connection_steps
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.dashboard.qlikcloud.client import QlikCloudClient
+
+
+def get_connection(connection: QlikCloudConnection) -> QlikCloudClient:
+    """
+    Create connection
+    """
+    return QlikCloudClient(connection)
+
+
+def test_connection(
+    metadata: OpenMetadata,
+    client: QlikCloudClient,
+    service_connection: QlikCloudConnection,
+    automation_workflow: Optional[AutomationWorkflow] = None,
+) -> None:
+    """
+    Test connection. This can be executed either as part
+    of a metadata workflow or during an Automation Workflow
+    """
+
+    def custom_executor():
+        return client.get_dashboards_list()
+
+    test_fn = {"GetDashboards": custom_executor}
+
+    test_connection_steps(
+        metadata=metadata,
+        test_fn=test_fn,
+        service_type=service_connection.type.value,
+        automation_workflow=automation_workflow,
+    )

ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/constants.py

@@ -0,0 +1,56 @@
+"""
+QlikCloud Constants
+"""
+
+OPEN_DOC_REQ = {
+    "method": "OpenDoc",
+    "handle": -1,
+    "outKey": -1,
+    "id": 1,
+}
+CREATE_SHEET_SESSION = {
+    "method": "CreateSessionObject",
+    "handle": 1,
+    "params": [
+        {
+            "qInfo": {"qType": "SheetList"},
+            "qAppObjectListDef": {
+                "qType": "sheet",
+                "qData": {
+                    "title": "/qMetaDef/title",
+                    "description": "/qMetaDef/description",
+                    "thumbnail": "/thumbnail",
+                    "cells": "/cells",
+                    "rank": "/rank",
+                    "columns": "/columns",
+                    "rows": "/rows",
+                },
+            },
+        }
+    ],
+    "outKey": -1,
+    "id": 2,
+}
+GET_SHEET_LAYOUT = {
+    "method": "GetLayout",
+    "handle": 2,
+    "params": [],
+    "outKey": -1,
+    "id": 3,
+}
+APP_LOADMODEL_REQ = {
+    "delta": True,
+    "handle": 1,
+    "method": "GetObject",
+    "params": ["LoadModel"],
+    "id": 4,
+    "jsonrpc": "2.0",
+}
+GET_LOADMODEL_LAYOUT = {
+    "delta": True,
+    "handle": 3,
+    "method": "GetLayout",
+    "params": [],
+    "id": 5,
+    "jsonrpc": "2.0",
+}