Fix open-metadata#14089: removed unnecessary defaults from models, re…

…moved tableQualifier parsing code
harshsoni2024 · Mar 28, 2024 · 70a8193 · 70a8193
1 parent c328e9a
commit 70a8193
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 27 deletions.
diff --git a/ingestion/src/metadata/examples/workflows/qlikcloud.yaml b/ingestion/src/metadata/examples/workflows/qlikcloud.yaml
@@ -4,8 +4,8 @@ source:
   serviceConnection:
     config:
       type: QlikCloud
-      token:
-      hostPort:
+      token: eyJhbGciOiJFUzM4NCIsImtpZCI6ImM3YWUzNjRjLTNhN2UtNDQyYy1hNDliLWI0M2MzOGEwMjM3YyIsInR5cCI6IkpXVCJ9.eyJzdWJUeXBlIjoidXNlciIsInRlbmFudElkIjoiZGJpWnVFS04yNmE2NkFuSjJ5WGZtTE1zSGlPdllmd3AiLCJqdGkiOiJjN2FlMzY0Yy0zYTdlLTQ0MmMtYTQ5Yi1iNDNjMzhhMDIzN2MiLCJhdWQiOiJxbGlrLmFwaSIsImlzcyI6InFsaWsuYXBpL2FwaS1rZXlzIiwic3ViIjoiNjVlMTg3ZTUyZDNjOTM2ZWE3YjE1ZTkxIn0.4AY1N-XQ5js5qGl1OoQGPQSaz4uhPMNq09LVM4WMqqZVu3zFap2MGqKT0k7wJeaDRLaySnPR6y1O3TeYl9WVWaqzp5hRfTfkZHe2h_oJA4GNz3po6HVmvspmvqtZPPU8
+      hostPort: http://localhost:2000
 
   sourceConfig:
     config:

diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/client.py
@@ -75,7 +75,7 @@ def connect_websocket(self, dashboard_id: str = None) -> None:
         if self.socket_connection:
             self.socket_connection.close()
         self.socket_connection = create_connection(
-            f"wss{clean_uri(self.config.hostPort)[5:]}/app/{dashboard_id or ''}",
+            f"wss://{clean_uri(self.config.hostPort.host)}/app/{dashboard_id or ''}",
             sslopt={"cert_reqs": ssl.CERT_NONE},
             header={"Authorization": f"Bearer {self.config.token}"},
         )

diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/metadata.py
@@ -166,31 +166,23 @@ def _get_datamodel(self, datamodel_id):
         return None
 
     def _get_database_table(
-        self, db_service_entity: DatabaseService, datamodel: QlikTable
+        self,
+        db_service_entity: DatabaseService,
+        data_model_entity: DashboardDataModel,
     ) -> Optional[Table]:
         """
         Get the table entity for lineage
         """
         # table.name in tableau can come as db.schema.table_name. Hence the logic to split it
-        if datamodel.tableName and db_service_entity:
+        if data_model_entity and db_service_entity:
             try:
-                if len(datamodel.connectionInfo.tableQualifiers) > 1:
-                    (
-                        database_name,
-                        schema_name,
-                    ) = datamodel.connectionInfo.tableQualifiers[-2:]
-                elif len(datamodel.connectionInfo.tableQualifiers) == 1:
-                    schema_name = datamodel.connectionInfo.tableQualifiers[-1]
-                    database_name = None
-                else:
-                    schema_name, database_name = None, None
-
+                schema_name, database_name = None, None
                 table_fqn = fqn.build(
                     self.metadata,
                     entity_type=Table,
                     service_name=db_service_entity.name.__root__,
                     schema_name=schema_name,
-                    table_name=datamodel.tableName,
+                    table_name=data_model_entity.displayName,
                     database_name=database_name,
                 )
                 if table_fqn:
@@ -212,15 +204,12 @@ def yield_dashboard_lineage_details(
         db_service_entity = self.metadata.get_by_name(
             entity=DatabaseService, fqn=db_service_name
         )
-        for datamodel in self.data_models or []:
+        for datamodel_id in self.context.dataModels or []:
             try:
-                if not datamodel.id in self.context.dataModels:
-                    # only process datamodels which are parsed in context
-                    continue
-                data_model_entity = self._get_datamodel(datamodel_id=datamodel.id)
+                data_model_entity = self._get_datamodel(datamodel_id=datamodel_id)
                 if data_model_entity:
                     om_table = self._get_database_table(
-                        db_service_entity, datamodel=datamodel
+                        db_service_entity, data_model_entity
                     )
                     if om_table:
                         yield self._get_add_lineage_request(

diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/models.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/models.py
@@ -97,13 +97,11 @@ class QlikTablesList(BaseModel):
 
 
 class QlikDataModelValue(BaseModel):
-    value: Optional[QlikTablesList] = QlikTablesList()
+    value: Optional[QlikTablesList]
 
 
 class QlikDataModelLayout(BaseModel):
-    qLayout: Optional[
-        Union[QlikTablesList, List[QlikDataModelValue]]
-    ] = QlikTablesList()
+    qLayout: Optional[Union[QlikTablesList, List[QlikDataModelValue]]] = []
 
 
 class QlikDataModelResult(BaseModel):