Merge branch 'main' into issue-14694

open-metadata · Jan 15, 2024 · 7a652a7 · 7a652a7
2 parents cf84784 + 21a701f
commit 7a652a7
Show file tree

Hide file tree

Showing 12 changed files with 173 additions and 30 deletions.
diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py b/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py
@@ -203,9 +203,7 @@ def get_table_partition_details(
         self, table_name: str, schema_name: str, inspector
     ) -> Tuple[bool, TablePartition]:
         result = self.engine.execute(
-            POSTGRES_PARTITION_DETAILS.format(
-                table_name=table_name, schema_name=schema_name
-            )
+            POSTGRES_PARTITION_DETAILS, table_name=table_name, schema_name=schema_name
         ).all()
         if result:
             partition_details = TablePartition(

diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/queries.py b/ingestion/src/metadata/ingestion/source/database/postgres/queries.py
@@ -71,7 +71,7 @@
         col.table_schema = par.relnamespace::regnamespace::text
         and col.table_name = par.relname
         and ordinal_position = pt.column_index
-     where par.relname='{table_name}' and  par.relnamespace::regnamespace::text='{schema_name}'
+     where par.relname=%(table_name)s and  par.relnamespace::regnamespace::text=%(schema_name)s
     """
 )
 

diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js
@@ -23,6 +23,7 @@ export const LINEAGE_ITEMS = [
     entityType: 'Table',
     fqn: 'sample_data.ecommerce_db.shopify.fact_sale',
     searchIndex: SEARCH_INDEX.tables,
+    columns: ['sample_data.ecommerce_db.shopify.fact_sale.shop_id'],
   },
   {
     term: 'fact_session',
@@ -33,6 +34,7 @@ export const LINEAGE_ITEMS = [
     entityType: 'Table',
     fqn: 'sample_data.ecommerce_db.shopify.fact_session',
     searchIndex: SEARCH_INDEX.tables,
+    columns: ['sample_data.ecommerce_db.shopify.fact_session.shop_id'],
   },
   {
     term: 'shop_products',
@@ -60,3 +62,13 @@ export const LINEAGE_ITEMS = [
     searchIndex: SEARCH_INDEX.containers,
   },
 ];
+
+export const PIPELINE_ITEMS = [
+  {
+    term: 'dim_location_etl',
+    name: 'dim_location etl',
+    entity: DATA_ASSETS.pipelines,
+    fqn: 'sample_airflow.dim_location_etl',
+    searchIndex: SEARCH_INDEX.pipelines,
+  },
+];
diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js
@@ -16,17 +16,20 @@ import {
   verifyResponseStatusCode,
   visitEntityDetailsPage,
 } from '../../common/common';
-import { LINEAGE_ITEMS } from '../../constants/lineage.constants';
+import {
+  LINEAGE_ITEMS,
+  PIPELINE_ITEMS,
+} from '../../constants/lineage.constants';
 
 const dataTransfer = new DataTransfer();
 
-const dragConnection = (sourceFqn, targetFqn) => {
-  cy.get(
-    `[data-testid="lineage-node-${sourceFqn}"] .react-flow__handle-right`
-  ).click({ force: true }); // Adding force true for handles because it can be hidden behind the node
+const dragConnection = (sourceId, targetId) => {
+  cy.get(`[data-testid="${sourceId}"] .react-flow__handle-right`).click({
+    force: true,
+  }); // Adding force true for handles because it can be hidden behind the node
 
   return cy
-    .get(`[data-testid="lineage-node-${targetFqn}"] .react-flow__handle-left`)
+    .get(`[data-testid="${targetId}"] .react-flow__handle-left`)
     .click({ force: true }); // Adding force true for handles because it can be hidden behind the node
 };
 
@@ -54,7 +57,7 @@ const connectEdgeBetweenNodes = (fromNode, toNode) => {
   cy.get('[data-testid="suggestion-node"] input').click().type(toNode.term);
   cy.get('.ant-select-dropdown .ant-select-item').eq(0).click();
 
-  dragConnection(fromNode.fqn, toNode.fqn);
+  dragConnection(`lineage-node-${fromNode.fqn}`, `lineage-node-${toNode.fqn}`);
   verifyResponseStatusCode('@lineageApi', 200);
 };
 
@@ -75,7 +78,81 @@ const deleteNode = (node) => {
   verifyResponseStatusCode('@lineageDeleteApi', 200);
 };
 
-describe('Entity Details Page', () => {
+const applyPipelineFromModal = (fromNode, toNode, pipelineData) => {
+  interceptURL('PUT', '/api/v1/lineage', 'lineageApi');
+  cy.get(`[data-testid="edge-${fromNode.fqn}-${toNode.fqn}"]`).click({
+    force: true,
+  });
+  cy.get('[data-testid="add-pipeline"]').click();
+
+  cy.get('[data-testid="add-edge-modal"] [data-testid="field-input"]')
+    .click()
+    .type(pipelineData.term);
+
+  cy.get(`[data-testid="pipeline-entry-${pipelineData.fqn}"]`).click();
+  cy.get('[data-testid="save-button"]').click();
+
+  verifyResponseStatusCode('@lineageApi', 200);
+};
+
+const verifyPipelineDataInDrawer = (fromNode, toNode, pipelineData) => {
+  cy.get(
+    `[data-testid="pipeline-label-${fromNode.fqn}-${toNode.fqn}"]`
+  ).click();
+  cy.get('.edge-info-drawer').should('be.visible');
+  cy.get('.edge-info-drawer [data-testid="Edge"] a').contains(
+    pipelineData.name
+  );
+  cy.get('.edge-info-drawer .ant-drawer-header .anticon-close').click();
+};
+
+const addPipelineBetweenNodes = (sourceEntity, targetEntity, pipelineItem) => {
+  visitEntityDetailsPage({
+    term: sourceEntity.term,
+    serviceName: sourceEntity.serviceName,
+    entity: sourceEntity.entity,
+  });
+
+  cy.get('[data-testid="lineage"]').click();
+  cy.get('[data-testid="edit-lineage"]').click();
+  connectEdgeBetweenNodes(sourceEntity, targetEntity);
+  if (pipelineItem) {
+    applyPipelineFromModal(sourceEntity, targetEntity, pipelineItem);
+    cy.get('[data-testid="edit-lineage"]').click();
+    verifyPipelineDataInDrawer(sourceEntity, targetEntity, pipelineItem);
+  }
+};
+
+const addColumnLineage = (fromNode, toNode) => {
+  interceptURL('PUT', '/api/v1/lineage', 'lineageApi');
+  cy.get('.react-flow__controls-fitview').click({ force: true });
+  cy.get(
+    `[data-testid="lineage-node-${fromNode.fqn}"] [data-testid="expand-cols-btn"]`
+  ).click({ force: true });
+  cy.get(
+    `[data-testid="lineage-node-${fromNode.fqn}"] [data-testid="show-more-cols-btn"]`
+  ).click({ force: true });
+  cy.get('.react-flow__controls-fitview').click({ force: true });
+  cy.get(
+    `[data-testid="lineage-node-${toNode.fqn}"] [data-testid="expand-cols-btn"]`
+  ).click({ force: true });
+  cy.get(
+    `[data-testid="lineage-node-${toNode.fqn}"] [data-testid="show-more-cols-btn"]`
+  ).click({ force: true });
+  cy.get('.react-flow__controls-fitview').click({ force: true });
+
+  dragConnection(
+    `column-${fromNode.columns[0]}`,
+    `column-${toNode.columns[0]}`
+  );
+  verifyResponseStatusCode('@lineageApi', 200);
+  cy.get('[data-testid="edit-lineage"]').click();
+  cy.get(
+    `[data-testid="column-edge-${fromNode.columns[0]}-${toNode.columns[0]}"]`
+  );
+};
+
+describe('Lineage verification', () => {
   beforeEach(() => {
     cy.login();
   });
@@ -134,4 +211,30 @@ describe('Entity Details Page', () => {
       cy.get('[data-testid="edit-lineage"]').click();
     });
   });
+
+  it('Lineage Add Pipeline Between Tables', () => {
+    const sourceEntity = LINEAGE_ITEMS[0];
+    const targetEntity = LINEAGE_ITEMS[1];
+    addPipelineBetweenNodes(sourceEntity, targetEntity, PIPELINE_ITEMS[0]);
+    cy.get('[data-testid="edit-lineage"]').click();
+    deleteNode(targetEntity);
+  });
+
+  it('Lineage Add Pipeline Between Table and Topic', () => {
+    const sourceEntity = LINEAGE_ITEMS[1];
+    const targetEntity = LINEAGE_ITEMS[2];
+    addPipelineBetweenNodes(sourceEntity, targetEntity, PIPELINE_ITEMS[0]);
+    cy.get('[data-testid="edit-lineage"]').click();
+    deleteNode(targetEntity);
+  });
+
+  it('Add column lineage', () => {
+    const sourceEntity = LINEAGE_ITEMS[0];
+    const targetEntity = LINEAGE_ITEMS[1];
+    addPipelineBetweenNodes(sourceEntity, targetEntity);
+    // Add column lineage
+    addColumnLineage(sourceEntity, targetEntity);
+    cy.get('[data-testid="edit-lineage"]').click();
+    deleteNode(targetEntity);
+  });
 });
diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/postgres.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/postgres.spec.js
@@ -37,14 +37,14 @@ const clearQuery = 'select pg_stat_statements_reset()';
 const selectQuery =
   'SELECT * FROM sales.order_items oi INNER JOIN sales.orders o ON oi.order_id=o.order_id';
 
-// @ayush - Need to fix postgres ingestion issue
-describe.skip('Postgres Ingestion', () => {
+describe('Postgres Ingestion', () => {
   beforeEach(() => {
     cy.login();
   });
 
   it('Trigger select query', () => {
     cy.postgreSQL(clearQuery);
+    cy.wait(500);
     cy.postgreSQL(selectQuery);
   });
 
@@ -132,7 +132,7 @@ describe.skip('Postgres Ingestion', () => {
 
     cy.get('#root\\/filterCondition')
       .scrollIntoView()
-      .type(`s.query like '%%${tableName}%%'`);
+      .type(`lower(s.query) like '%%${tableName}%%'`);
     cy.get('[data-testid="submit-btn"]')
       .scrollIntoView()
       .should('be.visible')

diff --git a/openmetadata-ui/src/main/resources/ui/package.json b/openmetadata-ui/src/main/resources/ui/package.json
@@ -34,7 +34,7 @@
     "parse-conn-schema": "node parseConnectionSchema && rm -rf connTemp",
     "parse-ingestion-schema": "node parseIngestionSchema && rm -rf ingestionTemp",
     "js-antlr": "PWD=$(echo $PWD) antlr4 -Dlanguage=JavaScript -o src/generated/antlr \"$PWD\"/../../../../../openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4",
-    "cypress:open": "CYPRESS_BASE_URL=http://localhost:8585 cypress open --e2e",
+    "cypress:open": "CYPRESS_BASE_URL=http://localhost:3000 cypress open --e2e",
     "cypress:run": "CYPRESS_BASE_URL=http://localhost:8585 cypress run --config-file=cypress.config.ts",
     "cypress:run:record": "cypress run --config-file=cypress.config.ts --record --parallel",
     "i18n": "sync-i18n --files '**/locale/languages/*.json' --primary en-us --space 2 --fn",
@@ -235,4 +235,4 @@
     "prosemirror-transform": "1.7.0",
     "prosemirror-view": "1.28.2"
   }
-}
+}
diff --git a/...src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx b/...src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx
@@ -190,7 +190,7 @@ const EdgeInfoDrawer = ({
       <Drawer
         destroyOnClose
         bodyStyle={{ padding: 16 }}
-        className="entity-panel-container"
+        className="entity-panel-container edge-info-drawer"
         closable={false}
         extra={<CloseOutlined onClick={onClose} />}
         getContainer={false}
@@ -207,7 +207,7 @@ const EdgeInfoDrawer = ({
               Object.values(edgeData).map(
                 (data) =>
                   data.value && (
-                    <Col key={data.key} span={24}>
+                    <Col data-testid={data.key} key={data.key} span={24}>
                       <Typography.Text className="m-r-sm summary-panel-section-title">
                         {`${data.key}:`}
                       </Typography.Text>

diff --git a/...in/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx b/...in/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx
@@ -149,6 +149,7 @@ const AddPipeLineModal = ({
               className={classNames('edge-option-item gap-2', {
                 active: edgeSelection?.id === item.id,
               })}
+              data-testid={`pipeline-entry-${item.fullyQualifiedName}`}
               key={item.id}
               onClick={() => setEdgeSelection(item)}>
               <div className="flex-center mention-icon-image">{icon}</div>

diff --git a/...i/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.test.tsx b/...i/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.test.tsx
@@ -45,10 +45,12 @@ const mockCustomEdgeProp = {
     edge: {
       fromEntity: {
         id: '1',
+        fqn: 'table1',
         type: 'table',
       },
       toEntity: {
         id: '2',
+        fqn: 'table2',
         type: 'table',
       },
       pipeline: {
@@ -107,7 +109,9 @@ describe('Test CustomEdge Component', () => {
       }
     );
 
-    const pipelineLabelAsEdge = await screen.findByTestId('pipeline-label');
+    const pipelineLabelAsEdge = await screen.findByTestId(
+      'pipeline-label-table1-table2'
+    );
 
     expect(pipelineLabelAsEdge).toBeInTheDocument();
   });

diff --git a/...ata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx b/...ata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx
@@ -273,7 +273,7 @@ export const CustomEdge = ({
         hasLabel &&
         getLineageEdgeIcon(
           <PipelineIcon />,
-          'pipeline-label',
+          `pipeline-label-${edge.fromEntity.fqn}-${edge.toEntity.fqn}`,
           currentPipelineStatus
         )}
       {isColumnLineageAllowed &&
@@ -286,7 +286,10 @@ export const CustomEdge = ({
       {!isColumnLineageAllowed &&
         data.columnFunctionValue &&
         data.isExpanded &&
-        getLineageEdgeIcon(<FunctionIcon />, 'function-icon')}
+        getLineageEdgeIcon(
+          <FunctionIcon />,
+          `function-icon-${edge.fromEntity.fqn}-${edge.toEntity.fqn}`
+        )}
     </Fragment>
   );
 };
diff --git a/...a-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx b/...a-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx
@@ -422,7 +422,7 @@ const CustomNodeV1 = (props: NodeProps) => {
                               ? 'custom-node-header-tracing'
                               : 'custom-node-column-lineage-normal bg-white'
                           )}
-                          data-testid="column"
+                          data-testid={`column-${column.fullyQualifiedName}`}
                           key={column.fullyQualifiedName}
                           onClick={(e) => {
                             e.stopPropagation();

diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx
@@ -211,12 +211,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
           isEqual
         );
 
-        setEntityLineage((prev) => {
-          return {
-            ...prev,
-            nodes: allNodes,
-            edges: allEdges,
-          };
+        setEntityLineage({
+          nodes: allNodes,
+          edges: allEdges,
+          entity: res.entity,
         });
       } catch (err) {
         showErrorToast(
@@ -377,8 +375,15 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
       if (!entityLineage) {
         return;
       }
+
+      // Filter column edges, as main edge will automatically remove column
+      // edge on delete
+      const nodeEdges = edges.filter(
+        (item) => item?.data?.isColumnLineage === false
+      );
+
       // Get edges connected to selected node
-      const edgesToRemove = getConnectedEdges([node as Node], edges);
+      const edgesToRemove = getConnectedEdges([node as Node], nodeEdges);
       edgesToRemove.forEach((edge) => {
         removeEdgeHandler(edge, true);
       });
@@ -777,6 +782,23 @@ const LineageProvider = ({ children }: LineageProviderProps) => {
 
           return edge;
         });
+        setSelectedEdge((pre) => {
+          if (!pre) {
+            return pre;
+          }
+
+          return {
+            ...pre,
+            data: {
+              ...pre.data,
+              edge: {
+                ...pre.data.edge,
+                description,
+                sqlQuery,
+              },
+            },
+          };
+        });
         setEntityLineage((prev) => {
           return {
             ...prev,