diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py b/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py index 8fd954de1ec5..20c05cc7a394 100644 --- a/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py +++ b/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py @@ -203,9 +203,7 @@ def get_table_partition_details( self, table_name: str, schema_name: str, inspector ) -> Tuple[bool, TablePartition]: result = self.engine.execute( - POSTGRES_PARTITION_DETAILS.format( - table_name=table_name, schema_name=schema_name - ) + POSTGRES_PARTITION_DETAILS, table_name=table_name, schema_name=schema_name ).all() if result: partition_details = TablePartition( diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/queries.py b/ingestion/src/metadata/ingestion/source/database/postgres/queries.py index 4c8556320de8..a2554a699968 100644 --- a/ingestion/src/metadata/ingestion/source/database/postgres/queries.py +++ b/ingestion/src/metadata/ingestion/source/database/postgres/queries.py @@ -71,7 +71,7 @@ col.table_schema = par.relnamespace::regnamespace::text and col.table_name = par.relname and ordinal_position = pt.column_index - where par.relname='{table_name}' and par.relnamespace::regnamespace::text='{schema_name}' + where par.relname=%(table_name)s and par.relnamespace::regnamespace::text=%(schema_name)s """ ) diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js index 60d470a230f0..3fc6d9e2e7ec 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js @@ -23,6 +23,7 @@ export const LINEAGE_ITEMS = [ entityType: 'Table', fqn: 'sample_data.ecommerce_db.shopify.fact_sale', searchIndex: SEARCH_INDEX.tables, + columns: ['sample_data.ecommerce_db.shopify.fact_sale.shop_id'], }, { term: 'fact_session', @@ -33,6 +34,7 @@ export const LINEAGE_ITEMS = [ entityType: 'Table', fqn: 'sample_data.ecommerce_db.shopify.fact_session', searchIndex: SEARCH_INDEX.tables, + columns: ['sample_data.ecommerce_db.shopify.fact_session.shop_id'], }, { term: 'shop_products', @@ -60,3 +62,13 @@ export const LINEAGE_ITEMS = [ searchIndex: SEARCH_INDEX.containers, }, ]; + +export const PIPELINE_ITEMS = [ + { + term: 'dim_location_etl', + name: 'dim_location etl', + entity: DATA_ASSETS.pipelines, + fqn: 'sample_airflow.dim_location_etl', + searchIndex: SEARCH_INDEX.pipelines, + }, +]; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js index cbecfab9f0ff..9d773264569c 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js @@ -16,17 +16,20 @@ import { verifyResponseStatusCode, visitEntityDetailsPage, } from '../../common/common'; -import { LINEAGE_ITEMS } from '../../constants/lineage.constants'; +import { + LINEAGE_ITEMS, + PIPELINE_ITEMS, +} from '../../constants/lineage.constants'; const dataTransfer = new DataTransfer(); -const dragConnection = (sourceFqn, targetFqn) => { - cy.get( - `[data-testid="lineage-node-${sourceFqn}"] .react-flow__handle-right` - ).click({ force: true }); // Adding force true for handles because it can be hidden behind the node +const dragConnection = (sourceId, targetId) => { + cy.get(`[data-testid="${sourceId}"] .react-flow__handle-right`).click({ + force: true, + }); // Adding force true for handles because it can be hidden behind the node return cy - .get(`[data-testid="lineage-node-${targetFqn}"] .react-flow__handle-left`) + .get(`[data-testid="${targetId}"] .react-flow__handle-left`) .click({ force: true }); // Adding force true for handles because it can be hidden behind the node }; @@ -54,7 +57,7 @@ const connectEdgeBetweenNodes = (fromNode, toNode) => { cy.get('[data-testid="suggestion-node"] input').click().type(toNode.term); cy.get('.ant-select-dropdown .ant-select-item').eq(0).click(); - dragConnection(fromNode.fqn, toNode.fqn); + dragConnection(`lineage-node-${fromNode.fqn}`, `lineage-node-${toNode.fqn}`); verifyResponseStatusCode('@lineageApi', 200); }; @@ -75,7 +78,81 @@ const deleteNode = (node) => { verifyResponseStatusCode('@lineageDeleteApi', 200); }; -describe('Entity Details Page', () => { +const applyPipelineFromModal = (fromNode, toNode, pipelineData) => { + interceptURL('PUT', '/api/v1/lineage', 'lineageApi'); + cy.get(`[data-testid="edge-${fromNode.fqn}-${toNode.fqn}"]`).click({ + force: true, + }); + cy.get('[data-testid="add-pipeline"]').click(); + + cy.get('[data-testid="add-edge-modal"] [data-testid="field-input"]') + .click() + .type(pipelineData.term); + + cy.get(`[data-testid="pipeline-entry-${pipelineData.fqn}"]`).click(); + cy.get('[data-testid="save-button"]').click(); + + verifyResponseStatusCode('@lineageApi', 200); +}; + +const verifyPipelineDataInDrawer = (fromNode, toNode, pipelineData) => { + cy.get( + `[data-testid="pipeline-label-${fromNode.fqn}-${toNode.fqn}"]` + ).click(); + cy.get('.edge-info-drawer').should('be.visible'); + cy.get('.edge-info-drawer [data-testid="Edge"] a').contains( + pipelineData.name + ); + cy.get('.edge-info-drawer .ant-drawer-header .anticon-close').click(); +}; + +const addPipelineBetweenNodes = (sourceEntity, targetEntity, pipelineItem) => { + visitEntityDetailsPage({ + term: sourceEntity.term, + serviceName: sourceEntity.serviceName, + entity: sourceEntity.entity, + }); + + cy.get('[data-testid="lineage"]').click(); + cy.get('[data-testid="edit-lineage"]').click(); + connectEdgeBetweenNodes(sourceEntity, targetEntity); + if (pipelineItem) { + applyPipelineFromModal(sourceEntity, targetEntity, pipelineItem); + cy.get('[data-testid="edit-lineage"]').click(); + verifyPipelineDataInDrawer(sourceEntity, targetEntity, pipelineItem); + } +}; + +const addColumnLineage = (fromNode, toNode) => { + interceptURL('PUT', '/api/v1/lineage', 'lineageApi'); + cy.get('.react-flow__controls-fitview').click({ force: true }); + cy.get( + `[data-testid="lineage-node-${fromNode.fqn}"] [data-testid="expand-cols-btn"]` + ).click({ force: true }); + cy.get( + `[data-testid="lineage-node-${fromNode.fqn}"] [data-testid="show-more-cols-btn"]` + ).click({ force: true }); + cy.get('.react-flow__controls-fitview').click({ force: true }); + cy.get( + `[data-testid="lineage-node-${toNode.fqn}"] [data-testid="expand-cols-btn"]` + ).click({ force: true }); + cy.get( + `[data-testid="lineage-node-${toNode.fqn}"] [data-testid="show-more-cols-btn"]` + ).click({ force: true }); + cy.get('.react-flow__controls-fitview').click({ force: true }); + + dragConnection( + `column-${fromNode.columns[0]}`, + `column-${toNode.columns[0]}` + ); + verifyResponseStatusCode('@lineageApi', 200); + cy.get('[data-testid="edit-lineage"]').click(); + cy.get( + `[data-testid="column-edge-${fromNode.columns[0]}-${toNode.columns[0]}"]` + ); +}; + +describe('Lineage verification', () => { beforeEach(() => { cy.login(); }); @@ -134,4 +211,30 @@ describe('Entity Details Page', () => { cy.get('[data-testid="edit-lineage"]').click(); }); }); + + it('Lineage Add Pipeline Between Tables', () => { + const sourceEntity = LINEAGE_ITEMS[0]; + const targetEntity = LINEAGE_ITEMS[1]; + addPipelineBetweenNodes(sourceEntity, targetEntity, PIPELINE_ITEMS[0]); + cy.get('[data-testid="edit-lineage"]').click(); + deleteNode(targetEntity); + }); + + it('Lineage Add Pipeline Between Table and Topic', () => { + const sourceEntity = LINEAGE_ITEMS[1]; + const targetEntity = LINEAGE_ITEMS[2]; + addPipelineBetweenNodes(sourceEntity, targetEntity, PIPELINE_ITEMS[0]); + cy.get('[data-testid="edit-lineage"]').click(); + deleteNode(targetEntity); + }); + + it('Add column lineage', () => { + const sourceEntity = LINEAGE_ITEMS[0]; + const targetEntity = LINEAGE_ITEMS[1]; + addPipelineBetweenNodes(sourceEntity, targetEntity); + // Add column lineage + addColumnLineage(sourceEntity, targetEntity); + cy.get('[data-testid="edit-lineage"]').click(); + deleteNode(targetEntity); + }); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/postgres.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/postgres.spec.js index baa8ce25fb52..b4b57780854e 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/postgres.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/postgres.spec.js @@ -37,14 +37,14 @@ const clearQuery = 'select pg_stat_statements_reset()'; const selectQuery = 'SELECT * FROM sales.order_items oi INNER JOIN sales.orders o ON oi.order_id=o.order_id'; -// @ayush - Need to fix postgres ingestion issue -describe.skip('Postgres Ingestion', () => { +describe('Postgres Ingestion', () => { beforeEach(() => { cy.login(); }); it('Trigger select query', () => { cy.postgreSQL(clearQuery); + cy.wait(500); cy.postgreSQL(selectQuery); }); @@ -132,7 +132,7 @@ describe.skip('Postgres Ingestion', () => { cy.get('#root\\/filterCondition') .scrollIntoView() - .type(`s.query like '%%${tableName}%%'`); + .type(`lower(s.query) like '%%${tableName}%%'`); cy.get('[data-testid="submit-btn"]') .scrollIntoView() .should('be.visible') diff --git a/openmetadata-ui/src/main/resources/ui/package.json b/openmetadata-ui/src/main/resources/ui/package.json index 0b227efd7c9c..0ea47b7f1457 100644 --- a/openmetadata-ui/src/main/resources/ui/package.json +++ b/openmetadata-ui/src/main/resources/ui/package.json @@ -34,7 +34,7 @@ "parse-conn-schema": "node parseConnectionSchema && rm -rf connTemp", "parse-ingestion-schema": "node parseIngestionSchema && rm -rf ingestionTemp", "js-antlr": "PWD=$(echo $PWD) antlr4 -Dlanguage=JavaScript -o src/generated/antlr \"$PWD\"/../../../../../openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4", - "cypress:open": "CYPRESS_BASE_URL=http://localhost:8585 cypress open --e2e", + "cypress:open": "CYPRESS_BASE_URL=http://localhost:3000 cypress open --e2e", "cypress:run": "CYPRESS_BASE_URL=http://localhost:8585 cypress run --config-file=cypress.config.ts", "cypress:run:record": "cypress run --config-file=cypress.config.ts --record --parallel", "i18n": "sync-i18n --files '**/locale/languages/*.json' --primary en-us --space 2 --fn", @@ -235,4 +235,4 @@ "prosemirror-transform": "1.7.0", "prosemirror-view": "1.28.2" } -} +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx index 48bea0630089..651cbc719705 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx @@ -190,7 +190,7 @@ const EdgeInfoDrawer = ({ } getContainer={false} @@ -207,7 +207,7 @@ const EdgeInfoDrawer = ({ Object.values(edgeData).map( (data) => data.value && ( - + {`${data.key}:`} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx index c22df4e72333..54850f9dc060 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/AppPipelineModel/AddPipeLineModal.tsx @@ -149,6 +149,7 @@ const AddPipeLineModal = ({ className={classNames('edge-option-item gap-2', { active: edgeSelection?.id === item.id, })} + data-testid={`pipeline-entry-${item.fullyQualifiedName}`} key={item.id} onClick={() => setEdgeSelection(item)}>
{icon}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.test.tsx index 5357a87d85ea..e1377f1bf81a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.test.tsx @@ -45,10 +45,12 @@ const mockCustomEdgeProp = { edge: { fromEntity: { id: '1', + fqn: 'table1', type: 'table', }, toEntity: { id: '2', + fqn: 'table2', type: 'table', }, pipeline: { @@ -107,7 +109,9 @@ describe('Test CustomEdge Component', () => { } ); - const pipelineLabelAsEdge = await screen.findByTestId('pipeline-label'); + const pipelineLabelAsEdge = await screen.findByTestId( + 'pipeline-label-table1-table2' + ); expect(pipelineLabelAsEdge).toBeInTheDocument(); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx index 6c15e2d8abd9..d07d82183cf4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomEdge.component.tsx @@ -273,7 +273,7 @@ export const CustomEdge = ({ hasLabel && getLineageEdgeIcon( , - 'pipeline-label', + `pipeline-label-${edge.fromEntity.fqn}-${edge.toEntity.fqn}`, currentPipelineStatus )} {isColumnLineageAllowed && @@ -286,7 +286,10 @@ export const CustomEdge = ({ {!isColumnLineageAllowed && data.columnFunctionValue && data.isExpanded && - getLineageEdgeIcon(, 'function-icon')} + getLineageEdgeIcon( + , + `function-icon-${edge.fromEntity.fqn}-${edge.toEntity.fqn}` + )} ); }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx index 0618f79e7a8e..561bbadd03d7 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNodeV1.component.tsx @@ -422,7 +422,7 @@ const CustomNodeV1 = (props: NodeProps) => { ? 'custom-node-header-tracing' : 'custom-node-column-lineage-normal bg-white' )} - data-testid="column" + data-testid={`column-${column.fullyQualifiedName}`} key={column.fullyQualifiedName} onClick={(e) => { e.stopPropagation(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx index 9eee79c5918c..f64dd017e383 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/LineageProvider/LineageProvider.tsx @@ -211,12 +211,10 @@ const LineageProvider = ({ children }: LineageProviderProps) => { isEqual ); - setEntityLineage((prev) => { - return { - ...prev, - nodes: allNodes, - edges: allEdges, - }; + setEntityLineage({ + nodes: allNodes, + edges: allEdges, + entity: res.entity, }); } catch (err) { showErrorToast( @@ -377,8 +375,15 @@ const LineageProvider = ({ children }: LineageProviderProps) => { if (!entityLineage) { return; } + + // Filter column edges, as main edge will automatically remove column + // edge on delete + const nodeEdges = edges.filter( + (item) => item?.data?.isColumnLineage === false + ); + // Get edges connected to selected node - const edgesToRemove = getConnectedEdges([node as Node], edges); + const edgesToRemove = getConnectedEdges([node as Node], nodeEdges); edgesToRemove.forEach((edge) => { removeEdgeHandler(edge, true); }); @@ -777,6 +782,23 @@ const LineageProvider = ({ children }: LineageProviderProps) => { return edge; }); + setSelectedEdge((pre) => { + if (!pre) { + return pre; + } + + return { + ...pre, + data: { + ...pre.data, + edge: { + ...pre.data.edge, + description, + sqlQuery, + }, + }, + }; + }); setEntityLineage((prev) => { return { ...prev,