diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py b/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py
index 8fd954de1ec5..20c05cc7a394 100644
--- a/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/database/postgres/metadata.py
@@ -203,9 +203,7 @@ def get_table_partition_details(
self, table_name: str, schema_name: str, inspector
) -> Tuple[bool, TablePartition]:
result = self.engine.execute(
- POSTGRES_PARTITION_DETAILS.format(
- table_name=table_name, schema_name=schema_name
- )
+ POSTGRES_PARTITION_DETAILS, table_name=table_name, schema_name=schema_name
).all()
if result:
partition_details = TablePartition(
diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/queries.py b/ingestion/src/metadata/ingestion/source/database/postgres/queries.py
index 4c8556320de8..a2554a699968 100644
--- a/ingestion/src/metadata/ingestion/source/database/postgres/queries.py
+++ b/ingestion/src/metadata/ingestion/source/database/postgres/queries.py
@@ -71,7 +71,7 @@
col.table_schema = par.relnamespace::regnamespace::text
and col.table_name = par.relname
and ordinal_position = pt.column_index
- where par.relname='{table_name}' and par.relnamespace::regnamespace::text='{schema_name}'
+ where par.relname=%(table_name)s and par.relnamespace::regnamespace::text=%(schema_name)s
"""
)
diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js
index 60d470a230f0..3fc6d9e2e7ec 100644
--- a/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js
+++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/lineage.constants.js
@@ -23,6 +23,7 @@ export const LINEAGE_ITEMS = [
entityType: 'Table',
fqn: 'sample_data.ecommerce_db.shopify.fact_sale',
searchIndex: SEARCH_INDEX.tables,
+ columns: ['sample_data.ecommerce_db.shopify.fact_sale.shop_id'],
},
{
term: 'fact_session',
@@ -33,6 +34,7 @@ export const LINEAGE_ITEMS = [
entityType: 'Table',
fqn: 'sample_data.ecommerce_db.shopify.fact_session',
searchIndex: SEARCH_INDEX.tables,
+ columns: ['sample_data.ecommerce_db.shopify.fact_session.shop_id'],
},
{
term: 'shop_products',
@@ -60,3 +62,13 @@ export const LINEAGE_ITEMS = [
searchIndex: SEARCH_INDEX.containers,
},
];
+
+export const PIPELINE_ITEMS = [
+ {
+ term: 'dim_location_etl',
+ name: 'dim_location etl',
+ entity: DATA_ASSETS.pipelines,
+ fqn: 'sample_airflow.dim_location_etl',
+ searchIndex: SEARCH_INDEX.pipelines,
+ },
+];
diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js
index cbecfab9f0ff..9d773264569c 100644
--- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js
+++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Flow/Lineage.spec.js
@@ -16,17 +16,20 @@ import {
verifyResponseStatusCode,
visitEntityDetailsPage,
} from '../../common/common';
-import { LINEAGE_ITEMS } from '../../constants/lineage.constants';
+import {
+ LINEAGE_ITEMS,
+ PIPELINE_ITEMS,
+} from '../../constants/lineage.constants';
const dataTransfer = new DataTransfer();
-const dragConnection = (sourceFqn, targetFqn) => {
- cy.get(
- `[data-testid="lineage-node-${sourceFqn}"] .react-flow__handle-right`
- ).click({ force: true }); // Adding force true for handles because it can be hidden behind the node
+const dragConnection = (sourceId, targetId) => {
+ cy.get(`[data-testid="${sourceId}"] .react-flow__handle-right`).click({
+ force: true,
+ }); // Adding force true for handles because it can be hidden behind the node
return cy
- .get(`[data-testid="lineage-node-${targetFqn}"] .react-flow__handle-left`)
+ .get(`[data-testid="${targetId}"] .react-flow__handle-left`)
.click({ force: true }); // Adding force true for handles because it can be hidden behind the node
};
@@ -54,7 +57,7 @@ const connectEdgeBetweenNodes = (fromNode, toNode) => {
cy.get('[data-testid="suggestion-node"] input').click().type(toNode.term);
cy.get('.ant-select-dropdown .ant-select-item').eq(0).click();
- dragConnection(fromNode.fqn, toNode.fqn);
+ dragConnection(`lineage-node-${fromNode.fqn}`, `lineage-node-${toNode.fqn}`);
verifyResponseStatusCode('@lineageApi', 200);
};
@@ -75,7 +78,81 @@ const deleteNode = (node) => {
verifyResponseStatusCode('@lineageDeleteApi', 200);
};
-describe('Entity Details Page', () => {
+const applyPipelineFromModal = (fromNode, toNode, pipelineData) => {
+ interceptURL('PUT', '/api/v1/lineage', 'lineageApi');
+ cy.get(`[data-testid="edge-${fromNode.fqn}-${toNode.fqn}"]`).click({
+ force: true,
+ });
+ cy.get('[data-testid="add-pipeline"]').click();
+
+ cy.get('[data-testid="add-edge-modal"] [data-testid="field-input"]')
+ .click()
+ .type(pipelineData.term);
+
+ cy.get(`[data-testid="pipeline-entry-${pipelineData.fqn}"]`).click();
+ cy.get('[data-testid="save-button"]').click();
+
+ verifyResponseStatusCode('@lineageApi', 200);
+};
+
+const verifyPipelineDataInDrawer = (fromNode, toNode, pipelineData) => {
+ cy.get(
+ `[data-testid="pipeline-label-${fromNode.fqn}-${toNode.fqn}"]`
+ ).click();
+ cy.get('.edge-info-drawer').should('be.visible');
+ cy.get('.edge-info-drawer [data-testid="Edge"] a').contains(
+ pipelineData.name
+ );
+ cy.get('.edge-info-drawer .ant-drawer-header .anticon-close').click();
+};
+
+const addPipelineBetweenNodes = (sourceEntity, targetEntity, pipelineItem) => {
+ visitEntityDetailsPage({
+ term: sourceEntity.term,
+ serviceName: sourceEntity.serviceName,
+ entity: sourceEntity.entity,
+ });
+
+ cy.get('[data-testid="lineage"]').click();
+ cy.get('[data-testid="edit-lineage"]').click();
+ connectEdgeBetweenNodes(sourceEntity, targetEntity);
+ if (pipelineItem) {
+ applyPipelineFromModal(sourceEntity, targetEntity, pipelineItem);
+ cy.get('[data-testid="edit-lineage"]').click();
+ verifyPipelineDataInDrawer(sourceEntity, targetEntity, pipelineItem);
+ }
+};
+
+const addColumnLineage = (fromNode, toNode) => {
+ interceptURL('PUT', '/api/v1/lineage', 'lineageApi');
+ cy.get('.react-flow__controls-fitview').click({ force: true });
+ cy.get(
+ `[data-testid="lineage-node-${fromNode.fqn}"] [data-testid="expand-cols-btn"]`
+ ).click({ force: true });
+ cy.get(
+ `[data-testid="lineage-node-${fromNode.fqn}"] [data-testid="show-more-cols-btn"]`
+ ).click({ force: true });
+ cy.get('.react-flow__controls-fitview').click({ force: true });
+ cy.get(
+ `[data-testid="lineage-node-${toNode.fqn}"] [data-testid="expand-cols-btn"]`
+ ).click({ force: true });
+ cy.get(
+ `[data-testid="lineage-node-${toNode.fqn}"] [data-testid="show-more-cols-btn"]`
+ ).click({ force: true });
+ cy.get('.react-flow__controls-fitview').click({ force: true });
+
+ dragConnection(
+ `column-${fromNode.columns[0]}`,
+ `column-${toNode.columns[0]}`
+ );
+ verifyResponseStatusCode('@lineageApi', 200);
+ cy.get('[data-testid="edit-lineage"]').click();
+ cy.get(
+ `[data-testid="column-edge-${fromNode.columns[0]}-${toNode.columns[0]}"]`
+ );
+};
+
+describe('Lineage verification', () => {
beforeEach(() => {
cy.login();
});
@@ -134,4 +211,30 @@ describe('Entity Details Page', () => {
cy.get('[data-testid="edit-lineage"]').click();
});
});
+
+ it('Lineage Add Pipeline Between Tables', () => {
+ const sourceEntity = LINEAGE_ITEMS[0];
+ const targetEntity = LINEAGE_ITEMS[1];
+ addPipelineBetweenNodes(sourceEntity, targetEntity, PIPELINE_ITEMS[0]);
+ cy.get('[data-testid="edit-lineage"]').click();
+ deleteNode(targetEntity);
+ });
+
+ it('Lineage Add Pipeline Between Table and Topic', () => {
+ const sourceEntity = LINEAGE_ITEMS[1];
+ const targetEntity = LINEAGE_ITEMS[2];
+ addPipelineBetweenNodes(sourceEntity, targetEntity, PIPELINE_ITEMS[0]);
+ cy.get('[data-testid="edit-lineage"]').click();
+ deleteNode(targetEntity);
+ });
+
+ it('Add column lineage', () => {
+ const sourceEntity = LINEAGE_ITEMS[0];
+ const targetEntity = LINEAGE_ITEMS[1];
+ addPipelineBetweenNodes(sourceEntity, targetEntity);
+ // Add column lineage
+ addColumnLineage(sourceEntity, targetEntity);
+ cy.get('[data-testid="edit-lineage"]').click();
+ deleteNode(targetEntity);
+ });
});
diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/postgres.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/postgres.spec.js
index baa8ce25fb52..b4b57780854e 100644
--- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/postgres.spec.js
+++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Service/postgres.spec.js
@@ -37,14 +37,14 @@ const clearQuery = 'select pg_stat_statements_reset()';
const selectQuery =
'SELECT * FROM sales.order_items oi INNER JOIN sales.orders o ON oi.order_id=o.order_id';
-// @ayush - Need to fix postgres ingestion issue
-describe.skip('Postgres Ingestion', () => {
+describe('Postgres Ingestion', () => {
beforeEach(() => {
cy.login();
});
it('Trigger select query', () => {
cy.postgreSQL(clearQuery);
+ cy.wait(500);
cy.postgreSQL(selectQuery);
});
@@ -132,7 +132,7 @@ describe.skip('Postgres Ingestion', () => {
cy.get('#root\\/filterCondition')
.scrollIntoView()
- .type(`s.query like '%%${tableName}%%'`);
+ .type(`lower(s.query) like '%%${tableName}%%'`);
cy.get('[data-testid="submit-btn"]')
.scrollIntoView()
.should('be.visible')
diff --git a/openmetadata-ui/src/main/resources/ui/package.json b/openmetadata-ui/src/main/resources/ui/package.json
index 0b227efd7c9c..0ea47b7f1457 100644
--- a/openmetadata-ui/src/main/resources/ui/package.json
+++ b/openmetadata-ui/src/main/resources/ui/package.json
@@ -34,7 +34,7 @@
"parse-conn-schema": "node parseConnectionSchema && rm -rf connTemp",
"parse-ingestion-schema": "node parseIngestionSchema && rm -rf ingestionTemp",
"js-antlr": "PWD=$(echo $PWD) antlr4 -Dlanguage=JavaScript -o src/generated/antlr \"$PWD\"/../../../../../openmetadata-spec/src/main/antlr4/org/openmetadata/schema/*.g4",
- "cypress:open": "CYPRESS_BASE_URL=http://localhost:8585 cypress open --e2e",
+ "cypress:open": "CYPRESS_BASE_URL=http://localhost:3000 cypress open --e2e",
"cypress:run": "CYPRESS_BASE_URL=http://localhost:8585 cypress run --config-file=cypress.config.ts",
"cypress:run:record": "cypress run --config-file=cypress.config.ts --record --parallel",
"i18n": "sync-i18n --files '**/locale/languages/*.json' --primary en-us --space 2 --fn",
@@ -235,4 +235,4 @@
"prosemirror-transform": "1.7.0",
"prosemirror-view": "1.28.2"
}
-}
+}
\ No newline at end of file
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx
index 48bea0630089..651cbc719705 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityInfoDrawer/EdgeInfoDrawer.component.tsx
@@ -190,7 +190,7 @@ const EdgeInfoDrawer = ({