Skip to content

Commit

Permalink
#18677 - Added Lineage Field back to SearchLineage (#18687)
Browse files Browse the repository at this point in the history 
* fix: add lineage field back to searchLineage

* add lineage tests

---------

Co-authored-by: karanh37 <karanh37@gmail.com>
  • Loading branch information
@TeddyCr @karanh37 @harshach
2 people authored and harshach committed Nov 24, 2024
1 parent 93ef523 commit 4d9ac01
Show file tree
Hide file tree
Showing 6 changed files with 174 additions and 41 deletions.
3 changes: 0 additions & 3 deletions ...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -783,7 +783,6 @@ public Map<String, Object> searchLineageInternal(
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
List<String> sourceFieldsToExcludeCopy = new ArrayList<>(SOURCE_FIELDS_TO_EXCLUDE);
sourceFieldsToExcludeCopy.add("lineage");
searchSourceBuilder.fetchSource(null, sourceFieldsToExcludeCopy.toArray(String[]::new));
searchSourceBuilder.query(
QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn)));
Expand Down Expand Up @@ -1059,7 +1058,6 @@ private void getLineage(
List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.remove("lineage");
nodes.add(tempMap);
for (Map<String, Object> lin : lineage) {
Map<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");
Expand Down Expand Up @@ -1249,7 +1247,6 @@ private Map<String, Object> searchPipelineLineage(
List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.remove("lineage");
nodes.add(tempMap);
for (Map<String, Object> lin : lineage) {
HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");
Expand Down
3 changes: 0 additions & 3 deletions ...ta-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -778,7 +778,6 @@ public Map<String, Object> searchLineageInternal(
Entity.getSearchRepository().getIndexOrAliasName(GLOBAL_SEARCH_ALIAS));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
List<String> sourceFieldsToExcludeCopy = new ArrayList<>(SOURCE_FIELDS_TO_EXCLUDE);
sourceFieldsToExcludeCopy.add("lineage");
searchSourceBuilder.fetchSource(null, sourceFieldsToExcludeCopy.toArray(String[]::new));
searchSourceBuilder.query(
QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fullyQualifiedName", fqn)));
Expand Down Expand Up @@ -1055,7 +1054,6 @@ private void getLineage(
List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.remove("lineage");
nodes.add(tempMap);
for (Map<String, Object> lin : lineage) {
HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");
Expand Down Expand Up @@ -1229,7 +1227,6 @@ private Map<String, Object> searchPipelineLineage(
List<Map<String, Object>> lineage =
(List<Map<String, Object>>) hit.getSourceAsMap().get("lineage");
HashMap<String, Object> tempMap = new HashMap<>(JsonUtils.getMap(hit.getSourceAsMap()));
tempMap.remove("lineage");
nodes.add(tempMap);
for (Map<String, Object> lin : lineage) {
HashMap<String, String> fromEntity = (HashMap<String, String>) lin.get("fromEntity");
Expand Down
80 changes: 80 additions & 0 deletions ...service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,14 @@
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Response.Status;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.client.HttpResponseException;
import org.junit.jupiter.api.BeforeAll;
Expand Down Expand Up @@ -569,6 +572,42 @@ void get_dataQualityLineage(TestInfo test)
deleteEdge(TABLES.get(6), TABLES.get(7));
}

@Order(7)
@Test
void get_SearchLineage(TestInfo testInfo) throws HttpResponseException {
// our lineage is
// 0
// +-----+-----+
// v v
// 2 4
// +---+---+ v
// v | 5
// 1 | v
// | 6
// | v
// +-----> 7

addEdge(TABLES.get(4), TABLES.get(5));
addEdge(TABLES.get(5), TABLES.get(6));
addEdge(TABLES.get(0), TABLES.get(4));
addEdge(TABLES.get(0), TABLES.get(2));
addEdge(TABLES.get(2), TABLES.get(1));
addEdge(TABLES.get(2), TABLES.get(7));
addEdge(TABLES.get(6), TABLES.get(7));

Map<String, List<Map<String, Object>>> entity =
searchLineage(TABLES.get(5).getEntityReference(), 1, 1);
assertSearchLineageResponseFields(entity);

deleteEdge(TABLES.get(4), TABLES.get(5));
deleteEdge(TABLES.get(5), TABLES.get(6));
deleteEdge(TABLES.get(0), TABLES.get(4));
deleteEdge(TABLES.get(0), TABLES.get(2));
deleteEdge(TABLES.get(2), TABLES.get(1));
deleteEdge(TABLES.get(2), TABLES.get(7));
deleteEdge(TABLES.get(6), TABLES.get(7));
}

public Edge getEdge(Table from, Table to) {
return getEdge(from.getId(), to.getId(), null);
}
Expand Down Expand Up @@ -738,6 +777,32 @@ public void assertLineage(
assertEquals(lineageById, lineageByName);
}

private void assertSearchLineageResponseFields(Map<String, List<Map<String, Object>>> entity) {
List<Map<String, Object>> entities = entity.get("nodes");
Set<String> nodesFields = Set.of("id", "name", "displayName", "fullyQualifiedName", "lineage");
Set<String> nodesColumnsFields = Set.of("name", "fullyQualifiedName");
entities.forEach(
e -> {
Set<String> keys = e.keySet();
Set<String> missingKeys = new HashSet<>(nodesFields);
missingKeys.removeAll(keys);
String err = String.format("Nodes keys not found in the response: %s", missingKeys);
assertTrue(keys.containsAll(nodesFields), err);

List<Map<String, Object>> columns = (List<Map<String, Object>>) e.get("columns");
columns.forEach(
c -> {
Set<String> columnsKeys = c.keySet();
Set<String> missingColumnKeys = new HashSet<>(nodesColumnsFields);
missingColumnKeys.removeAll(columnsKeys);
String columnErr =
String.format(
"Column nodes keys not found in the response: %s", missingColumnKeys);
assertTrue(columnsKeys.containsAll(nodesColumnsFields), columnErr);
});
});
}

public EntityLineage getLineage(
String entity,
UUID id,
Expand All @@ -754,6 +819,21 @@ public EntityLineage getLineage(
return lineage;
}

public Map<String, List<Map<String, Object>>> searchLineage(
@NonNull EntityReference entityReference,
@NonNull int upstreamDepth,
@NonNull int downstreamDepth)
throws HttpResponseException {
WebTarget target = getResource("lineage/getLineage");
target = target.queryParam("fqn", entityReference.getFullyQualifiedName());
target = target.queryParam("type", entityReference.getType());
target = target.queryParam("upstreamDepth", upstreamDepth);
target = target.queryParam("downstreamDepth", downstreamDepth);
Map<String, List<Map<String, Object>>> entity =
TestUtils.get(target, Map.class, ADMIN_AUTH_HEADERS);
return entity;
}

public EntityLineage getLineageByName(
String entity,
String fqn,
Expand Down
98 changes: 65 additions & 33 deletions openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Lineage.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import {
deleteNode,
editLineage,
fillLineageConfigForm,
performExpand,
performZoomOut,
removeColumnLineage,
setupEntitiesForLineage,
Expand Down Expand Up @@ -374,57 +375,88 @@ test('Verify global lineage config', async ({ browser }) => {
const topic = new TopicClass();
const dashboard = new DashboardClass();
const mlModel = new MlModelClass();
const searchIndex = new SearchIndexClass();

try {
await table.create(apiContext);
await topic.create(apiContext);
await dashboard.create(apiContext);
await mlModel.create(apiContext);
await searchIndex.create(apiContext);

await addPipelineBetweenNodes(page, table, topic);
await addPipelineBetweenNodes(page, topic, dashboard);
await addPipelineBetweenNodes(page, dashboard, mlModel);

await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG);
await fillLineageConfigForm(page, {
upstreamDepth: 1,
downstreamDepth: 1,
layer: 'Column Level Lineage',
});

await topic.visitEntityPage(page);
await visitLineageTab(page);

await verifyNodePresent(page, table);
await verifyNodePresent(page, dashboard);

const mlModelFqn = get(mlModel, 'entityResponseData.fullyQualifiedName');
const mlModelNode = page.locator(
`[data-testid="lineage-node-${mlModelFqn}"]`
await addPipelineBetweenNodes(page, mlModel, searchIndex);

await test.step(
'Update global lineage config and verify lineage',
async () => {
await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG);
await fillLineageConfigForm(page, {
upstreamDepth: 1,
downstreamDepth: 1,
layer: 'Column Level Lineage',
});

await topic.visitEntityPage(page);
await visitLineageTab(page);
await verifyNodePresent(page, table);
await verifyNodePresent(page, dashboard);
const mlModelFqn = get(
mlModel,
'entityResponseData.fullyQualifiedName'
);
const mlModelNode = page.locator(
`[data-testid="lineage-node-${mlModelFqn}"]`
);

await expect(mlModelNode).not.toBeVisible();

await verifyColumnLayerActive(page);
}
);

await expect(mlModelNode).not.toBeVisible();

await verifyColumnLayerActive(page);

await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG);
await fillLineageConfigForm(page, {
upstreamDepth: 2,
downstreamDepth: 2,
layer: 'Entity Lineage',
});

await topic.visitEntityPage(page);
await visitLineageTab(page);
await test.step(
'Verify Upstream and Downstream expand collapse buttons',
async () => {
await dashboard.visitEntityPage(page);
await visitLineageTab(page);
await page.getByTestId('entity-panel-close-icon').click();
await performZoomOut(page);
await verifyNodePresent(page, topic);
await verifyNodePresent(page, mlModel);
await performExpand(page, mlModel, false, searchIndex);
await performExpand(page, topic, true);
}
);

await verifyNodePresent(page, table);
await verifyNodePresent(page, dashboard);
await verifyNodePresent(page, mlModel);
await test.step(
'Reset global lineage config and verify lineage',
async () => {
await settingClick(page, GlobalSettingOptions.LINEAGE_CONFIG);
await fillLineageConfigForm(page, {
upstreamDepth: 2,
downstreamDepth: 2,
layer: 'Entity Lineage',
});

await dashboard.visitEntityPage(page);
await visitLineageTab(page);

await verifyNodePresent(page, table);
await verifyNodePresent(page, dashboard);
await verifyNodePresent(page, mlModel);
await verifyNodePresent(page, searchIndex);
await verifyNodePresent(page, topic);
}
);
} finally {
await table.delete(apiContext);
await topic.delete(apiContext);
await dashboard.delete(apiContext);
await mlModel.delete(apiContext);
await searchIndex.delete(apiContext);

await afterAction();
}
Expand Down
23 changes: 23 additions & 0 deletions openmetadata-ui/src/main/resources/ui/playwright/utils/lineage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,29 @@ export const connectEdgeBetweenNodes = async (
);
};

export const performExpand = async (
page: Page,
node: EntityClass,
upstream: boolean,
newNode?: EntityClass
) => {
const nodeFqn = get(node, 'entityResponseData.fullyQualifiedName');
const handleDirection = upstream ? 'left' : 'right';
const expandBtn = page
.locator(`[data-testid="lineage-node-${nodeFqn}"]`)
.locator(`.react-flow__handle-${handleDirection}`)
.getByTestId('plus-icon');

if (newNode) {
const expandRes = page.waitForResponse('/api/v1/lineage/getLineage?*');
await expandBtn.click();
await expandRes;
await verifyNodePresent(page, newNode);
} else {
await expect(expandBtn).toBeVisible();
}
};

export const verifyNodePresent = async (page: Page, node: EntityClass) => {
const nodeFqn = get(node, 'entityResponseData.fullyQualifiedName');
const name = get(node, 'entityResponseData.name');
Expand Down
8 changes: 6 additions & 2 deletions ...etadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/CustomNode.utils.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,9 @@ export const getExpandHandle = (
? 'react-flow__handle-right'
: 'react-flow__handle-left'
)}
icon={<PlusIcon className="lineage-expand-icon" />}
icon={
<PlusIcon className="lineage-expand-icon" data-testid="plus-icon" />
}
shape="circle"
size="small"
onClick={(e) => {
Expand Down Expand Up @@ -106,7 +108,9 @@ export const getCollapseHandle = (
? 'downstream-collapse-handle'
: 'upstream-collapse-handle'
}
icon={<MinusIcon className="lineage-expand-icon" />}
icon={
<MinusIcon className="lineage-expand-icon " data-testid="minus-icon" />
}
shape="circle"
size="small"
onClick={(e) => {
Expand Down

0 comments on commit 4d9ac01

Please sign in to comment.