Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ER Diagrams]: Fix EntityRelationship data disappeared after reindexing #18477

Merged
merged 2 commits into from
Nov 6, 2024
Merged

[ER Diagrams]: Fix EntityRelationship data disappeared after reindexing #18477

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f59aa3a
fix er diag reindex
OnkarVO7 Oct 30, 2024
c022fed
fix col index mapping
OnkarVO7 Oct 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2613,6 +2613,25 @@ default List<String> listAfter(ListFilter filter, int limit, String afterName, S
return listAfter(
getTableName(), filter.getQueryParams(), condition, condition, limit, afterName, afterId);
}

@ConnectionAwareSqlQuery(
value =
"SELECT json FROM table_entity "
+ "WHERE JSON_SEARCH(JSON_EXTRACT(json, '$.tableConstraints[*].referredColumns'), "
+ "'one', :fqn) IS NOT NULL",
connectionType = MYSQL)
@ConnectionAwareSqlQuery(
value =
"SELECT json "
+ "FROM table_entity "
+ "WHERE EXISTS ("
+ " SELECT 1"
+ " FROM jsonb_array_elements(json->'tableConstraints') AS constraints"
+ " CROSS JOIN jsonb_array_elements_text(constraints->'referredColumns') AS referredColumn "
+ " WHERE referredColumn LIKE :fqn"
+ ")",
connectionType = POSTGRES)
List<String> findRelatedTables(@Bind("fqn") String fqn);
}

interface StoredProcedureDAO extends EntityDAO<StoredProcedure> {
Expand Down
191 changes: 112 additions & 79 deletions openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import java.util.Set;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.openmetadata.common.utils.CommonUtil;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.entity.data.Table;
import org.openmetadata.schema.type.EntityReference;
Expand Down Expand Up @@ -174,62 +173,95 @@ static List<Map<String, Object>> getLineageData(EntityReference entity) {
return data;
}

static List<Map<String, Object>> populateEntityRelationshipData(Table entity) {
List<Map<String, Object>> constraints = new ArrayList<>();
if (CommonUtil.nullOrEmpty(entity.getTableConstraints())) {
return constraints;
}
for (TableConstraint tableConstraint : entity.getTableConstraints()) {
if (!tableConstraint
.getConstraintType()
.value()
.equalsIgnoreCase(TableConstraint.ConstraintType.FOREIGN_KEY.value())) {
continue;
}
for (String referredColumn : tableConstraint.getReferredColumns()) {
String relatedEntityFQN = getParentFQN(referredColumn);
Table relatedEntity;
try {
relatedEntity = getEntityByName(Entity.TABLE, relatedEntityFQN, "*", NON_DELETED);
IndexMapping destinationIndexMapping =
Entity.getSearchRepository()
.getIndexMapping(relatedEntity.getEntityReference().getType());
String destinationIndexName =
destinationIndexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias());
Map<String, Object> relationshipsMap = buildRelationshipsMap(entity, relatedEntity);
int relatedEntityIndex =
checkRelatedEntity(relatedEntity.getFullyQualifiedName(), constraints);
if (relatedEntityIndex >= 0) {
updateExistingConstraint(
entity,
tableConstraint,
constraints.get(relatedEntityIndex),
destinationIndexName,
relatedEntity,
referredColumn);
} else {
addNewConstraint(
entity,
tableConstraint,
constraints,
relationshipsMap,
destinationIndexName,
relatedEntity,
referredColumn);
private static void processConstraints(
Table entity,
Table relatedEntity,
List<Map<String, Object>> constraints,
Boolean updateForeignTableIndex) {
if (!nullOrEmpty(entity.getTableConstraints())) {
for (TableConstraint tableConstraint : entity.getTableConstraints()) {
if (!tableConstraint
.getConstraintType()
.value()
.equalsIgnoreCase(TableConstraint.ConstraintType.FOREIGN_KEY.value())) {
continue;
}
int columnIndex = 0;
for (String referredColumn : tableConstraint.getReferredColumns()) {
String relatedEntityFQN = getParentFQN(referredColumn);
String destinationIndexName = null;
try {
if (updateForeignTableIndex) {
relatedEntity = getEntityByName(Entity.TABLE, relatedEntityFQN, "*", NON_DELETED);
IndexMapping destinationIndexMapping =
Entity.getSearchRepository()
.getIndexMapping(relatedEntity.getEntityReference().getType());
destinationIndexName =
destinationIndexMapping.getIndexName(
Entity.getSearchRepository().getClusterAlias());
}
Map<String, Object> relationshipsMap = buildRelationshipsMap(entity, relatedEntity);
int relatedEntityIndex =
checkRelatedEntity(
entity.getFullyQualifiedName(),
relatedEntity.getFullyQualifiedName(),
constraints);
if (relatedEntityIndex >= 0) {
updateExistingConstraint(
entity,
tableConstraint,
constraints.get(relatedEntityIndex),
destinationIndexName,
relatedEntity,
referredColumn,
columnIndex,
updateForeignTableIndex);
} else {
addNewConstraint(
entity,
tableConstraint,
constraints,
relationshipsMap,
destinationIndexName,
relatedEntity,
referredColumn,
columnIndex,
updateForeignTableIndex);
}
columnIndex++;
} catch (EntityNotFoundException ex) {
}
} catch (EntityNotFoundException ex) {
}
}
}
}

static List<Map<String, Object>> populateEntityRelationshipData(Table entity) {
List<Map<String, Object>> constraints = new ArrayList<>();
processConstraints(entity, null, constraints, true);

// We need to query the table_entity table to find the references this current table
// has with other tables. We pick this info from the ES however in case of re-indexing this info
// needs to be picked from the db
CollectionDAO dao = Entity.getCollectionDAO();
List<String> json_array =
dao.tableDAO().findRelatedTables(entity.getFullyQualifiedName() + "%");
for (String json : json_array) {
Table foreign_table = JsonUtils.readValue(json, Table.class);
processConstraints(foreign_table, entity, constraints, false);
}
return constraints;
}

static int checkRelatedEntity(String relatedEntityFQN, List<Map<String, Object>> constraints) {
static int checkRelatedEntity(
String entityFQN, String relatedEntityFQN, List<Map<String, Object>> constraints) {
int index = 0;
for (Map<String, Object> constraint : constraints) {
Map<String, Object> relatedConstraintEntity =
(Map<String, Object>) constraint.get("relatedEntity");
if (relatedConstraintEntity.get("fqn").equals(relatedEntityFQN)) {
Map<String, Object> constraintEntity = (Map<String, Object>) constraint.get("entity");
if (relatedConstraintEntity.get("fqn").equals(relatedEntityFQN)
&& constraintEntity.get("fqn").equals(entityFQN)) {
return index;
}
index++;
Expand Down Expand Up @@ -259,23 +291,23 @@ private static void updateExistingConstraint(
Map<String, Object> presentConstraint,
String destinationIndexName,
Table relatedEntity,
String referredColumn) {
for (String currentColumn : tableConstraint.getColumns()) {
if (currentColumn.equals(FullyQualifiedName.getColumnName(referredColumn))) {
String columnFQN = FullyQualifiedName.add(entity.getFullyQualifiedName(), currentColumn);

Map<String, Object> columnMap = new HashMap<>();
columnMap.put("columnFQN", columnFQN);
columnMap.put("relatedColumnFQN", referredColumn);
columnMap.put("relationshipType", tableConstraint.getRelationshipType());

List<Map<String, Object>> presentColumns =
(List<Map<String, Object>>) presentConstraint.get("columns");
presentColumns.add(columnMap);

updateRelatedEntityIndex(destinationIndexName, relatedEntity, presentConstraint);
break;
}
String referredColumn,
int columnIndex,
Boolean updateForeignTableIndex) {
String columnFQN =
FullyQualifiedName.add(
entity.getFullyQualifiedName(), tableConstraint.getColumns().get(columnIndex));

Map<String, Object> columnMap = new HashMap<>();
columnMap.put("columnFQN", columnFQN);
columnMap.put("relatedColumnFQN", referredColumn);
columnMap.put("relationshipType", tableConstraint.getRelationshipType());

List<Map<String, Object>> presentColumns =
(List<Map<String, Object>>) presentConstraint.get("columns");
presentColumns.add(columnMap);
if (updateForeignTableIndex) {
updateRelatedEntityIndex(destinationIndexName, relatedEntity, presentConstraint);
}
}

Expand All @@ -286,22 +318,23 @@ private static void addNewConstraint(
Map<String, Object> relationshipsMap,
String destinationIndexName,
Table relatedEntity,
String referredColumn) {
for (String currentColumn : tableConstraint.getColumns()) {
if (currentColumn.equals(FullyQualifiedName.getColumnName(referredColumn))) {
List<Map<String, Object>> columns = new ArrayList<>();
String columnFQN = FullyQualifiedName.add(entity.getFullyQualifiedName(), currentColumn);

Map<String, Object> columnMap = new HashMap<>();
columnMap.put("columnFQN", columnFQN);
columnMap.put("relatedColumnFQN", referredColumn);
columnMap.put("relationshipType", tableConstraint.getRelationshipType());
columns.add(columnMap);
relationshipsMap.put("columns", columns);
constraints.add(JsonUtils.getMap(relationshipsMap));

updateRelatedEntityIndex(destinationIndexName, relatedEntity, relationshipsMap);
}
String referredColumn,
int columnIndex,
Boolean updateForeignTableIndex) {
List<Map<String, Object>> columns = new ArrayList<>();
String columnFQN =
FullyQualifiedName.add(
entity.getFullyQualifiedName(), tableConstraint.getColumns().get(columnIndex));

Map<String, Object> columnMap = new HashMap<>();
columnMap.put("columnFQN", columnFQN);
columnMap.put("relatedColumnFQN", referredColumn);
columnMap.put("relationshipType", tableConstraint.getRelationshipType());
columns.add(columnMap);
relationshipsMap.put("columns", columns);
constraints.add(JsonUtils.getMap(relationshipsMap));
if (updateForeignTableIndex) {
updateRelatedEntityIndex(destinationIndexName, relatedEntity, relationshipsMap);
}
}

Expand Down
Loading