diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index f6051c5010d1..51c8a6587f66 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -2613,6 +2613,25 @@ default List listAfter(ListFilter filter, int limit, String afterName, S return listAfter( getTableName(), filter.getQueryParams(), condition, condition, limit, afterName, afterId); } + + @ConnectionAwareSqlQuery( + value = + "SELECT json FROM table_entity " + + "WHERE JSON_SEARCH(JSON_EXTRACT(json, '$.tableConstraints[*].referredColumns'), " + + "'one', :fqn) IS NOT NULL", + connectionType = MYSQL) + @ConnectionAwareSqlQuery( + value = + "SELECT json " + + "FROM table_entity " + + "WHERE EXISTS (" + + " SELECT 1" + + " FROM jsonb_array_elements(json->'tableConstraints') AS constraints" + + " CROSS JOIN jsonb_array_elements_text(constraints->'referredColumns') AS referredColumn " + + " WHERE referredColumn LIKE :fqn" + + ")", + connectionType = POSTGRES) + List findRelatedTables(@Bind("fqn") String fqn); } interface StoredProcedureDAO extends EntityDAO { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index 05931f4ddaa8..da66bd862862 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -21,7 +21,6 @@ import java.util.Set; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; -import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.type.EntityReference; @@ -174,62 +173,95 @@ static List> getLineageData(EntityReference entity) { return data; } - static List> populateEntityRelationshipData(Table entity) { - List> constraints = new ArrayList<>(); - if (CommonUtil.nullOrEmpty(entity.getTableConstraints())) { - return constraints; - } - for (TableConstraint tableConstraint : entity.getTableConstraints()) { - if (!tableConstraint - .getConstraintType() - .value() - .equalsIgnoreCase(TableConstraint.ConstraintType.FOREIGN_KEY.value())) { - continue; - } - for (String referredColumn : tableConstraint.getReferredColumns()) { - String relatedEntityFQN = getParentFQN(referredColumn); - Table relatedEntity; - try { - relatedEntity = getEntityByName(Entity.TABLE, relatedEntityFQN, "*", NON_DELETED); - IndexMapping destinationIndexMapping = - Entity.getSearchRepository() - .getIndexMapping(relatedEntity.getEntityReference().getType()); - String destinationIndexName = - destinationIndexMapping.getIndexName(Entity.getSearchRepository().getClusterAlias()); - Map relationshipsMap = buildRelationshipsMap(entity, relatedEntity); - int relatedEntityIndex = - checkRelatedEntity(relatedEntity.getFullyQualifiedName(), constraints); - if (relatedEntityIndex >= 0) { - updateExistingConstraint( - entity, - tableConstraint, - constraints.get(relatedEntityIndex), - destinationIndexName, - relatedEntity, - referredColumn); - } else { - addNewConstraint( - entity, - tableConstraint, - constraints, - relationshipsMap, - destinationIndexName, - relatedEntity, - referredColumn); + private static void processConstraints( + Table entity, + Table relatedEntity, + List> constraints, + Boolean updateForeignTableIndex) { + if (!nullOrEmpty(entity.getTableConstraints())) { + for (TableConstraint tableConstraint : entity.getTableConstraints()) { + if (!tableConstraint + .getConstraintType() + .value() + .equalsIgnoreCase(TableConstraint.ConstraintType.FOREIGN_KEY.value())) { + continue; + } + int columnIndex = 0; + for (String referredColumn : tableConstraint.getReferredColumns()) { + String relatedEntityFQN = getParentFQN(referredColumn); + String destinationIndexName = null; + try { + if (updateForeignTableIndex) { + relatedEntity = getEntityByName(Entity.TABLE, relatedEntityFQN, "*", NON_DELETED); + IndexMapping destinationIndexMapping = + Entity.getSearchRepository() + .getIndexMapping(relatedEntity.getEntityReference().getType()); + destinationIndexName = + destinationIndexMapping.getIndexName( + Entity.getSearchRepository().getClusterAlias()); + } + Map relationshipsMap = buildRelationshipsMap(entity, relatedEntity); + int relatedEntityIndex = + checkRelatedEntity( + entity.getFullyQualifiedName(), + relatedEntity.getFullyQualifiedName(), + constraints); + if (relatedEntityIndex >= 0) { + updateExistingConstraint( + entity, + tableConstraint, + constraints.get(relatedEntityIndex), + destinationIndexName, + relatedEntity, + referredColumn, + columnIndex, + updateForeignTableIndex); + } else { + addNewConstraint( + entity, + tableConstraint, + constraints, + relationshipsMap, + destinationIndexName, + relatedEntity, + referredColumn, + columnIndex, + updateForeignTableIndex); + } + columnIndex++; + } catch (EntityNotFoundException ex) { } - } catch (EntityNotFoundException ex) { } } } + } + + static List> populateEntityRelationshipData(Table entity) { + List> constraints = new ArrayList<>(); + processConstraints(entity, null, constraints, true); + + // We need to query the table_entity table to find the references this current table + // has with other tables. We pick this info from the ES however in case of re-indexing this info + // needs to be picked from the db + CollectionDAO dao = Entity.getCollectionDAO(); + List json_array = + dao.tableDAO().findRelatedTables(entity.getFullyQualifiedName() + "%"); + for (String json : json_array) { + Table foreign_table = JsonUtils.readValue(json, Table.class); + processConstraints(foreign_table, entity, constraints, false); + } return constraints; } - static int checkRelatedEntity(String relatedEntityFQN, List> constraints) { + static int checkRelatedEntity( + String entityFQN, String relatedEntityFQN, List> constraints) { int index = 0; for (Map constraint : constraints) { Map relatedConstraintEntity = (Map) constraint.get("relatedEntity"); - if (relatedConstraintEntity.get("fqn").equals(relatedEntityFQN)) { + Map constraintEntity = (Map) constraint.get("entity"); + if (relatedConstraintEntity.get("fqn").equals(relatedEntityFQN) + && constraintEntity.get("fqn").equals(entityFQN)) { return index; } index++; @@ -259,23 +291,23 @@ private static void updateExistingConstraint( Map presentConstraint, String destinationIndexName, Table relatedEntity, - String referredColumn) { - for (String currentColumn : tableConstraint.getColumns()) { - if (currentColumn.equals(FullyQualifiedName.getColumnName(referredColumn))) { - String columnFQN = FullyQualifiedName.add(entity.getFullyQualifiedName(), currentColumn); - - Map columnMap = new HashMap<>(); - columnMap.put("columnFQN", columnFQN); - columnMap.put("relatedColumnFQN", referredColumn); - columnMap.put("relationshipType", tableConstraint.getRelationshipType()); - - List> presentColumns = - (List>) presentConstraint.get("columns"); - presentColumns.add(columnMap); - - updateRelatedEntityIndex(destinationIndexName, relatedEntity, presentConstraint); - break; - } + String referredColumn, + int columnIndex, + Boolean updateForeignTableIndex) { + String columnFQN = + FullyQualifiedName.add( + entity.getFullyQualifiedName(), tableConstraint.getColumns().get(columnIndex)); + + Map columnMap = new HashMap<>(); + columnMap.put("columnFQN", columnFQN); + columnMap.put("relatedColumnFQN", referredColumn); + columnMap.put("relationshipType", tableConstraint.getRelationshipType()); + + List> presentColumns = + (List>) presentConstraint.get("columns"); + presentColumns.add(columnMap); + if (updateForeignTableIndex) { + updateRelatedEntityIndex(destinationIndexName, relatedEntity, presentConstraint); } } @@ -286,22 +318,23 @@ private static void addNewConstraint( Map relationshipsMap, String destinationIndexName, Table relatedEntity, - String referredColumn) { - for (String currentColumn : tableConstraint.getColumns()) { - if (currentColumn.equals(FullyQualifiedName.getColumnName(referredColumn))) { - List> columns = new ArrayList<>(); - String columnFQN = FullyQualifiedName.add(entity.getFullyQualifiedName(), currentColumn); - - Map columnMap = new HashMap<>(); - columnMap.put("columnFQN", columnFQN); - columnMap.put("relatedColumnFQN", referredColumn); - columnMap.put("relationshipType", tableConstraint.getRelationshipType()); - columns.add(columnMap); - relationshipsMap.put("columns", columns); - constraints.add(JsonUtils.getMap(relationshipsMap)); - - updateRelatedEntityIndex(destinationIndexName, relatedEntity, relationshipsMap); - } + String referredColumn, + int columnIndex, + Boolean updateForeignTableIndex) { + List> columns = new ArrayList<>(); + String columnFQN = + FullyQualifiedName.add( + entity.getFullyQualifiedName(), tableConstraint.getColumns().get(columnIndex)); + + Map columnMap = new HashMap<>(); + columnMap.put("columnFQN", columnFQN); + columnMap.put("relatedColumnFQN", referredColumn); + columnMap.put("relationshipType", tableConstraint.getRelationshipType()); + columns.add(columnMap); + relationshipsMap.put("columns", columns); + constraints.add(JsonUtils.getMap(relationshipsMap)); + if (updateForeignTableIndex) { + updateRelatedEntityIndex(destinationIndexName, relatedEntity, relationshipsMap); } }