diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 47a99887a6f9..a76c9052bf5a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -38,6 +38,7 @@ import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TagLabel; @@ -47,7 +48,6 @@ import org.openmetadata.schema.type.csv.CsvFile; import org.openmetadata.schema.type.csv.CsvHeader; import org.openmetadata.schema.type.csv.CsvImportResult; -import org.openmetadata.schema.type.csv.CsvImportResult.Status; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.util.EntityUtil; @@ -413,7 +413,7 @@ public static String invalidBoolean(int field, String fieldValue) { } private void documentFailure(String error) { - importResult.withStatus(Status.ABORTED); + importResult.withStatus(ApiStatus.ABORTED); importResult.withAbortReason(error); } @@ -435,11 +435,11 @@ protected void importFailure(CSVPrinter printer, String failedReason, CSVRecord } private void setFinalStatus() { - Status status = Status.FAILURE; + ApiStatus status = ApiStatus.FAILURE; if (importResult.getNumberOfRowsPassed().equals(importResult.getNumberOfRowsProcessed())) { - status = Status.SUCCESS; + status = ApiStatus.SUCCESS; } else if (importResult.getNumberOfRowsPassed() > 1) { - status = Status.PARTIAL_SUCCESS; + status = ApiStatus.PARTIAL_SUCCESS; } importResult.setStatus(status); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 28c0029f2d22..317c7e263fc1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -15,6 +15,7 @@ import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; import static org.openmetadata.service.util.EntityUtil.getFlattenedEntityField; import com.fasterxml.jackson.annotation.JsonPropertyOrder; @@ -539,11 +540,15 @@ public static void populateEntityFieldTags( List flattenedFields = getFlattenedEntityField(fields); // Fetch All tags belonging to Prefix - Map> allTags = repository.getTagsByPrefix(fqnPrefix); + Map> allTags = repository.getTagsByPrefix(fqnPrefix, ".%"); for (T c : listOrEmpty(flattenedFields)) { if (setTags) { List columnTag = allTags.get(FullyQualifiedName.buildHash(c.getFullyQualifiedName())); - c.setTags(columnTag == null ? new ArrayList<>() : columnTag); + if (columnTag == null) { + c.setTags(new ArrayList<>()); + } else { + c.setTags(addDerivedTags(columnTag)); + } } else { c.setTags(c.getTags()); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 4e9ba20d12c1..94a574c353e5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -2195,9 +2195,11 @@ default List getTags(String targetFQN) { return tags; } - default Map> getTagsByPrefix(String targetFQNPrefix) { + default Map> getTagsByPrefix( + String targetFQNPrefix, String postfix, boolean requiresFqnHash) { + String fqnHash = requiresFqnHash ? FullyQualifiedName.buildHash(targetFQNPrefix) : targetFQNPrefix; Map> resultSet = new LinkedHashMap<>(); - List> tags = getTagsInternalByPrefix(targetFQNPrefix); + List> tags = getTagsInternalByPrefix(fqnHash, postfix); tags.forEach( pair -> { String targetHash = pair.getLeft(); @@ -2234,7 +2236,7 @@ default Map> getTagsByPrefix(String targetFQNPrefix) { + " ON ta.fqnHash = tu.tagFQNHash " + " WHERE tu.source = 0 " + ") AS combined_data " - + "WHERE combined_data.targetFQNHash LIKE CONCAT(:targetFQNHashPrefix, '.%')", + + "WHERE combined_data.targetFQNHash LIKE CONCAT(:targetFQNHashPrefix, :postfix)", connectionType = MYSQL) @ConnectionAwareSqlQuery( value = @@ -2250,10 +2252,11 @@ default Map> getTagsByPrefix(String targetFQNPrefix) { + " JOIN tag_usage AS tu ON ta.fqnHash = tu.tagFQNHash " + " WHERE tu.source = 0 " + ") AS combined_data " - + "WHERE combined_data.targetFQNHash LIKE CONCAT(:targetFQNHashPrefix, '.%')", + + "WHERE combined_data.targetFQNHash LIKE CONCAT(:targetFQNHashPrefix, :postfix)", connectionType = POSTGRES) @RegisterRowMapper(TagLabelRowMapperWithTargetFqnHash.class) - List> getTagsInternalByPrefix(@BindFQN("targetFQNHashPrefix") String targetFQNHashPrefix); + List> getTagsInternalByPrefix( + @Bind("targetFQNHashPrefix") String targetFQNHashPrefix, @Bind("postfix") String postfix); @SqlQuery("SELECT * FROM tag_usage") @Deprecated(since = "Release 1.1") @@ -2269,6 +2272,11 @@ default Map> getTagsByPrefix(String targetFQNPrefix) { @SqlUpdate("DELETE FROM tag_usage where targetFQNHash = :targetFQNHash") void deleteTagsByTarget(@BindFQN("targetFQNHash") String targetFQNHash); + @SqlUpdate( + "DELETE FROM tag_usage where tagFQNHash = :tagFqnHash AND targetFQNHash LIKE CONCAT(:targetFQNHash, '%')") + void deleteTagsByTagAndTargetEntity( + @BindFQN("tagFqnHash") String tagFqnHash, @BindFQN("targetFQNHash") String targetFQNHash); + @SqlUpdate("DELETE FROM tag_usage where tagFQNHash = :tagFQNHash AND source = :source") void deleteTagLabels(@Bind("source") int source, @BindFQN("tagFQNHash") String tagFQNHash); @@ -2333,6 +2341,15 @@ void renameInternal( @SqlUpdate("") void updateTagPrefixInternal(@Define("update") String update); + @SqlQuery( + "select * from tag_usage where targetFQNHash in (select targetFQNHash FROM tag_usage where tagFQNHash = :tagFQNHash) and tagFQNHash != :tagFQNHash") + @RegisterRowMapper(TagLabelMapper.class) + List getEntityTagsFromTag(@BindFQN("tagFQNHash") String tagFQNHash); + + @SqlQuery("select targetFQNHash FROM tag_usage where tagFQNHash = :tagFQNHash") + @RegisterRowMapper(TagLabelMapper.class) + List getTargetFQNHashForTag(@BindFQN("tagFQNHash") String tagFQNHash); + class TagLabelMapper implements RowMapper { @Override public TagLabel map(ResultSet r, StatementContext ctx) throws SQLException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index ceec49be8dff..a51fa1bc90d6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -42,6 +42,8 @@ import static org.openmetadata.service.Entity.getEntityFields; import static org.openmetadata.service.exception.CatalogExceptionMessage.csvNotSupported; import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import static org.openmetadata.service.util.EntityUtil.compareTagLabel; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.service.util.EntityUtil.fieldAdded; @@ -69,7 +71,6 @@ import java.net.URI; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -1227,31 +1228,6 @@ public Object getExtension(T entity) { return objectNode; } - /** Validate given list of tags and add derived tags to it */ - public final List addDerivedTags(List tagLabels) { - if (nullOrEmpty(tagLabels)) { - return tagLabels; - } - - List updatedTagLabels = new ArrayList<>(); - EntityUtil.mergeTags(updatedTagLabels, tagLabels); - for (TagLabel tagLabel : tagLabels) { - EntityUtil.mergeTags(updatedTagLabels, getDerivedTags(tagLabel)); - } - updatedTagLabels.sort(compareTagLabel); - return updatedTagLabels; - } - - /** Get tags associated with a given set of tags */ - private List getDerivedTags(TagLabel tagLabel) { - if (tagLabel.getSource() == TagLabel.TagSource.GLOSSARY) { // Related tags are only supported for Glossary - List derivedTags = daoCollection.tagUsageDAO().getTags(tagLabel.getTagFQN()); - derivedTags.forEach(tag -> tag.setLabelType(TagLabel.LabelType.DERIVED)); - return derivedTags; - } - return Collections.emptyList(); - } - protected void applyColumnTags(List columns) { // Add column level tags by adding tag to column relationship for (Column column : columns) { @@ -1274,26 +1250,18 @@ protected void applyTags(T entity) { public void applyTags(List tagLabels, String targetFQN) { for (TagLabel tagLabel : listOrEmpty(tagLabels)) { // Apply tagLabel to targetFQN that identifies an entity or field - daoCollection - .tagUsageDAO() - .applyTag( - tagLabel.getSource().ordinal(), - tagLabel.getTagFQN(), - tagLabel.getTagFQN(), - targetFQN, - tagLabel.getLabelType().ordinal(), - tagLabel.getState().ordinal()); - } - } - - void checkMutuallyExclusive(List tagLabels) { - Map map = new HashMap<>(); - for (TagLabel tagLabel : listOrEmpty(tagLabels)) { - // When two tags have the same parent that is mutuallyExclusive, then throw an error - String parentFqn = FullyQualifiedName.getParentFQN(tagLabel.getTagFQN()); - TagLabel stored = map.put(parentFqn, tagLabel); - if (stored != null && TagLabelUtil.mutuallyExclusive(tagLabel)) { - throw new IllegalArgumentException(CatalogExceptionMessage.mutuallyExclusiveLabels(tagLabel, stored)); + boolean isTagDerived = tagLabel.getLabelType().equals(TagLabel.LabelType.DERIVED); + // Derived Tags should not create Relationships, and needs to be built on the during Read + if (!isTagDerived) { + daoCollection + .tagUsageDAO() + .applyTag( + tagLabel.getSource().ordinal(), + tagLabel.getTagFQN(), + tagLabel.getTagFQN(), + targetFQN, + tagLabel.getLabelType().ordinal(), + tagLabel.getState().ordinal()); } } } @@ -1303,11 +1271,16 @@ protected List getTags(T entity) { } protected List getTags(String fqn) { - return !supportsTags ? null : daoCollection.tagUsageDAO().getTags(fqn); + if (!supportsTags) { + return null; + } + + // Populate Glossary Tags on Read + return addDerivedTags(daoCollection.tagUsageDAO().getTags(fqn)); } - public Map> getTagsByPrefix(String prefix) { - return !supportsTags ? null : daoCollection.tagUsageDAO().getTagsByPrefix(prefix); + public Map> getTagsByPrefix(String prefix, String postfix) { + return !supportsTags ? null : daoCollection.tagUsageDAO().getTagsByPrefix(prefix, postfix, true); } protected List getFollowers(T entity) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 3fe071ec7409..39ff4a50c538 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -23,20 +23,29 @@ import static org.openmetadata.service.Entity.GLOSSARY_TERM; import static org.openmetadata.service.exception.CatalogExceptionMessage.invalidGlossaryTermMove; import static org.openmetadata.service.exception.CatalogExceptionMessage.notReviewer; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusiveForParentAndSubField; +import static org.openmetadata.service.resources.tags.TagLabelUtil.getUniqueTags; +import static org.openmetadata.service.util.EntityUtil.compareTagLabel; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.service.util.EntityUtil.getId; import static org.openmetadata.service.util.EntityUtil.stringMatch; +import static org.openmetadata.service.util.EntityUtil.tagLabelMatch; import static org.openmetadata.service.util.EntityUtil.termReferenceMatch; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.UUID; import javax.json.JsonPatch; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.ImmutablePair; import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.api.AddGlossaryToAssetsRequest; import org.openmetadata.schema.api.data.TermReference; import org.openmetadata.schema.api.feed.CloseTask; import org.openmetadata.schema.api.feed.ResolveTask; @@ -44,15 +53,19 @@ import org.openmetadata.schema.entity.data.GlossaryTerm; import org.openmetadata.schema.entity.data.GlossaryTerm.Status; import org.openmetadata.schema.entity.feed.Thread; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.ProviderType; import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TagLabel.TagSource; import org.openmetadata.schema.type.TaskDetails; import org.openmetadata.schema.type.TaskStatus; import org.openmetadata.schema.type.TaskType; import org.openmetadata.schema.type.ThreadType; +import org.openmetadata.schema.type.api.BulkOperationResult; +import org.openmetadata.schema.type.api.FailureRequest; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; @@ -193,6 +206,122 @@ public void setFullyQualifiedName(GlossaryTerm entity) { } } + public BulkOperationResult bulkAddAndValidateGlossaryToAssets( + UUID glossaryTermId, AddGlossaryToAssetsRequest request) { + boolean dryRun = Boolean.TRUE.equals(request.getDryRun()); + + GlossaryTerm term = this.get(null, glossaryTermId, getFields("id,tags")); + + // Check if the tags are mutually exclusive for the glossary + checkMutuallyExclusive(request.getGlossaryTags()); + + BulkOperationResult result = new BulkOperationResult().withDryRun(dryRun); + List failures = new ArrayList<>(); + List success = new ArrayList<>(); + + if (dryRun && (CommonUtil.nullOrEmpty(request.getGlossaryTags()) || CommonUtil.nullOrEmpty(request.getAssets()))) { + // Nothing to Validate + return result.withStatus(ApiStatus.SUCCESS).withSuccessRequest("Nothing to Validate."); + } + + // Validation for entityReferences + EntityUtil.populateEntityReferences(request.getAssets()); + + TagLabel tagLabel = + new TagLabel() + .withTagFQN(term.getFullyQualifiedName()) + .withSource(TagSource.GLOSSARY) + .withLabelType(TagLabel.LabelType.MANUAL); + + for (EntityReference ref : request.getAssets()) { + // Update Result Processed + result.setNumberOfRowsProcessed(result.getNumberOfRowsProcessed() + 1); + + EntityRepository entityRepository = Entity.getEntityRepository(ref.getType()); + EntityInterface asset = entityRepository.get(null, ref.getId(), entityRepository.getFields("tags")); + + try { + Map> allAssetTags = + daoCollection.tagUsageDAO().getTagsByPrefix(asset.getFullyQualifiedName(), "%", true); + checkMutuallyExclusiveForParentAndSubField( + asset.getFullyQualifiedName(), + FullyQualifiedName.buildHash(asset.getFullyQualifiedName()), + allAssetTags, + request.getGlossaryTags(), + false); + success.add(ref); + result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); + } catch (Exception ex) { + failures.add(new FailureRequest().withRequest(ref).withError(ex.getMessage())); + result.withFailedRequest(failures); + result.setNumberOfRowsFailed(result.getNumberOfRowsFailed() + 1); + } + // Validate and Store Tags + if (!dryRun && CommonUtil.nullOrEmpty(result.getFailedRequest())) { + List tempList = new ArrayList<>(asset.getTags()); + tempList.add(tagLabel); + // Apply Tags to Entities + entityRepository.applyTags(getUniqueTags(tempList), asset.getFullyQualifiedName()); + + searchRepository.updateEntity(ref); + } + } + + // Apply the tags of glossary to the glossary term + if (!dryRun + && CommonUtil.nullOrEmpty(result.getFailedRequest()) + && (!(term.getTags().isEmpty() && request.getGlossaryTags().isEmpty()))) { + // Remove current entity tags in the database. It will be added back later from the merged tag list. + daoCollection.tagUsageDAO().deleteTagsByTarget(term.getFullyQualifiedName()); + applyTags(getUniqueTags(request.getGlossaryTags()), term.getFullyQualifiedName()); + + searchRepository.updateEntity(term.getEntityReference()); + } + + // Add Failed And Suceess Request + result.withFailedRequest(failures).withSuccessRequest(success); + + // Set Final Status + if (result.getNumberOfRowsPassed().equals(result.getNumberOfRowsProcessed())) { + result.withStatus(ApiStatus.SUCCESS); + } else if (result.getNumberOfRowsPassed() > 1) { + result.withStatus(ApiStatus.PARTIAL_SUCCESS); + } else { + result.withStatus(ApiStatus.FAILURE); + } + + return result; + } + + public BulkOperationResult bulkRemoveGlossaryToAssets(UUID glossaryTermId, AddGlossaryToAssetsRequest request) { + GlossaryTerm term = this.get(null, glossaryTermId, getFields("id,tags")); + + BulkOperationResult result = new BulkOperationResult().withStatus(ApiStatus.SUCCESS).withDryRun(false); + List success = new ArrayList<>(); + + // Validation for entityReferences + EntityUtil.populateEntityReferences(request.getAssets()); + + for (EntityReference ref : request.getAssets()) { + // Update Result Processed + result.setNumberOfRowsProcessed(result.getNumberOfRowsProcessed() + 1); + + EntityRepository entityRepository = Entity.getEntityRepository(ref.getType()); + EntityInterface asset = entityRepository.get(null, ref.getId(), entityRepository.getFields("id")); + + daoCollection + .tagUsageDAO() + .deleteTagsByTagAndTargetEntity(term.getFullyQualifiedName(), asset.getFullyQualifiedName()); + success.add(ref); + result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); + + // Update ES + searchRepository.updateEntity(ref); + } + + return result.withSuccessRequest(success); + } + protected EntityReference getGlossary(GlossaryTerm term) { Relationship relationship = term.getParent() != null ? Relationship.HAS : Relationship.CONTAINS; return term.getGlossary() != null @@ -377,6 +506,40 @@ public void entitySpecificUpdate() { updateParent(original, updated); } + @Override + protected void updateTags(String fqn, String fieldName, List origTags, List updatedTags) { + // Remove current entity tags in the database. It will be added back later from the merged tag list. + origTags = listOrEmpty(origTags); + // updatedTags cannot be immutable list, as we are adding the origTags to updatedTags even if its empty. + updatedTags = Optional.ofNullable(updatedTags).orElse(new ArrayList<>()); + if (origTags.isEmpty() && updatedTags.isEmpty()) { + return; // Nothing to update + } + + List targetFQNHashes = daoCollection.tagUsageDAO().getTargetFQNHashForTag(fqn); + for (String fqnHash : targetFQNHashes) { + Map> allAssetTags = daoCollection.tagUsageDAO().getTagsByPrefix(fqnHash, "%", false); + + // Assets FQN is not available / we can use fqnHash for now + checkMutuallyExclusiveForParentAndSubField("", fqnHash, allAssetTags, updatedTags, true); + } + + // Remove current entity tags in the database. It will be added back later from the merged tag list. + daoCollection.tagUsageDAO().deleteTagsByTarget(fqn); + + if (operation.isPut()) { + // PUT operation merges tags in the request with what already exists + EntityUtil.mergeTags(updatedTags, origTags); + checkMutuallyExclusive(updatedTags); + } + + List addedTags = new ArrayList<>(); + List deletedTags = new ArrayList<>(); + recordListChange(fieldName, origTags, updatedTags, addedTags, deletedTags, tagLabelMatch); + updatedTags.sort(compareTagLabel); + applyTags(updatedTags, fqn); + } + private void updateStatus(GlossaryTerm origTerm, GlossaryTerm updatedTerm) { if (origTerm.getStatus() == updatedTerm.getStatus()) { return; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java index ef341601ac7d..433726bcb810 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java @@ -17,6 +17,7 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.Entity.DASHBOARD; import static org.openmetadata.service.Entity.MLMODEL; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.service.util.EntityUtil.mlFeatureMatch; import static org.openmetadata.service.util.EntityUtil.mlHyperParameterMatch; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java index a5c1ec2adea5..37005c91e1d9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java @@ -19,6 +19,8 @@ import static org.openmetadata.schema.type.Include.NON_DELETED; import static org.openmetadata.service.Entity.CONTAINER; import static org.openmetadata.service.Entity.FIELD_TAGS; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import static org.openmetadata.service.util.EntityUtil.taskMatch; import java.util.ArrayList; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java index d80c55271328..cffb5c2d722d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java @@ -21,6 +21,8 @@ import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; import static org.openmetadata.service.Entity.FIELD_TAGS; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import static org.openmetadata.service.util.EntityUtil.getSearchIndexField; import java.util.ArrayList; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java index 90e9b259cda2..9b4ccf998d78 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java @@ -21,6 +21,8 @@ import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_TAGS; import static org.openmetadata.service.Entity.populateEntityFieldTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import java.util.ArrayList; import java.util.HashSet; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java index a95430a36782..c6e79e4114eb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java @@ -49,6 +49,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; +import org.openmetadata.schema.api.AddGlossaryToAssetsRequest; import org.openmetadata.schema.api.VoteRequest; import org.openmetadata.schema.api.data.CreateGlossaryTerm; import org.openmetadata.schema.api.data.LoadGlossary; @@ -60,6 +61,7 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.type.api.BulkOperationResult; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.CatalogExceptionMessage; @@ -415,6 +417,49 @@ public Response updateVote( return repository.updateVote(securityContext.getUserPrincipal().getName(), id, request).toResponse(); } + @PUT + @Path("/{id}/assets/add") + @Operation( + operationId = "bulkAddGlossaryTermToAssets", + summary = "Bulk Add Glossary Term to Assets", + description = "Bulk Add Glossary Term to Assets", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = BulkOperationResult.class))), + @ApiResponse(responseCode = "404", description = "model for instance {id} is not found") + }) + public Response bulkAddGlossaryToAssets( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Entity", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Valid AddGlossaryToAssetsRequest request) { + return Response.ok().entity(repository.bulkAddAndValidateGlossaryToAssets(id, request)).build(); + } + + @PUT + @Path("/{id}/assets/remove") + @Operation( + operationId = "bulkRemoveGlossaryTermFromAssets", + summary = "Bulk Remove Glossary Term from Assets", + description = "Bulk Remove Glossary Term from Assets", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = ChangeEvent.class))), + @ApiResponse(responseCode = "404", description = "model for instance {id} is not found") + }) + public Response bulkRemoveGlossaryFromAssets( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Entity", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Valid AddGlossaryToAssetsRequest request) { + return Response.ok().entity(repository.bulkRemoveGlossaryToAssets(id, request)).build(); + } + @DELETE @Path("/{id}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java index 3e1256da8d3c..b335ac73f87f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java @@ -13,8 +13,19 @@ package org.openmetadata.service.resources.tags; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.type.Include.NON_DELETED; +import static org.openmetadata.service.util.EntityUtil.compareTagLabel; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.classification.Classification; import org.openmetadata.schema.entity.classification.Tag; @@ -23,6 +34,8 @@ import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TagLabel.TagSource; import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.CatalogExceptionMessage; +import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.FullyQualifiedName; @Slf4j @@ -82,4 +95,102 @@ public static boolean mutuallyExclusive(TagLabel label) { throw new IllegalArgumentException("Invalid source type " + label.getSource()); } } + + public static List addDerivedTags(List tagLabels) { + if (nullOrEmpty(tagLabels)) { + return tagLabels; + } + + List updatedTagLabels = new ArrayList<>(); + EntityUtil.mergeTags(updatedTagLabels, tagLabels); + for (TagLabel tagLabel : tagLabels) { + EntityUtil.mergeTags(updatedTagLabels, getDerivedTags(tagLabel)); + } + updatedTagLabels.sort(compareTagLabel); + return updatedTagLabels; + } + + private static List getDerivedTags(TagLabel tagLabel) { + if (tagLabel.getSource() == TagLabel.TagSource.GLOSSARY) { // Related tags are only supported for Glossary + List derivedTags = Entity.getCollectionDAO().tagUsageDAO().getTags(tagLabel.getTagFQN()); + derivedTags.forEach(tag -> tag.setLabelType(TagLabel.LabelType.DERIVED)); + return derivedTags; + } + return Collections.emptyList(); + } + + public static List getUniqueTags(List tags) { + Set uniqueTags = new TreeSet<>(compareTagLabel); + uniqueTags.addAll(tags); + return uniqueTags.stream().toList(); + } + + public static void checkMutuallyExclusive(List tagLabels) { + Map map = new HashMap<>(); + for (TagLabel tagLabel : listOrEmpty(tagLabels)) { + // When two tags have the same parent that is mutuallyExclusive, then throw an error + String parentFqn = FullyQualifiedName.getParentFQN(tagLabel.getTagFQN()); + TagLabel stored = map.put(parentFqn, tagLabel); + if (stored != null && TagLabelUtil.mutuallyExclusive(tagLabel)) { + throw new IllegalArgumentException(CatalogExceptionMessage.mutuallyExclusiveLabels(tagLabel, stored)); + } + } + } + + public static void checkMutuallyExclusiveForParentAndSubField( + String assetFqn, + String assetFqnHash, + Map> allAssetTags, + List glossaryTags, + boolean validateSubFields) { + boolean failed = false; + StringBuilder errorMessage = new StringBuilder(); + + Map> filteredTags = + allAssetTags.entrySet().stream() + .filter(entry -> entry.getKey().startsWith(assetFqnHash)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + // Check Parent Tags + List parentTags = filteredTags.remove(assetFqnHash); + + if (parentTags != null) { + List tempList = new ArrayList<>(addDerivedTags(parentTags)); + tempList.addAll(glossaryTags); + try { + checkMutuallyExclusive(getUniqueTags(tempList)); + } catch (IllegalArgumentException ex) { + failed = true; + errorMessage.append( + String.format( + "Asset %s has a tag %s which is mutually exclusive with the one of the glossary tags %s. %n", + assetFqn, converTagLabelArrayToString(tempList), converTagLabelArrayToString(glossaryTags))); + } + } + + if (validateSubFields) { + // Check SubFields Tags + Set subFieldTags = filteredTags.values().stream().flatMap(List::stream).collect(Collectors.toSet()); + List tempList = new ArrayList<>(addDerivedTags(subFieldTags.stream().toList())); + tempList.addAll(glossaryTags); + try { + checkMutuallyExclusive(getUniqueTags(tempList)); + } catch (IllegalArgumentException ex) { + failed = true; + errorMessage.append( + String.format( + "Asset %s has a Subfield Column/Schema/Field containing tags %s which is mutually exclusive with the one of the glossary tags %s", + assetFqn, converTagLabelArrayToString(tempList), converTagLabelArrayToString(glossaryTags))); + } + } + + // Throw Exception if failed + if (failed) { + throw new IllegalArgumentException(errorMessage.toString()); + } + } + + public static String converTagLabelArrayToString(List tags) { + return String.format("[%s]", tags.stream().map(TagLabel::getTagFQN).collect(Collectors.joining(", "))); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 9e32fd90645e..bf8fc4d0955c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -52,6 +52,7 @@ import org.openmetadata.schema.type.UsageDetails; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.UnhandledServerException; +import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.search.elasticsearch.ElasticSearchClient; import org.openmetadata.service.search.indexes.SearchIndex; import org.openmetadata.service.search.models.IndexMapping; @@ -261,6 +262,13 @@ public void updateEntity(EntityInterface entity) { } } + public void updateEntity(EntityReference entityReference) { + EntityRepository entityRepository = Entity.getEntityRepository(entityReference.getType()); + EntityInterface entity = entityRepository.get(null, entityReference.getId(), entityRepository.getFields("*")); + // Update Entity + updateEntity(entity); + } + public void propagateInheritedFieldsToChildren( String entityType, String entityId, ChangeDescription changeDescription, IndexMapping indexMapping) { if (changeDescription != null) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java b/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java index 46406cd5ba5d..2be797df8fb8 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java @@ -18,10 +18,10 @@ import org.mockito.Mockito; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.csv.CsvFile; import org.openmetadata.schema.type.csv.CsvHeader; import org.openmetadata.schema.type.csv.CsvImportResult; -import org.openmetadata.schema.type.csv.CsvImportResult.Status; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.TableRepository; @@ -56,14 +56,14 @@ void test_validateCsvInvalidHeader() throws IOException { String csv = ",h2,h3" + LINE_SEPARATOR; // Header h1 is missing in the CSV file TestCsv testCsv = new TestCsv(); CsvImportResult importResult = testCsv.importCsv(csv, true); - assertSummary(importResult, Status.ABORTED, 1, 0, 1); + assertSummary(importResult, ApiStatus.ABORTED, 1, 0, 1); assertNull(importResult.getImportResultsCsv()); assertEquals(TestCsv.invalidHeader("h1*,h2,h3", ",h2,h3"), importResult.getAbortReason()); } public static void assertSummary( CsvImportResult importResult, - Status expectedStatus, + ApiStatus expectedStatus, int expectedRowsProcessed, int expectedRowsPassed, int expectedRowsFailed) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index a697da19a823..6f2ffadc667d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -150,6 +150,7 @@ import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.type.AccessDetails; import org.openmetadata.schema.type.AnnouncementDetails; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.Column; @@ -2906,7 +2907,7 @@ protected void importCsvAndValidate( // Validate the imported result summary - it should include both created and updated records int totalRows = 1 + createRecords.size() + updateRecords.size(); - assertSummary(dryRunResult, CsvImportResult.Status.SUCCESS, totalRows, totalRows, 0); + assertSummary(dryRunResult, ApiStatus.SUCCESS, totalRows, totalRows, 0); String expectedResultsCsv = EntityCsvTest.createCsvResult(csvHeaders, createRecords, updateRecords); assertEquals(expectedResultsCsv, dryRunResult.getImportResultsCsv()); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index 5bf53487c279..002519a2a4ff 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -63,6 +63,7 @@ import org.openmetadata.schema.entity.data.Glossary; import org.openmetadata.schema.entity.data.GlossaryTerm; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; @@ -348,7 +349,7 @@ void testImportInvalidCsv() { String csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(glossaryName, csv, false); Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); String[] expectedRows = { resultsHeader, getFailedRecord(record, "[name must match \"\"^(?U)[\\w'\\- .&()%]+$\"\"]") }; @@ -359,7 +360,7 @@ record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,," csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); result = importCsv(glossaryName, csv, false); Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, entityNotFound(0, "invalidParent"))}; assertRows(result, expectedRows); @@ -367,7 +368,7 @@ record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,," record = ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tag.invalidTag,,,"; csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); result = importCsv(glossaryName, csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, entityNotFound(7, "Tag.invalidTag"))}; assertRows(result, expectedRows); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java index c94a520055b0..728e7b1544b3 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java @@ -91,6 +91,7 @@ import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.TeamHierarchy; import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.ImageList; @@ -735,7 +736,7 @@ void testImportInvalidCsv() throws IOException { String record = getRecord(1, GROUP, team.getName(), "", false, "", "invalidPolicy"); String csv = createCsv(TeamCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); String[] expectedRows = {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(8, "invalidPolicy"))}; assertRows(result, expectedRows); @@ -743,7 +744,7 @@ void testImportInvalidCsv() throws IOException { record = getRecord(1, GROUP, team.getName(), "", false, "invalidRole", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(7, "invalidRole"))}; assertRows(result, expectedRows); @@ -751,7 +752,7 @@ record = getRecord(1, GROUP, team.getName(), "", false, "invalidRole", ""); record = getRecord(1, GROUP, team.getName(), "invalidOwner", false, "", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(5, "invalidOwner"))}; assertRows(result, expectedRows); @@ -759,7 +760,7 @@ record = getRecord(1, GROUP, team.getName(), "invalidOwner", false, "", ""); record = getRecord(1, GROUP, "invalidParent", "", false, "", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(4, "invalidParent"))}; assertRows(result, expectedRows); @@ -767,7 +768,7 @@ record = getRecord(1, GROUP, "invalidParent", "", false, "", ""); record = getRecord(1, GROUP, TEAM21.getName(), "", false, "", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] { resultsHeader, getFailedRecord(record, TeamCsv.invalidTeam(4, team.getName(), "x1", TEAM21.getName())) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java index aaa2c6a36ed1..85666fd4c3e3 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java @@ -111,6 +111,7 @@ import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.security.client.GoogleSSOClientConfig; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.ImageList; @@ -991,7 +992,7 @@ void testImportInvalidCsv() throws IOException { String record = "invalid::User,,,user@domain.com,,,team-invalidCsv,"; String csv = createCsv(UserCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); String[] expectedRows = {resultsHeader, getFailedRecord(record, "[name must match \"\"^(?U)[\\w\\-.]+$\"\"]")}; assertRows(result, expectedRows); @@ -1000,7 +1001,7 @@ void testImportInvalidCsv() throws IOException { record = "user,,,user@domain.com,,,invalidTeam,"; csv = createCsv(UserCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(6, "invalidTeam"))}; assertRows(result, expectedRows); @@ -1008,7 +1009,7 @@ record = "user,,,user@domain.com,,,invalidTeam,"; record = "user,,,user@domain.com,,,team-invalidCsv,invalidRole"; csv = createCsv(UserCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(7, "invalidRole"))}; assertRows(result, expectedRows); } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/addGlossaryToAssetsRequest.json b/openmetadata-spec/src/main/resources/json/schema/api/addGlossaryToAssetsRequest.json new file mode 100644 index 000000000000..4e10674b10e8 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/addGlossaryToAssetsRequest.json @@ -0,0 +1,28 @@ +{ + "$id": "https://open-metadata.org/schema/api/createBot.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AddGlossaryToAssetsRequest", + "description": "Create Request for adding a glossary to assets", + "type": "object", + "javaType": "org.openmetadata.schema.api.AddGlossaryToAssetsRequest", + "properties": { + "dryRun": { + "description": "If true, the request will be validated but no changes will be made", + "type": "boolean", + "default": true + }, + "glossaryTags": { + "description": "Glossary Tags to be added", + "type": "array", + "items": { + "$ref": "../type/tagLabel.json" + }, + "default": null + }, + "assets": { + "description": "List of assets to be created against which the glossary needs to be added.", + "$ref": "../type/entityReferenceList.json" + } + }, + "additionalProperties": false +} diff --git a/openmetadata-spec/src/main/resources/json/schema/type/basic.json b/openmetadata-spec/src/main/resources/json/schema/type/basic.json index 9080afc37592..fa13e00d5e65 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/basic.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/basic.json @@ -148,6 +148,7 @@ } }, "status" : { + "javaType": "org.openmetadata.schema.type.ApiStatus", "description": "State of an action over API.", "type" : "string", "enum" : ["success", "failure", "aborted", "partialSuccess"] diff --git a/openmetadata-spec/src/main/resources/json/schema/type/bulkOperationResult.json b/openmetadata-spec/src/main/resources/json/schema/type/bulkOperationResult.json new file mode 100644 index 000000000000..cbf1eabe6aa3 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/type/bulkOperationResult.json @@ -0,0 +1,71 @@ +{ + "$id": "https://open-metadata.org/schema/type/csvImportResult.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "BulkOperationResult", + "description": "Represents result of bulk Operation performed on entities.", + "type": "object", + "javaType": "org.openmetadata.schema.type.api.BulkOperationResult", + "definitions": { + "rowCount" : { + "description": "Type used to indicate row count", + "type": "integer", + "format" : "int64", + "minimum": 0, + "default": 0 + }, + "index" : { + "description": "Type used to indicate row number or field number. In CSV the indexes start with 1.", + "type": "integer", + "format" : "int64", + "minimum": 1 + }, + "failureRequest": { + "description": "Request that can be processed successfully.", + "type": "object", + "properties": { + "request": { + "description": "Request that can be processed successfully." + }, + "error": { + "description": "Error that occurred while processing the request.", + "type": "string" + } + }, + "additionalProperties": false + } + }, + "properties": { + "dryRun" : { + "description": "True if the operation has dryRun flag enabled", + "type" : "boolean" + }, + "status" : { + "$ref" : "basic.json#/definitions/status" + }, + "abortReason": { + "description": "Reason why import was aborted. This is set only when the `status` field is set to `aborted`", + "type" : "string" + }, + "numberOfRowsProcessed" : { + "$ref": "#/definitions/rowCount" + }, + "numberOfRowsPassed" : { + "$ref": "#/definitions/rowCount" + }, + "numberOfRowsFailed" : { + "$ref": "#/definitions/rowCount" + }, + "successRequest": { + "description": "Request that can be processed successfully." + }, + "failedRequest": { + "description": "Failure Request that can be processed successfully.", + "type": "array", + "items": { + "$ref": "#/definitions/failureRequest" + }, + "default": null + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js index 6e6ba1d6fb75..14e31cfe1834 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js @@ -65,13 +65,13 @@ const checkDisplayName = (displayName) => { const checkDataProductsCount = (dataProductsCount) => { cy.get('[data-testid="data_products"] [data-testid="count"]') .scrollIntoView() - .eq(dataProductsCount); + .should('have.text', dataProductsCount); }; const checkAssetsCount = (assetsCount) => { cy.get('[data-testid="assets"] [data-testid="count"]') .scrollIntoView() - .eq(assetsCount); + .should('have.text', assetsCount); }; const updateOwner = (newOwner) => { @@ -106,6 +106,13 @@ const goToAssetsTab = (domainObj) => { cy.get('.ant-tabs-tab-active').contains('Assets').should('be.visible'); }; +const goToDataProductsTab = (domainObj) => { + cy.get('[data-testid="domain-left-panel"]').contains(domainObj.name).click(); + checkDisplayName(domainObj.name); + cy.get('[data-testid="data_products"]').click(); + cy.get('.ant-tabs-tab-active').contains('Data Products').should('be.visible'); +}; + export const updateAssets = (domainObj) => { interceptURL( 'GET', @@ -361,7 +368,7 @@ export const createDataProducts = (dataProduct, domainObj) => { cy.wait('@createDataProducts').then(({ request }) => { expect(request.body.name).equals(dataProduct.name); - expect(request.body.domain).equals(domainObj.name); + expect(request.body.domain).equals(domainObj.fullyQualifiedName); expect(request.body.description).equals(dataProduct.description); expect(request.body.experts).has.length(1); }); @@ -385,3 +392,136 @@ export const renameDomain = (domainObj) => { checkDisplayName(domainObj.updatedDisplayName); }; + +export const addAssetsToDomain = (domainObj) => { + goToAssetsTab(domainObj); + checkAssetsCount(0); + cy.contains('Adding a new Asset is easy, just give it a spin!').should( + 'be.visible' + ); + + cy.get('[data-testid="domain-details-add-button"]').click(); + cy.get('.ant-dropdown-menu .ant-dropdown-menu-title-content') + .contains('Assets') + .click(); + + cy.get('[data-testid="asset-selection-modal"] .ant-modal-title').should( + 'contain', + 'Add Assets' + ); + + domainObj.assets.forEach((asset) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + cy.get('[data-testid="asset-selection-modal"] [data-testid="searchbar"]') + .click() + .clear() + .type(asset.name); + + verifyResponseStatusCode('@searchAssets', 200); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"] input[type="checkbox"]` + ).click(); + }); + + cy.get('[data-testid="save-btn"]').click(); + + checkAssetsCount(domainObj.assets.length); +}; + +export const removeAssetsFromDomain = (domainObj) => { + goToAssetsTab(domainObj); + checkAssetsCount(domainObj.assets.length); + + domainObj.assets.forEach((asset, index) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"]` + ).within(() => { + cy.get('.explore-card-actions').invoke('show'); + cy.get('.explore-card-actions').within(() => { + cy.get('[data-testid="delete-tag"]').click(); + }); + }); + + cy.get("[data-testid='save-button']").click(); + + goToDataProductsTab(domainObj); + + interceptURL('GET', '/api/v1/search/query*', 'assetTab'); + // go assets tab + goToAssetsTab(domainObj); + verifyResponseStatusCode('@assetTab', 200); + + checkAssetsCount(domainObj.assets.length - (index + 1)); + }); +}; + +export const addAssetsToDataProduct = (dataProductObj, domainObj) => { + interceptURL('GET', `/api/v1/search/query**`, 'getDataProductAssets'); + + goToDataProductsTab(domainObj); + cy.get(`[data-testid="explore-card-${dataProductObj.name}"]`).click(); + + cy.get('[data-testid="assets"]').should('be.visible').click(); + cy.get('.ant-tabs-tab-active').contains('Assets').should('be.visible'); + + verifyResponseStatusCode('@getDataProductAssets', 200); + + cy.contains('Adding a new Asset is easy, just give it a spin!').should( + 'be.visible' + ); + + cy.get('[data-testid="data-product-details-add-button"]').click(); + + cy.get('[data-testid="asset-selection-modal"] .ant-modal-title').should( + 'contain', + 'Add Assets' + ); + + dataProductObj.assets.forEach((asset) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + cy.get('[data-testid="asset-selection-modal"] [data-testid="searchbar"]') + .click() + .clear() + .type(asset.name); + + verifyResponseStatusCode('@searchAssets', 200); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"] input[type="checkbox"]` + ).click(); + }); + + cy.get('[data-testid="save-btn"]').click(); + + checkAssetsCount(dataProductObj.assets.length); +}; + +export const removeAssetsFromDataProduct = (dataProductObj, domainObj) => { + goToDataProductsTab(domainObj); + cy.get(`[data-testid="explore-card-${dataProductObj.name}"]`).click(); + + cy.get('[data-testid="assets"]').should('be.visible').click(); + cy.get('.ant-tabs-tab-active').contains('Assets').should('be.visible'); + + checkAssetsCount(dataProductObj.assets.length); + + dataProductObj.assets.forEach((asset, index) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"]` + ).within(() => { + cy.get('.explore-card-actions').invoke('show'); + cy.get('.explore-card-actions').within(() => { + cy.get('[data-testid="delete-tag"]').click(); + }); + }); + + cy.get("[data-testid='save-button']").click(); + + checkAssetsCount(domainObj.assets.length - (index + 1)); + }); +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js index 2a16c0613554..3c62dde6f00c 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js @@ -327,6 +327,26 @@ export const NEW_GLOSSARY_TERMS = { synonyms: 'give,disposal,deal', fullyQualifiedName: 'Cypress Glossary.CypressSales', }, + term_3: { + name: 'Cypress Space', + description: 'This is the Cypress with space', + synonyms: 'tea,coffee,water', + fullyQualifiedName: 'Cypress Glossary.Cypress Space', + assets: [ + { + name: 'dim_customer', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address', + }, + { + name: 'raw_order', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.raw_order', + }, + { + name: 'presto_etl', + fullyQualifiedName: 'sample_airflow.presto_etl', + }, + ], + }, }; export const GLOSSARY_TERM_WITH_DETAILS = { name: 'Accounts', @@ -518,6 +538,7 @@ export const NAME_MAX_LENGTH_VALIDATION_ERROR = export const DOMAIN_1 = { name: 'Cypress%Domain', updatedName: 'Cypress_Domain_Name', + fullyQualifiedName: 'Cypress%Domain', updatedDisplayName: 'Cypress_Domain_Display_Name', description: 'This is the Cypress for testing domain creation with percent and dot', @@ -549,8 +570,46 @@ export const DOMAIN_2 = { name: 'Cypress.Domain.New', updatedName: 'Cypress.Domain.New', updatedDisplayName: 'Cypress.Domain.New', + fullyQualifiedName: '"Cypress.Domain.New"', description: 'This is the Cypress for testing domain creation', experts: 'Alex Pollard', owner: 'Alex Pollard', domainType: 'Source-aligned', + dataProducts: [ + { + name: 'Cypress DataProduct Assets', + description: + 'This is the data product description for Cypress DataProduct Assets', + experts: 'Aaron Johnson', + owner: 'Aaron Johnson', + assets: [ + { + name: 'dim_customer', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address', + }, + { + name: 'raw_order', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.raw_order', + }, + { + name: 'presto_etl', + fullyQualifiedName: 'sample_airflow.presto_etl', + }, + ], + }, + ], + assets: [ + { + name: 'dim_customer', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address', + }, + { + name: 'raw_order', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.raw_order', + }, + { + name: 'presto_etl', + fullyQualifiedName: 'sample_airflow.presto_etl', + }, + ], }; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js index b833dda73f04..48a23ce5eacb 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js @@ -14,10 +14,14 @@ /// import { + addAssetsToDataProduct, + addAssetsToDomain, createDataProducts, createDomain, deleteDomain, removeAssets, + removeAssetsFromDataProduct, + removeAssetsFromDomain, renameDomain, updateAssets, updateDomainDetails, @@ -44,6 +48,10 @@ describe('Domain page should work properly', () => { verifyDomain(DOMAIN_2); }); + it('Add assets to domain using asset selection modal should work properly', () => { + addAssetsToDomain(DOMAIN_2); + }); + it('Create new data product should work properly', () => { DOMAIN_1.dataProducts.forEach((dataProduct) => { createDataProducts(dataProduct, DOMAIN_1); @@ -53,15 +61,34 @@ describe('Domain page should work properly', () => { }); }); + it('Add data product assets using asset selection modal should work properly', () => { + DOMAIN_2.dataProducts.forEach((dp) => { + createDataProducts(dp, DOMAIN_2); + cy.get('[data-testid="app-bar-item-domain"]') + .should('be.visible') + .click({ force: true }); + }); + + addAssetsToDataProduct(DOMAIN_2.dataProducts[0], DOMAIN_2); + }); + + it('Remove data product assets using asset selection modal should work properly', () => { + removeAssetsFromDataProduct(DOMAIN_2.dataProducts[0], DOMAIN_2); + }); + it('Update domain details should work properly', () => { updateDomainDetails(DOMAIN_1); }); + it('Remove assets to domain using asset selection modal should work properly', () => { + removeAssetsFromDomain(DOMAIN_2); + }); + it('Assets Tab should work properly', () => { updateAssets(DOMAIN_1); }); - it.skip('Remove Domain from entity should work properly', () => { + it('Remove Domain from entity should work properly', () => { removeAssets(DOMAIN_1); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js index c491a5b921ae..990c19557c27 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js @@ -88,6 +88,12 @@ const checkDisplayName = (displayName) => { }); }; +const checkAssetsCount = (assetsCount) => { + cy.get('[data-testid="assets"] [data-testid="count"]') + .scrollIntoView() + .should('have.text', assetsCount); +}; + const validateForm = () => { // error messages cy.get('#name_help') @@ -967,6 +973,86 @@ describe('Glossary page should work properly', () => { .should('be.visible'); }); + it('Add asset to glossary term using asset modal', () => { + selectActiveGlossary(NEW_GLOSSARY.name); + goToAssetsTab( + NEW_GLOSSARY_TERMS.term_3.name, + NEW_GLOSSARY_TERMS.term_3.fullyQualifiedName, + true + ); + + checkAssetsCount(0); + cy.contains('Adding a new Asset is easy, just give it a spin!').should( + 'be.visible' + ); + + cy.get('[data-testid="glossary-term-add-button-menu"]').click(); + cy.get('.ant-dropdown-menu .ant-dropdown-menu-title-content') + .contains('Assets') + .click(); + + cy.get('[data-testid="asset-selection-modal"] .ant-modal-title').should( + 'contain', + 'Add Assets' + ); + + NEW_GLOSSARY_TERMS.term_3.assets.forEach((asset) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + cy.get('[data-testid="asset-selection-modal"] [data-testid="searchbar"]') + .click() + .clear() + .type(asset.name); + + verifyResponseStatusCode('@searchAssets', 200); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"] input[type="checkbox"]` + ).click(); + }); + + cy.get('[data-testid="save-btn"]').click(); + + checkAssetsCount(NEW_GLOSSARY_TERMS.term_3.assets); + }); + + it('Remove asset from glossary term using asset modal', () => { + selectActiveGlossary(NEW_GLOSSARY.name); + goToAssetsTab( + NEW_GLOSSARY_TERMS.term_3.name, + NEW_GLOSSARY_TERMS.term_3.fullyQualifiedName, + true + ); + + checkAssetsCount(NEW_GLOSSARY_TERMS.term_3.assets.length); + NEW_GLOSSARY_TERMS.term_3.assets.assets.forEach((asset, index) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"]` + ).within(() => { + cy.get('.explore-card-actions').invoke('show'); + cy.get('.explore-card-actions').within(() => { + cy.get('[data-testid="delete-tag"]').click(); + }); + }); + + cy.get("[data-testid='save-button']").click(); + + selectActiveGlossary(NEW_GLOSSARY.name); + + interceptURL('GET', '/api/v1/search/query*', 'assetTab'); + // go assets tab + goToAssetsTab( + NEW_GLOSSARY_TERMS.term_3.name, + NEW_GLOSSARY_TERMS.term_3.fullyQualifiedName, + true + ); + verifyResponseStatusCode('@assetTab', 200); + + checkAssetsCount(NEW_GLOSSARY_TERMS.term_3.assets.length - (index + 1)); + }); + }); + it('Remove Glossary term from entity should work properly', () => { const glossaryName = NEW_GLOSSARY_1.name; const { name, fullyQualifiedName } = NEW_GLOSSARY_1_TERMS.term_1; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx index be0d2026ccb2..473e45d9df7c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Button, List, Modal, Space, Typography } from 'antd'; +import { Button, Checkbox, List, Modal, Space, Typography } from 'antd'; import { AxiosError } from 'axios'; import { compare } from 'fast-json-patch'; import { EntityDetailUnion } from 'Models'; @@ -25,6 +25,7 @@ import { import { useTranslation } from 'react-i18next'; import { PAGE_SIZE_MEDIUM } from '../../../constants/constants'; import { SearchIndex } from '../../../enums/search.enum'; +import { GlossaryTerm } from '../../../generated/entity/data/glossaryTerm'; import { Table } from '../../../generated/entity/data/table'; import { DataProduct } from '../../../generated/entity/domains/dataProduct'; import { Domain } from '../../../generated/entity/domains/domain'; @@ -33,6 +34,10 @@ import { patchDataProduct, } from '../../../rest/dataProductAPI'; import { getDomainByName } from '../../../rest/domainAPI'; +import { + addAssetsToGlossaryTerm, + getGlossaryTermByFQN, +} from '../../../rest/glossaryAPI'; import { searchQuery } from '../../../rest/searchAPI'; import { getAPIfromSource, @@ -40,6 +45,7 @@ import { getEntityAPIfromSource, } from '../../../utils/Assets/AssetsUtils'; import { getEntityReferenceFromEntity } from '../../../utils/EntityUtils'; +import { getDecodedFqn } from '../../../utils/StringsUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import Searchbar from '../../common/SearchBarComponent/SearchBar.component'; @@ -109,6 +115,9 @@ export const AssetSelectionModal = ({ 'domain,assets' ); setActiveEntity(data); + } else if (type === AssetsOfEntity.GLOSSARY) { + const data = await getGlossaryTermByFQN(getDecodedFqn(entityFqn), 'tags'); + setActiveEntity(data); } }, [type, entityFqn]); @@ -240,9 +249,37 @@ export const AssetSelectionModal = ({ } }; - const domainAndDataProductsSave = async () => { + const glossarySave = async () => { + try { + setIsSaveLoading(true); + if (!activeEntity) { + return; + } + + const entities = [...(selectedItems?.values() ?? [])].map((item) => { + return getEntityReferenceFromEntity(item, item.entityType); + }); + + await addAssetsToGlossaryTerm(activeEntity as GlossaryTerm, entities); + await new Promise((resolve) => { + setTimeout(() => { + resolve(''); + onSave?.(); + }, ES_UPDATE_DELAY); + }); + } catch (err) { + showErrorToast(err as AxiosError); + } finally { + setIsSaveLoading(false); + onCancel(); + } + }; + + const handleSave = async () => { if (type === AssetsOfEntity.DATA_PRODUCT) { dataProductsSave(); + } else if (type === AssetsOfEntity.GLOSSARY) { + glossarySave(); } else { try { setIsSaveLoading(true); @@ -291,72 +328,9 @@ export const AssetSelectionModal = ({ } }; - const handleSave = async () => { - setIsSaveLoading(true); - const entityDetails = [...(selectedItems?.values() ?? [])].map((item) => - getEntityAPIfromSource(item.entityType)( - item.fullyQualifiedName, - getAssetsFields(type) - ) - ); - - try { - const entityDetailsResponse = await Promise.allSettled(entityDetails); - const map = new Map(); - - entityDetailsResponse.forEach((response) => { - if (response.status === 'fulfilled') { - const entity = response.value; - entity && map.set(entity.fullyQualifiedName, (entity as Table).tags); - } - }); - const patchAPIPromises = [...(selectedItems?.values() ?? [])] - .map((item) => { - if (map.has(item.fullyQualifiedName)) { - const jsonPatch = compare( - { tags: map.get(item.fullyQualifiedName) }, - { - tags: [ - ...(item.tags ?? []), - { - tagFQN: entityFqn, - source: 'Glossary', - labelType: 'Manual', - }, - ], - } - ); - const api = getAPIfromSource(item.entityType); - - return api(item.id, jsonPatch); - } - - return; - }) - .filter(Boolean); - - await Promise.all(patchAPIPromises); - await new Promise((resolve) => { - setTimeout(() => { - resolve(''); - onSave?.(); - }, ES_UPDATE_DELAY); - }); - } catch (err) { - showErrorToast(err as AxiosError); - } finally { - setIsSaveLoading(false); - onCancel(); - } - }; - const onSaveAction = useCallback(() => { - if (type === AssetsOfEntity.GLOSSARY) { - handleSave(); - } else { - domainAndDataProductsSave(); - } - }, [type, handleSave, domainAndDataProductsSave]); + handleSave(); + }, [type, handleSave]); const onScroll: UIEventHandler = useCallback( (e) => { @@ -387,27 +361,62 @@ export const AssetSelectionModal = ({ ] ); + const onSelectAll = (selectAll: boolean) => { + setSelectedItems((prevItems) => { + const selectedItemMap = new Map(prevItems ?? []); + + if (selectAll) { + items.forEach(({ _source }) => { + const id = _source.id; + if (id) { + selectedItemMap.set(id, _source); + } + }); + } else { + // Clear selection + selectedItemMap.clear(); + } + + return selectedItemMap; + }); + }; + return ( - - - +
+
+ {selectedItems && selectedItems.size > 1 && ( + + {selectedItems.size} {t('label.selected-lowercase')} + + )} +
+ +
+ + +
+
} open={open} style={{ top: 40 }} title={t('label.add-entity', { entity: t('label.asset-plural') })} - width={750} + width={675} onCancel={onCancel}> - {isLoading && } - - {!isLoading && items.length > 0 && ( - - - {({ _source: item }) => ( - - )} - - + {items.length > 0 && ( +
+ onSelectAll(e.target.checked)}> + {t('label.select-field', { + field: t('label.all'), + })} + + + + {({ _source: item }) => ( + + )} + + +
)} + {!isLoading && items.length === 0 && ( {emptyPlaceHolderText && ( @@ -453,6 +472,8 @@ export const AssetSelectionModal = ({ )} )} + + {isLoading && } ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/asset-selection-model.style.less b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/asset-selection-model.style.less index 46fac30167de..463ceb9376a8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/asset-selection-model.style.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/asset-selection-model.style.less @@ -24,15 +24,30 @@ // CheckBox .ant-checkbox-inner { border-width: 2px; - height: 18px; - width: 18px; + height: 20px; + width: 20px; border-color: @primary-color; } .ant-checkbox-inner::after { - top: 48%; + top: 44%; left: 18.5%; - width: 6.25px; - height: 10px; + width: 6.5px; + height: 11px; + } + + .data-asset-info-row { + gap: 16px; + } + + .entity-header-display-name { + font-weight: 500 !important; + max-width: 500px; + } +} + +.asset-selection-modal { + .rc-virtual-list-holder { + overflow-x: hidden; } } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx index 64cbfcdaf9af..10b8fb0ac18d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx @@ -64,6 +64,7 @@ import { setMsalInstance, } from '../../../utils/AuthProvider.util'; import localState from '../../../utils/LocalStorageUtils'; +import { escapeESReservedCharacters } from '../../../utils/StringsUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import { fetchAllUsers, @@ -429,7 +430,9 @@ export const AuthProvider = ({ // Parse and update the query parameter const queryParams = Qs.parse(config.url.split('?')[1]); // adding quotes for exact matching - const domainStatement = `(domain.fullyQualifiedName:${activeDomain})`; + const domainStatement = `(domain.fullyQualifiedName:${escapeESReservedCharacters( + activeDomain + )})`; queryParams.q = queryParams.q ?? ''; queryParams.q += isEmpty(queryParams.q) ? domainStatement diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/DataProductsDetailsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/DataProductsDetailsPage.component.tsx index da55ed0f02af..eaf05064edb1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/DataProductsDetailsPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/DataProductsDetailsPage.component.tsx @@ -77,6 +77,7 @@ import { getDataProductVersionsPath, getDomainPath, } from '../../../utils/RouterUtils'; +import { escapeESReservedCharacters } from '../../../utils/StringsUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import { EntityDetailsObjectInterface } from '../../Explore/ExplorePage.interface'; import StyleModal from '../../Modals/StyleModal/StyleModal.component'; @@ -202,13 +203,15 @@ const DataProductsDetailsPage = ({ }, [permissions, isVersionsView]); const fetchDataProductAssets = async () => { - if (fqn) { + if (dataProduct) { try { const res = await searchData( '', 1, 0, - `(dataProducts.fullyQualifiedName:${fqn})`, + `(dataProducts.fullyQualifiedName:"${escapeESReservedCharacters( + dataProduct.fullyQualifiedName + )}")`, '', '', SearchIndex.ALL diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/data-products-details-page.less b/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/data-products-details-page.less index 6807fa2e733b..88460ebd1e90 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/data-products-details-page.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataProducts/DataProductsDetailsPage/data-products-details-page.less @@ -23,10 +23,6 @@ .ant-tabs-tab { margin-left: 24px; } - .assets-data-container { - padding-left: 18px; - padding-right: 18px; - } .page-layout-v1-vertical-scroll, .page-layout-leftpanel { height: @domain-page-height; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainDetailsPage/DomainDetailsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainDetailsPage/DomainDetailsPage.component.tsx index cbdc0ad4c1d6..da4b4f20b449 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainDetailsPage/DomainDetailsPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainDetailsPage/DomainDetailsPage.component.tsx @@ -81,6 +81,7 @@ import { getDomainPath, getDomainVersionsPath, } from '../../../utils/RouterUtils'; +import { escapeESReservedCharacters } from '../../../utils/StringsUtils'; import { showErrorToast } from '../../../utils/ToastUtils'; import DeleteWidgetModal from '../../common/DeleteWidget/DeleteWidgetModal'; import { EntityDetailsObjectInterface } from '../../Explore/ExplorePage.interface'; @@ -238,7 +239,9 @@ const DomainDetailsPage = ({ '', 1, 0, - `(domain.fullyQualifiedName:${domainFqn})`, + `(domain.fullyQualifiedName:${escapeESReservedCharacters( + domain.fullyQualifiedName + )})`, '', '', SearchIndex.DATA_PRODUCT @@ -258,7 +261,9 @@ const DomainDetailsPage = ({ '', 1, 0, - `(domain.fullyQualifiedName:${fqn}) AND !(entityType:"dataProduct")`, + `(domain.fullyQualifiedName:"${escapeESReservedCharacters( + domain.fullyQualifiedName + )}") AND !(entityType:"dataProduct")`, '', '', SearchIndex.ALL diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx index f7cab2837bfb..54c67e194946 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DataProductsTab/DataProductsTab.component.tsx @@ -29,6 +29,10 @@ import { SearchIndex } from '../../../../enums/search.enum'; import { DataProduct } from '../../../../generated/entity/domains/dataProduct'; import { searchData } from '../../../../rest/miscAPI'; import { formatDataProductResponse } from '../../../../utils/APIUtils'; +import { + escapeESReservedCharacters, + getDecodedFqn, +} from '../../../../utils/StringsUtils'; import ErrorPlaceHolder from '../../../common/ErrorWithPlaceholder/ErrorPlaceHolder'; import EntitySummaryPanel from '../../../Explore/EntitySummaryPanel/EntitySummaryPanel.component'; import ExploreSearchCard from '../../../ExploreV1/ExploreSearchCard/ExploreSearchCard'; @@ -58,7 +62,9 @@ const DataProductsTab = forwardRef( '', 1, PAGE_SIZE_LARGE, - `(domain.fullyQualifiedName:${domainFqn})`, + `(domain.fullyQualifiedName:"${escapeESReservedCharacters( + getDecodedFqn(domainFqn) + )}")`, '', '', SearchIndex.DATA_PRODUCT diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeader/EntityHeader.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeader/EntityHeader.component.tsx index 4b2bff18416d..ddded7b20f2c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeader/EntityHeader.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeader/EntityHeader.component.tsx @@ -35,6 +35,7 @@ interface Props { serviceName: string; titleColor?: string; badge?: React.ReactNode; + showName?: boolean; } export const EntityHeader = ({ @@ -48,6 +49,7 @@ export const EntityHeader = ({ serviceName, badge, titleColor, + showName = true, }: Props) => { return (
@@ -74,6 +76,7 @@ export const EntityHeader = ({ name={entityData.name} openEntityInNewPage={openEntityInNewPage} serviceName={serviceName} + showName={showName} />
); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.component.tsx index 35fc94e0876a..f13bff49575d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.component.tsx @@ -34,6 +34,7 @@ const EntityHeaderTitle = ({ isDisabled, className, color, + showName = true, }: EntityHeaderTitleProps) => { const { t } = useTranslation(); const location = useLocation(); @@ -51,9 +52,9 @@ const EntityHeaderTitle = ({ gutter={12} wrap={false}> {icon} - + {/* If we do not have displayName name only be shown in the bold from the below code */} - {!isEmpty(displayName) ? ( + {!isEmpty(displayName) && showName ? ( @@ -98,7 +99,7 @@ const EntityHeaderTitle = ({ return link && !isTourRoute ? ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.interface.ts index 69057e78699e..02462f5fbed1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.interface.ts @@ -22,4 +22,5 @@ export interface EntityHeaderTitleProps { serviceName: string; badge?: React.ReactNode; isDisabled?: boolean; + showName?: boolean; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts index d165e95e7016..472c6f5baec1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts @@ -20,7 +20,7 @@ export interface ExploreQuickFiltersProps { fields: Array; aggregations?: Aggregations; onFieldValueSelect: (field: ExploreQuickFilterField) => void; - onAdvanceSearch: () => void; + onAdvanceSearch?: () => void; showDeleted?: boolean; onChangeShowDeleted?: (showDeleted: boolean) => void; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.interface.ts index 79539a50c3ba..1dba6ed64737 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.interface.ts @@ -32,4 +32,5 @@ export interface ExploreSearchCardProps { openEntityInNewPage?: boolean; hideBreadcrumbs?: boolean; actionPopoverContent?: React.ReactNode; + onCheckboxChange?: (checked: boolean) => void; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx index cbd219fe05fe..f21e29959e87 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Button, Col, Row, Space, Typography } from 'antd'; +import { Button, Checkbox, Col, Row, Space, Typography } from 'antd'; import classNames from 'classnames'; import { isString, startCase, uniqueId } from 'lodash'; import { ExtraInfo } from 'Models'; @@ -59,6 +59,9 @@ const ExploreSearchCard: React.FC = forwardRef< openEntityInNewPage, hideBreadcrumbs = false, actionPopoverContent, + showCheckboxes = false, + checked = false, + onCheckboxChange, }, ref ) => { @@ -98,6 +101,11 @@ const ExploreSearchCard: React.FC = forwardRef< isLink: true, openInNewTab: false, }); + } else { + _otherDetails.push({ + key: 'Domain', + value: '', + }); } if ( @@ -163,8 +171,17 @@ const ExploreSearchCard: React.FC = forwardRef< const header = useMemo(() => { return ( + {showCheckboxes && ( + + onCheckboxChange?.(e.target.checked)} + /> + + )} {!hideBreadcrumbs && ( - +
{serviceIcon}
@@ -176,7 +193,11 @@ const ExploreSearchCard: React.FC = forwardRef<
)} - + {isTourOpen ? ( + ) : null } + checked={selectedItems?.has(_source.id ?? '')} className={classNames( 'm-b-sm cursor-pointer', selectedCard?.id === _source.id ? 'highlight-card' : '' @@ -528,6 +645,9 @@ const AssetsTabs = forwardRef( key={'assets_' + _id} showTags={false} source={_source} + onCheckboxChange={(selected) => + handleCheckboxChange(selected, _source) + } /> ))} {showPagination && ( @@ -554,16 +674,48 @@ const AssetsTabs = forwardRef( currentPage, selectedCard, assetErrorPlaceHolder, + selectedItems, setSelectedCard, handlePageChange, showPagination, handlePageSizeChange, + handleCheckboxChange, ] ); + const onSelectAll = (selectAll: boolean) => { + setSelectedItems((prevItems) => { + const selectedItemMap = new Map(prevItems ?? []); + + if (selectAll) { + data.forEach(({ _source }) => { + const id = _source.id; + if (id) { + selectedItemMap.set(id, _source); + } + }); + } else { + // Clear selection + selectedItemMap.clear(); + } + + return selectedItemMap; + }); + }; + const assetsHeader = useMemo(() => { return ( -
+
+ {data.length > 0 && ( + onSelectAll(e.target.checked)}> + {t('label.select-field', { + field: t('label.all'), + })} + + )} + { @@ -630,71 +783,119 @@ const AssetsTabs = forwardRef( ); }, [assetsHeader, assetListing, selectedCard]); - const onAssetRemove = useCallback(async () => { - if (!activeEntity) { - return; - } + const onAssetRemove = useCallback( + async (assetsData: SourceType[]) => { + if (!activeEntity) { + return; + } - try { - let updatedEntity; - switch (type) { - case AssetsOfEntity.DATA_PRODUCT: - const updatedAssets = ( - (activeEntity as DataProduct)?.assets ?? [] - ).filter((asset) => asset.id !== assetToDelete?.id); - updatedEntity = { - ...activeEntity, - assets: updatedAssets, - }; - const jsonPatch = compare(activeEntity, updatedEntity); - const res = await patchDataProduct( - (activeEntity as DataProduct).id, - jsonPatch - ); - setActiveEntity(res); - - break; - - case AssetsOfEntity.DOMAIN: - case AssetsOfEntity.GLOSSARY: - const selectedItemMap = new Map(); - selectedItemMap.set(assetToDelete?.id, assetToDelete); - - if (type === AssetsOfEntity.DOMAIN) { - await updateDomainAssets(undefined, type, selectedItemMap); - } else if (type === AssetsOfEntity.GLOSSARY) { - await removeGlossaryTermAssets( - entityFqn ?? '', - type, - selectedItemMap + try { + let updatedEntity; + switch (type) { + case AssetsOfEntity.DATA_PRODUCT: + const updatedAssets = ( + (activeEntity as DataProduct)?.assets ?? [] + ).filter( + (asset) => !assetsData.some((item) => item.id === asset.id) ); - } - break; - default: - // Handle other entity types here - break; + updatedEntity = { + ...activeEntity, + assets: updatedAssets, + }; + const jsonPatch = compare(activeEntity, updatedEntity); + const res = await patchDataProduct( + (activeEntity as DataProduct).id, + jsonPatch + ); + setActiveEntity(res); + + break; + + case AssetsOfEntity.GLOSSARY: + const entities = [...(assetsData?.values() ?? [])].map((item) => { + return getEntityReferenceFromEntity( + item, + (item as EntityDetailUnion).entityType + ); + }); + await removeAssetsFromGlossaryTerm( + activeEntity as GlossaryTerm, + entities + ); + + break; + + case AssetsOfEntity.DOMAIN: + const selectedItemMap = new Map(); + assetsData.forEach((item) => { + selectedItemMap.set(item.id, item); + }); + await updateDomainAssets(undefined, type, selectedItemMap); + + break; + default: + // Handle other entity types here + break; + } + + await new Promise((resolve) => { + setTimeout(() => { + resolve(''); + }, ES_UPDATE_DELAY); + }); + } catch (err) { + showErrorToast(err as AxiosError); + } finally { + setShowDeleteModal(false); + onRemoveAsset?.(); + hideNotification(); + setSelectedItems(new Map()); // Reset selected items } + }, + [type, activeEntity, entityFqn] + ); - await new Promise((resolve) => { - setTimeout(() => { - resolve(''); - }, ES_UPDATE_DELAY); - }); - } catch (err) { - showErrorToast(err as AxiosError); - } finally { - setShowDeleteModal(false); - onRemoveAsset?.(); - } - }, [type, activeEntity, assetToDelete, entityFqn]); + const openNotification = () => { + notification.warning({ + key: 'asset-tab-notification-key', + message: ( +
+ {selectedItems && selectedItems.size > 1 && ( + + {selectedItems.size} {t('label.selected-lowercase')} + + )} + +
+ ), + placement: 'bottom', + className: 'asset-tab-delete-notification', + duration: 0, + }); + }; useEffect(() => { fetchAssets({ index: isEmpty(activeFilter) ? [SearchIndex.ALL] : activeFilter, page: currentPage, }); - }, [activeFilter, currentPage, pageSize]); + }, [activeFilter, currentPage, pageSize, searchValue]); + + useEffect(() => { + const dropdownItems = getAssetsPageQuickFilters(); + + setSelectedQuickFilters( + dropdownItems.map((item) => ({ + ...item, + value: getSelectedValuesFromQuickFilter( + item, + dropdownItems, + undefined // pass in state variable + ), + })) + ); + }, [type]); useImperativeHandle(ref, () => ({ refreshAssets() { @@ -721,24 +922,46 @@ const AssetsTabs = forwardRef( } }, [isSummaryPanelOpen]); - if (isLoading || isCountLoading) { - return ( - + return ( +
+ - + - - + + - ); - } - return ( -
- {layout} + {isLoading || isCountLoading ? ( + + + + + + + + + ) : ( + layout + )} + setShowDeleteModal(false)} - onConfirm={onAssetRemove} + onConfirm={() => onAssetRemove(assetToDelete ? [assetToDelete] : [])} />
); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/assets-tabs.less b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/assets-tabs.less index 6cc55f21491b..489d473ffbff 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/assets-tabs.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/assets-tabs.less @@ -71,3 +71,11 @@ padding: 2px 6px; } } + +.assets-tab-container { + .explore-search-card { + .service-icon { + height: 16px; + } + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/SearchedData/SearchedData.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/SearchedData/SearchedData.test.tsx index 3718d43a25fd..27fbb8ff340d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/SearchedData/SearchedData.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/SearchedData/SearchedData.test.tsx @@ -125,10 +125,13 @@ describe('Test SearchedData Component', () => { const { container } = render(, { wrapper: MemoryRouter, }); + const card1 = getByTestId(container, 'table-data-card_fullyQualifiedName1'); + const card2 = getByTestId(container, 'table-data-card_fullyQualifiedName2'); + const card3 = getByTestId(container, 'table-data-card_fullyQualifiedName3'); - const searchedDataContainer = getAllByTestId(container, 'table-data-card'); - - expect(searchedDataContainer).toHaveLength(3); + expect(card1).toBeInTheDocument(); + expect(card2).toBeInTheDocument(); + expect(card3).toBeInTheDocument(); }); it('Should display table card with name and display name highlighted', () => { @@ -136,9 +139,13 @@ describe('Test SearchedData Component', () => { wrapper: MemoryRouter, }); - const searchedDataContainer = getAllByTestId(container, 'table-data-card'); + const card1 = getByTestId(container, 'table-data-card_fullyQualifiedName1'); + const card2 = getByTestId(container, 'table-data-card_fullyQualifiedName2'); + const card3 = getByTestId(container, 'table-data-card_fullyQualifiedName3'); - expect(searchedDataContainer).toHaveLength(3); + expect(card1).toBeInTheDocument(); + expect(card2).toBeInTheDocument(); + expect(card3).toBeInTheDocument(); const headerDisplayName = getAllByTestId( container, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx index 6ae6155972f6..65d59938efc1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx @@ -14,7 +14,14 @@ import { Button, Space } from 'antd'; import Tooltip, { RenderFunction } from 'antd/lib/tooltip'; import classNames from 'classnames'; -import { isString, isUndefined, lowerCase, noop, toLower } from 'lodash'; +import { + isEmpty, + isString, + isUndefined, + lowerCase, + noop, + toLower, +} from 'lodash'; import { ExtraInfo } from 'Models'; import React, { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; @@ -189,7 +196,7 @@ const EntitySummaryDetails = ({ case 'Domain': { - retVal = ( + retVal = !isEmpty(displayVal) ? ( + ) : ( + + {t('label.no-entity', { entity: t('label.domain') })} + ); } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.test.tsx index 78e2f8f1d809..a758d61d2979 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.test.tsx @@ -53,11 +53,12 @@ describe('Test TableDataCard Component', () => { source={{ id: '1', name: 'Name1', + fullyQualifiedName: 'test', }} />, { wrapper: MemoryRouter } ); - const tableDataCard = getByTestId('table-data-card'); + const tableDataCard = getByTestId('table-data-card_test'); const entityHeader = getByText('EntityHeader'); expect(tableDataCard).toBeInTheDocument(); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.tsx index b683d86685b4..2cbf68fb3cfc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.tsx @@ -18,6 +18,7 @@ import { ExtraInfo } from 'Models'; import React, { forwardRef, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { useParams } from 'react-router-dom'; +import { PRIMERY_COLOR } from '../../../constants/constants'; import { EntityType } from '../../../enums/entity.enum'; import { OwnerType } from '../../../enums/user.enum'; import { EntityReference } from '../../../generated/entity/type'; @@ -51,6 +52,8 @@ export interface TableDataCardPropsV2 { checked?: boolean; showCheckboxes?: boolean; openEntityInNewPage?: boolean; + showBody?: boolean; + showName?: boolean; } /** @@ -68,6 +71,8 @@ const TableDataCardV2: React.FC = forwardRef< matches, handleSummaryPanelDisplay, showCheckboxes, + showBody = true, + showName = true, checked, openEntityInNewPage, }, @@ -146,13 +151,18 @@ const TableDataCardV2: React.FC = forwardRef< 'table-data-card-container', className )} - data-testid="table-data-card" + data-testid={'table-data-card_' + (source.fullyQualifiedName ?? '')} id={id} ref={ref} onClick={() => { handleSummaryPanelDisplay && handleSummaryPanelDisplay(source, tab); }}> - + + {showCheckboxes && ( + + + + )} = forwardRef< icon={serviceIcon} openEntityInNewPage={openEntityInNewPage} serviceName={source?.service?.name ?? ''} + showName={showName} + titleColor={PRIMERY_COLOR} /> - {showCheckboxes && ( - - - - )} -
- -
+ {showBody && ( +
+ +
+ )} + {matches && matches.length > 0 ? (
{`${t('label.matches')}:`} diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json index 5f6224c99bd4..95451a8fad05 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Erhalten Sie die Produzenten und Verbraucher von Daten auf einer einzigen Plattform und steigern Sie die Produktivität. Die Zusammenarbeit wird besser mit Menschen und Daten aus verschiedenen Tools an zentraler Stelle.", "define-custom-property-for-entity": "Definieren Sie benutzerdefinierte Eigenschaften für {{entity}}, um Ihren organisatorischen Anforderungen gerecht zu werden.", "delete-action-description": "Das Löschen dieser {{entityType}} entfernt ihre Metadaten dauerhaft aus OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Nach dem Löschen dieser {{entityType}} wird sie dauerhaft entfernt.", "delete-entity-type-action-description": "Das Löschen dieser {{entityType}} entfernt ihre Metadaten dauerhaft aus OpenMetadata.", "delete-message-question-mark": "Nachricht löschen?", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json index 76190e921420..9221488ba761 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Get the Producers and Consumers of data in a single platform and accelerate productivity. Collaboration just gets better with people and data from multiple tools in a centralized location.", "define-custom-property-for-entity": "Define custom properties for {{entity}} to serve your organizational needs.", "delete-action-description": "Deleting this {{entityType}} will permanently remove its metadata from OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Once you delete this {{entityType}}, it will be removed permanently.", "delete-entity-type-action-description": "Deleting this {{entityType}} will permanently remove its metadata from OpenMetadata.", "delete-message-question-mark": "Delete Message?", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json index 177118b152b0..c331cf073a7e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Comprenda profundamente las relaciones de tabla; gracias al linaje a nivel de columna.", "define-custom-property-for-entity": "Defina propiedades personalizadas para {{entity}} para satisfacer las necesidades de su organización.", "delete-action-description": "Eliminar esta {{entityType}} eliminará permanentemente sus metadatos de OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Una vez que elimine esta {{entityType}}, se eliminará permanentemente.", "delete-entity-type-action-description": "Eliminar esta {{entityType}} eliminará permanentemente sua metadatoa de OpenMetadata.", "delete-message-question-mark": "¿Eliminar mensaje?", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json index d36612da1e15..a3be02c3e0f9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Comprenez en profondeur la relation entre vos tables avec la traçabilité au niveau de la colonne.", "define-custom-property-for-entity": "Définir les propriétés personnalisées pour {{entity}} pour répondre à vos besoins organisationnels.", "delete-action-description": "Supprimer cette {{entityType}} supprimera de manière permanente les métadonnées dans OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Une fois que vous supprimez {{entityType}}, il sera supprimé de manière permanente", "delete-entity-type-action-description": "Supprimer cette {{entityType}} supprimera définitivement ses métadonnées d'OpenMetadata.", "delete-message-question-mark": "Supprimer le Message ?", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json index 0c4450efb829..0fb2fbfbd2b0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "カラムレベルのリネージによりテーブルの関係をより深く理解できます。", "define-custom-property-for-entity": "Define custom properties for {{entity}} to serve your organizational needs.", "delete-action-description": "{{entityType}}の削除によってOpenMetadataからこのメタデータが削除されます。", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "この{{entityType}}を1度削除すると、これは永久に取り除かれます。", "delete-entity-type-action-description": "{{entityType}}を削除するとOpenMetadataからこのメタデータが永久に削除されます。", "delete-message-question-mark": "このメッセージを削除しますか？", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json index f1ea743ea5ce..4dd9123e9800 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Obtenha os Produtores e Consumidores de dados em uma única plataforma e acelere a produtividade. A colaboração fica ainda melhor com pessoas e dados de múltiplas ferramentas em um local centralizado.", "define-custom-property-for-entity": "Defina propriedades personalizadas para {{entity}} para atender às necessidades da sua organização.", "delete-action-description": "Excluir este {{entityType}} removerá permanentemente seus metadados do OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Uma vez que você exclua este {{entityType}}, ele será removido permanentemente.", "delete-entity-type-action-description": "Excluir este {{entityType}} removerá permanentemente seus metadados do OpenMetadata.", "delete-message-question-mark": "Excluir Mensagem?", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json index 544f9fba990c..c2ce93dc265f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Глубокое понимание связей между таблицами; благодаря отслеживанию происхождения на уровне столбцов.", "define-custom-property-for-entity": "Определите настраиваемые свойства для {{entity}} в соответствии с потребностями вашей организации.", "delete-action-description": "Удаление этого {{entityType}} навсегда удалит его метаданные из OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Как только вы удалите этот {{entityType}}, он будет удален навсегда.", "delete-entity-type-action-description": "Удаление этого {{entityType}} навсегда удалит его метаданные из OpenMetadata.", "delete-message-question-mark": "Удалить сообщение?", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json index da68eb4cb1bd..a543995d24ec 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "通过列级血缘关系图，深入了解数据表之间的关系", "define-custom-property-for-entity": "为{{entity}}添加自定义属性以满足您的组织需求", "delete-action-description": "删除此{{entityType}}将永久删除其在 OpenMetadata 中的元数据", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "一旦您删除此{{entityType}}，它将被永久删除", "delete-entity-type-action-description": "删除此{{entityType}}将永久删除其在 OpenMetadata 中的元数据", "delete-message-question-mark": "删除消息?", diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts index a455d5c0a9ec..f78c93c5c18a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts @@ -15,9 +15,10 @@ import { AxiosResponse } from 'axios'; import { Operation } from 'fast-json-patch'; import { PagingResponse } from 'Models'; import { VotingDataProps } from '../components/Voting/voting.interface'; +import { AddGlossaryToAssetsRequest } from '../generated/api/addGlossaryToAssetsRequest'; import { CreateGlossary } from '../generated/api/data/createGlossary'; import { CreateGlossaryTerm } from '../generated/api/data/createGlossaryTerm'; -import { Glossary } from '../generated/entity/data/glossary'; +import { EntityReference, Glossary } from '../generated/entity/data/glossary'; import { GlossaryTerm } from '../generated/entity/data/glossaryTerm'; import { CSVImportResult } from '../generated/type/csvImportResult'; import { EntityHistory } from '../generated/type/entityHistory'; @@ -249,3 +250,39 @@ export const updateGlossaryTermVotes = async ( return response.data; }; + +export const addAssetsToGlossaryTerm = async ( + glossaryTerm: GlossaryTerm, + assets: EntityReference[] +) => { + const data = { + assets: assets, + dryRun: false, + glossaryTags: glossaryTerm.tags ?? [], + }; + + const response = await APIClient.put< + AddGlossaryToAssetsRequest, + AxiosResponse + >(`/glossaryTerms/${glossaryTerm.id}/assets/add`, data); + + return response.data; +}; + +export const removeAssetsFromGlossaryTerm = async ( + glossaryTerm: GlossaryTerm, + assets: EntityReference[] +) => { + const data = { + assets: assets, + dryRun: false, + glossaryTags: glossaryTerm.tags ?? [], + }; + + const response = await APIClient.put< + AddGlossaryToAssetsRequest, + AxiosResponse + >(`/glossaryTerms/${glossaryTerm.id}/assets/remove`, data); + + return response.data; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx index ecfe96665815..b42c531201ef 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx @@ -88,6 +88,11 @@ export const getDropDownItems = (index: string) => { } }; +export const getAssetsPageQuickFilters = () => { + // TODO: Add more quick filters + return [...COMMON_DROPDOWN_ITEMS]; +}; + export const getAdvancedField = (field: string) => { switch (field) { case 'columns.name': diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/DomainUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/DomainUtils.tsx index bc7decc706ca..06b104b794c6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/DomainUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/DomainUtils.tsx @@ -136,6 +136,17 @@ export const getQueryFilterToIncludeDomain = ( ], }, }, + { + bool: { + must_not: [ + { + term: { + entityType: 'dataProduct', + }, + }, + ], + }, + }, ], }, }, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts index c634eac2af69..817d012074a2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts @@ -18,6 +18,7 @@ import { SearchHitCounts, } from '../components/Explore/ExplorePage.interface'; import { SearchDropdownOption } from '../components/SearchDropdown/SearchDropdown.interface'; +import { Aggregations } from '../interface/search.interface'; import { QueryFieldInterface, QueryFieldValueInterface, @@ -103,3 +104,12 @@ export const findActiveSearchIndex = ( return filteredKeys.length > 0 ? filteredKeys[0] : null; }; + +export const getAggregations = (data: Aggregations) => { + return Object.fromEntries( + Object.entries(data).map(([key, value]) => [ + key.replace('sterms#', ''), + value, + ]) + ) as Aggregations; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx index ba434930dd49..18eef834806e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx @@ -304,22 +304,38 @@ export const getEntityLink = ( export const getServiceIcon = (source: SourceType) => { if (source.entityType === EntityType.GLOSSARY_TERM) { - return ; + return ( + + ); } else if (source.entityType === EntityType.TAG) { return ( - + ); } else if (source.entityType === EntityType.DATA_PRODUCT) { return ( - + ); } else if (source.entityType === EntityType.DOMAIN) { - return ; + return ( + + ); } else { return ( service-icon );