From fea0c729eb85c7f767ef9a2a437fe1c488f55633 Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Fri, 1 Dec 2023 14:59:34 +0530 Subject: [PATCH 01/16] Fix User Startup Email --- .../java/org/openmetadata/csv/EntityCsv.java | 2 +- .../java/org/openmetadata/service/Entity.java | 7 +- .../service/jdbi3/CollectionDAO.java | 5 + .../service/jdbi3/EntityRepository.java | 29 +++-- .../service/jdbi3/GlossaryTermRepository.java | 112 +++++++++++++++++- .../glossary/GlossaryTermResource.java | 28 +++++ .../service/resources/tags/TagLabelUtil.java | 30 +++++ .../json/schema/type/bulkOperationResult.json | 71 +++++++++++ 8 files changed, 268 insertions(+), 16 deletions(-) create mode 100644 openmetadata-spec/src/main/resources/json/schema/type/bulkOperationResult.json diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 47a99887a6f9..774c580a362b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -42,12 +42,12 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TagLabel.TagSource; +import org.openmetadata.schema.type.api.BulkOperationResult.Status; import org.openmetadata.schema.type.csv.CsvDocumentation; import org.openmetadata.schema.type.csv.CsvErrorType; import org.openmetadata.schema.type.csv.CsvFile; import org.openmetadata.schema.type.csv.CsvHeader; import org.openmetadata.schema.type.csv.CsvImportResult; -import org.openmetadata.schema.type.csv.CsvImportResult.Status; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.util.EntityUtil; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 28c0029f2d22..cfc5ecf79cdd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -15,6 +15,7 @@ import static org.openmetadata.common.utils.CommonUtil.listOf; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; import static org.openmetadata.service.util.EntityUtil.getFlattenedEntityField; import com.fasterxml.jackson.annotation.JsonPropertyOrder; @@ -543,7 +544,11 @@ public static void populateEntityFieldTags( for (T c : listOrEmpty(flattenedFields)) { if (setTags) { List columnTag = allTags.get(FullyQualifiedName.buildHash(c.getFullyQualifiedName())); - c.setTags(columnTag == null ? new ArrayList<>() : columnTag); + if (columnTag == null) { + c.setTags(new ArrayList<>()); + } else { + c.setTags(addDerivedTags(columnTag)); + } } else { c.setTags(c.getTags()); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 4e9ba20d12c1..9a9bfebad5a1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -2333,6 +2333,11 @@ void renameInternal( @SqlUpdate("") void updateTagPrefixInternal(@Define("update") String update); + @SqlQuery( + "select * from tag_usage where targetFQNHash in (select targetFQNHash FROM tag_usage where tagFQNHash = :tagFQNHash) and tagFQNHash != :tagFQNHash") + @RegisterRowMapper(TagLabelMapper.class) + List getEntityTagsFromTag(@BindFQN("tagFQNHash") String tagFQNHash); + class TagLabelMapper implements RowMapper { @Override public TagLabel map(ResultSet r, StatementContext ctx) throws SQLException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index ceec49be8dff..bead76e40fae 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -1274,15 +1274,19 @@ protected void applyTags(T entity) { public void applyTags(List tagLabels, String targetFQN) { for (TagLabel tagLabel : listOrEmpty(tagLabels)) { // Apply tagLabel to targetFQN that identifies an entity or field - daoCollection - .tagUsageDAO() - .applyTag( - tagLabel.getSource().ordinal(), - tagLabel.getTagFQN(), - tagLabel.getTagFQN(), - targetFQN, - tagLabel.getLabelType().ordinal(), - tagLabel.getState().ordinal()); + boolean isTagDerived = tagLabel.getLabelType().equals(TagLabel.LabelType.DERIVED); + // Derived Tags should not create Relationships, and needs to be built on the during Read + if (!isTagDerived) { + daoCollection + .tagUsageDAO() + .applyTag( + tagLabel.getSource().ordinal(), + tagLabel.getTagFQN(), + tagLabel.getTagFQN(), + targetFQN, + tagLabel.getLabelType().ordinal(), + tagLabel.getState().ordinal()); + } } } @@ -1303,7 +1307,12 @@ protected List getTags(T entity) { } protected List getTags(String fqn) { - return !supportsTags ? null : daoCollection.tagUsageDAO().getTags(fqn); + if (!supportsTags) { + return null; + } + + // Populate Glossary Tags on Read + return addDerivedTags(daoCollection.tagUsageDAO().getTags(fqn)); } public Map> getTagsByPrefix(String prefix) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 3d3c4eed9b73..4c6b4705c787 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -21,16 +21,22 @@ import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.service.Entity.GLOSSARY; import static org.openmetadata.service.Entity.GLOSSARY_TERM; +import static org.openmetadata.service.Entity.getEntity; import static org.openmetadata.service.exception.CatalogExceptionMessage.invalidGlossaryTermMove; import static org.openmetadata.service.exception.CatalogExceptionMessage.notReviewer; +import static org.openmetadata.service.util.EntityUtil.compareTagLabel; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.service.util.EntityUtil.getId; import static org.openmetadata.service.util.EntityUtil.stringMatch; +import static org.openmetadata.service.util.EntityUtil.tagLabelMatch; import static org.openmetadata.service.util.EntityUtil.termReferenceMatch; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Optional; +import java.util.Set; import java.util.UUID; import javax.json.JsonPatch; import lombok.extern.slf4j.Slf4j; @@ -48,11 +54,14 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.ProviderType; import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TagLabel.TagSource; import org.openmetadata.schema.type.TaskDetails; import org.openmetadata.schema.type.TaskStatus; import org.openmetadata.schema.type.TaskType; import org.openmetadata.schema.type.ThreadType; +import org.openmetadata.schema.type.api.BulkOperationResult; +import org.openmetadata.schema.type.api.FailureRequest; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord; @@ -120,7 +129,7 @@ public void prepare(GlossaryTerm entity, boolean update) { // Validate parent term GlossaryTerm parentTerm = entity.getParent() != null - ? Entity.getEntity(entity.getParent().withType(GLOSSARY_TERM), "owner,reviewers", Include.NON_DELETED) + ? getEntity(entity.getParent().withType(GLOSSARY_TERM), "owner,reviewers", Include.NON_DELETED) : null; if (parentTerm != null) { parentReviewers = parentTerm.getReviewers(); @@ -128,7 +137,7 @@ public void prepare(GlossaryTerm entity, boolean update) { } // Validate glossary - Glossary glossary = Entity.getEntity(entity.getGlossary(), "reviewers", Include.NON_DELETED); + Glossary glossary = getEntity(entity.getGlossary(), "reviewers", Include.NON_DELETED); entity.setGlossary(glossary.getEntityReference()); parentReviewers = parentReviewers != null ? parentReviewers : glossary.getReviewers(); @@ -193,6 +202,68 @@ public void setFullyQualifiedName(GlossaryTerm entity) { } } + public BulkOperationResult bulkAddGlossaryToAssets( + UUID glossaryTermId, boolean dryRun, List entityReferences) { + GlossaryTerm term = this.get(null, glossaryTermId, getFields("id,tags")); + TagLabel tagLabel = + new TagLabel() + .withTagFQN(term.getFullyQualifiedName()) + .withSource(TagSource.GLOSSARY) + .withLabelType(TagLabel.LabelType.MANUAL); + + BulkOperationResult result = new BulkOperationResult().withDryRun(dryRun); + List failures = new ArrayList<>(); + List success = new ArrayList<>(); + + if (dryRun) { + // Validation for entityReferences + EntityUtil.populateEntityReferences(entityReferences); + } + + for (EntityReference ref : entityReferences) { + // Update Result Processed + result.setNumberOfRowsProcessed(result.getNumberOfRowsProcessed() + 1); + + EntityRepository entityRepository = Entity.getEntityRepository(ref.getType()); + EntityInterface original = entityRepository.get(null, ref.getId(), entityRepository.getFields("tags")); + // Original tags + List entityTags = new ArrayList<>(original.getTags()); + entityTags.add(tagLabel); + // Check if the Tags can be applied + if (dryRun) { + try { + checkMutuallyExclusive(addDerivedTags(entityTags)); + success.add(ref); + result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); + } catch (Exception ex) { + failures.add(new FailureRequest().withRequest(ref).withError(ex.getMessage())); + result.setNumberOfRowsFailed(result.getNumberOfRowsFailed() + 1); + } + } else { + // Apply Tags to Entities + entityRepository.applyTags(addDerivedTags(entityTags), original.getFullyQualifiedName()); + + // Update Result + success.add(ref); + result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); + } + } + + // Add Failed And Suceess Request + result.withFailedRequest(failures).withSuccessRequest(success); + + // Set Final Status + if (result.getNumberOfRowsPassed().equals(result.getNumberOfRowsProcessed())) { + result.withStatus(BulkOperationResult.Status.SUCCESS); + } else if (result.getNumberOfRowsPassed() > 1) { + result.withStatus(BulkOperationResult.Status.PARTIAL_SUCCESS); + } else { + result.withStatus(BulkOperationResult.Status.FAILURE); + } + + return result; + } + protected EntityReference getGlossary(GlossaryTerm term) { Relationship relationship = term.getParent() != null ? Relationship.HAS : Relationship.CONTAINS; return term.getGlossary() != null @@ -284,8 +355,8 @@ protected void closeTask(String user, CloseTask closeTask) { @Override public EntityInterface getParentEntity(GlossaryTerm entity, String fields) { return entity.getParent() != null - ? Entity.getEntity(entity.getParent(), fields, Include.NON_DELETED) - : Entity.getEntity(entity.getGlossary(), fields, Include.NON_DELETED); + ? getEntity(entity.getParent(), fields, Include.NON_DELETED) + : getEntity(entity.getGlossary(), fields, Include.NON_DELETED); } private void addGlossaryRelationship(GlossaryTerm term) { @@ -377,6 +448,39 @@ public void entitySpecificUpdate() { updateParent(original, updated); } + @Override + protected void updateTags(String fqn, String fieldName, List origTags, List updatedTags) { + // Remove current entity tags in the database. It will be added back later from the merged tag list. + origTags = listOrEmpty(origTags); + // updatedTags cannot be immutable list, as we are adding the origTags to updatedTags even if its empty. + updatedTags = Optional.ofNullable(updatedTags).orElse(new ArrayList<>()); + if (origTags.isEmpty() && updatedTags.isEmpty()) { + return; // Nothing to update + } + + // Get the list of tags that are used by + Set entityTags = new HashSet<>(daoCollection.tagUsageDAO().getEntityTagsFromTag(fqn)); + entityTags.addAll(updatedTags); + + // Check if the tags are mutually exclusive + checkMutuallyExclusive(entityTags.stream().toList()); + + // Remove current entity tags in the database. It will be added back later from the merged tag list. + daoCollection.tagUsageDAO().deleteTagsByTarget(fqn); + + if (operation.isPut()) { + // PUT operation merges tags in the request with what already exists + EntityUtil.mergeTags(updatedTags, origTags); + checkMutuallyExclusive(updatedTags); + } + + List addedTags = new ArrayList<>(); + List deletedTags = new ArrayList<>(); + recordListChange(fieldName, origTags, updatedTags, addedTags, deletedTags, tagLabelMatch); + updatedTags.sort(compareTagLabel); + applyTags(updatedTags, fqn); + } + private void updateStatus(GlossaryTerm origTerm, GlossaryTerm updatedTerm) { if (origTerm.getStatus() == updatedTerm.getStatus()) { return; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java index a95430a36782..8cf77ea4f53c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java @@ -415,6 +415,34 @@ public Response updateVote( return repository.updateVote(securityContext.getUserPrincipal().getName(), id, request).toResponse(); } + @PUT + @Path("/{id}/assets/add") + @Operation( + operationId = "bulkAddGlossaryToAssets", + summary = "Bulk Add Glossary to Assets", + description = "Bulk Add Glossary to Assets", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = ChangeEvent.class))), + @ApiResponse(responseCode = "404", description = "model for instance {id} is not found") + }) + public Response bulkAddGlossaryToAssets( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Entity", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Parameter( + description = + "Dry-run when true is used for validating the glossary without really applying it. (default=true)", + schema = @Schema(type = "boolean")) + @DefaultValue("true") + @QueryParam("dryRun") + boolean dryRun, + @Valid List request) { + return Response.ok().entity(repository.bulkAddGlossaryToAssets(id, dryRun, request)).build(); + } + @DELETE @Path("/{id}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java index 3e1256da8d3c..3e8bbc7fcaeb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java @@ -13,8 +13,13 @@ package org.openmetadata.service.resources.tags; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.type.Include.NON_DELETED; +import static org.openmetadata.service.util.EntityUtil.compareTagLabel; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.classification.Classification; import org.openmetadata.schema.entity.classification.Tag; @@ -23,6 +28,7 @@ import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TagLabel.TagSource; import org.openmetadata.service.Entity; +import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.FullyQualifiedName; @Slf4j @@ -82,4 +88,28 @@ public static boolean mutuallyExclusive(TagLabel label) { throw new IllegalArgumentException("Invalid source type " + label.getSource()); } } + + // TODO: Below two methods :addDerivedTags and :getDerivedTags can be removed from Entity Repository + public static List addDerivedTags(List tagLabels) { + if (nullOrEmpty(tagLabels)) { + return tagLabels; + } + + List updatedTagLabels = new ArrayList<>(); + EntityUtil.mergeTags(updatedTagLabels, tagLabels); + for (TagLabel tagLabel : tagLabels) { + EntityUtil.mergeTags(updatedTagLabels, getDerivedTags(tagLabel)); + } + updatedTagLabels.sort(compareTagLabel); + return updatedTagLabels; + } + + private static List getDerivedTags(TagLabel tagLabel) { + if (tagLabel.getSource() == TagLabel.TagSource.GLOSSARY) { // Related tags are only supported for Glossary + List derivedTags = Entity.getCollectionDAO().tagUsageDAO().getTags(tagLabel.getTagFQN()); + derivedTags.forEach(tag -> tag.setLabelType(TagLabel.LabelType.DERIVED)); + return derivedTags; + } + return Collections.emptyList(); + } } diff --git a/openmetadata-spec/src/main/resources/json/schema/type/bulkOperationResult.json b/openmetadata-spec/src/main/resources/json/schema/type/bulkOperationResult.json new file mode 100644 index 000000000000..cbf1eabe6aa3 --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/type/bulkOperationResult.json @@ -0,0 +1,71 @@ +{ + "$id": "https://open-metadata.org/schema/type/csvImportResult.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "BulkOperationResult", + "description": "Represents result of bulk Operation performed on entities.", + "type": "object", + "javaType": "org.openmetadata.schema.type.api.BulkOperationResult", + "definitions": { + "rowCount" : { + "description": "Type used to indicate row count", + "type": "integer", + "format" : "int64", + "minimum": 0, + "default": 0 + }, + "index" : { + "description": "Type used to indicate row number or field number. In CSV the indexes start with 1.", + "type": "integer", + "format" : "int64", + "minimum": 1 + }, + "failureRequest": { + "description": "Request that can be processed successfully.", + "type": "object", + "properties": { + "request": { + "description": "Request that can be processed successfully." + }, + "error": { + "description": "Error that occurred while processing the request.", + "type": "string" + } + }, + "additionalProperties": false + } + }, + "properties": { + "dryRun" : { + "description": "True if the operation has dryRun flag enabled", + "type" : "boolean" + }, + "status" : { + "$ref" : "basic.json#/definitions/status" + }, + "abortReason": { + "description": "Reason why import was aborted. This is set only when the `status` field is set to `aborted`", + "type" : "string" + }, + "numberOfRowsProcessed" : { + "$ref": "#/definitions/rowCount" + }, + "numberOfRowsPassed" : { + "$ref": "#/definitions/rowCount" + }, + "numberOfRowsFailed" : { + "$ref": "#/definitions/rowCount" + }, + "successRequest": { + "description": "Request that can be processed successfully." + }, + "failedRequest": { + "description": "Failure Request that can be processed successfully.", + "type": "array", + "items": { + "$ref": "#/definitions/failureRequest" + }, + "default": null + } + }, + "additionalProperties": false +} \ No newline at end of file From 996fc27e4075a07fd2876d60c657f765eb4a96d7 Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Fri, 1 Dec 2023 18:38:26 +0530 Subject: [PATCH 02/16] Add Apis for validation and addtion glossary tags and assets --- .../service/jdbi3/GlossaryTermRepository.java | 77 ++++++++++++------- .../glossary/GlossaryTermResource.java | 5 +- .../service/resources/tags/TagLabelUtil.java | 8 ++ .../api/addGlossaryToAssetsRequest.json | 29 +++++++ 4 files changed, 88 insertions(+), 31 deletions(-) create mode 100644 openmetadata-spec/src/main/resources/json/schema/api/addGlossaryToAssetsRequest.json diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 4c6b4705c787..349e660f8ec3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -24,6 +24,7 @@ import static org.openmetadata.service.Entity.getEntity; import static org.openmetadata.service.exception.CatalogExceptionMessage.invalidGlossaryTermMove; import static org.openmetadata.service.exception.CatalogExceptionMessage.notReviewer; +import static org.openmetadata.service.resources.tags.TagLabelUtil.getUniqueTags; import static org.openmetadata.service.util.EntityUtil.compareTagLabel; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.service.util.EntityUtil.getId; @@ -42,7 +43,9 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.ImmutablePair; import org.jdbi.v3.sqlobject.transaction.Transaction; +import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.api.AddGlossaryToAssetsRequest; import org.openmetadata.schema.api.data.TermReference; import org.openmetadata.schema.api.feed.CloseTask; import org.openmetadata.schema.api.feed.ResolveTask; @@ -202,50 +205,66 @@ public void setFullyQualifiedName(GlossaryTerm entity) { } } - public BulkOperationResult bulkAddGlossaryToAssets( - UUID glossaryTermId, boolean dryRun, List entityReferences) { + public BulkOperationResult bulkAddAndValidateGlossaryToAssets( + UUID glossaryTermId, AddGlossaryToAssetsRequest request) { + boolean dryRun = Boolean.TRUE.equals(request.getDryRun()); + GlossaryTerm term = this.get(null, glossaryTermId, getFields("id,tags")); - TagLabel tagLabel = - new TagLabel() - .withTagFQN(term.getFullyQualifiedName()) - .withSource(TagSource.GLOSSARY) - .withLabelType(TagLabel.LabelType.MANUAL); + + // Check if the tags are mutually exclusive for the glossary + checkMutuallyExclusive(request.getGlossaryTags()); BulkOperationResult result = new BulkOperationResult().withDryRun(dryRun); List failures = new ArrayList<>(); List success = new ArrayList<>(); - if (dryRun) { - // Validation for entityReferences - EntityUtil.populateEntityReferences(entityReferences); + if (dryRun && (CommonUtil.nullOrEmpty(request.getGlossaryTags()) || CommonUtil.nullOrEmpty(request.getAssets()))) { + // Nothing to Validate + return result.withStatus(BulkOperationResult.Status.SUCCESS).withSuccessRequest("Nothing to Validate."); } - for (EntityReference ref : entityReferences) { + // Validation for entityReferences + EntityUtil.populateEntityReferences(request.getAssets()); + + TagLabel tagLabel = + new TagLabel() + .withTagFQN(term.getFullyQualifiedName()) + .withSource(TagSource.GLOSSARY) + .withLabelType(TagLabel.LabelType.MANUAL); + + for (EntityReference ref : request.getAssets()) { // Update Result Processed result.setNumberOfRowsProcessed(result.getNumberOfRowsProcessed() + 1); EntityRepository entityRepository = Entity.getEntityRepository(ref.getType()); - EntityInterface original = entityRepository.get(null, ref.getId(), entityRepository.getFields("tags")); - // Original tags - List entityTags = new ArrayList<>(original.getTags()); - entityTags.add(tagLabel); - // Check if the Tags can be applied - if (dryRun) { - try { - checkMutuallyExclusive(addDerivedTags(entityTags)); - success.add(ref); - result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); - } catch (Exception ex) { - failures.add(new FailureRequest().withRequest(ref).withError(ex.getMessage())); - result.setNumberOfRowsFailed(result.getNumberOfRowsFailed() + 1); - } - } else { - // Apply Tags to Entities - entityRepository.applyTags(addDerivedTags(entityTags), original.getFullyQualifiedName()); + EntityInterface asset = entityRepository.get(null, ref.getId(), entityRepository.getFields("tags")); + + List allAssetTags = addDerivedTags(asset.getTags()); - // Update Result + try { + allAssetTags.addAll(request.getGlossaryTags()); + // Check Mutually Exclusive + checkMutuallyExclusive(getUniqueTags(allAssetTags)); success.add(ref); result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); + } catch (Exception ex) { + failures.add(new FailureRequest().withRequest(ref).withError(ex.getMessage())); + result.setNumberOfRowsFailed(result.getNumberOfRowsFailed() + 1); + } + // Validate and Store Tags + if (!dryRun && CommonUtil.nullOrEmpty(result.getFailedRequest())) { + allAssetTags.add(tagLabel); + // Apply Tags to Entities + entityRepository.applyTags(getUniqueTags(allAssetTags), asset.getFullyQualifiedName()); + } + } + + // Apply the tags of glossary to the glossary term + if (!dryRun && CommonUtil.nullOrEmpty(result.getFailedRequest())) { + if (!(term.getTags().isEmpty() && request.getGlossaryTags().isEmpty())) { + // Remove current entity tags in the database. It will be added back later from the merged tag list. + daoCollection.tagUsageDAO().deleteTagsByTarget(term.getFullyQualifiedName()); + applyTags(getUniqueTags(request.getGlossaryTags()), term.getFullyQualifiedName()); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java index 8cf77ea4f53c..2b01e4753f94 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java @@ -49,6 +49,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.SecurityContext; import javax.ws.rs.core.UriInfo; +import org.openmetadata.schema.api.AddGlossaryToAssetsRequest; import org.openmetadata.schema.api.VoteRequest; import org.openmetadata.schema.api.data.CreateGlossaryTerm; import org.openmetadata.schema.api.data.LoadGlossary; @@ -439,8 +440,8 @@ public Response bulkAddGlossaryToAssets( @DefaultValue("true") @QueryParam("dryRun") boolean dryRun, - @Valid List request) { - return Response.ok().entity(repository.bulkAddGlossaryToAssets(id, dryRun, request)).build(); + @Valid AddGlossaryToAssetsRequest request) { + return Response.ok().entity(repository.bulkAddAndValidateGlossaryToAssets(id, request)).build(); } @DELETE diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java index 3e8bbc7fcaeb..52950ccab31a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.TreeSet; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.classification.Classification; import org.openmetadata.schema.entity.classification.Tag; @@ -112,4 +114,10 @@ private static List getDerivedTags(TagLabel tagLabel) { } return Collections.emptyList(); } + + public static List getUniqueTags(List tags) { + Set uniqueTags = new TreeSet<>(compareTagLabel); + uniqueTags.addAll(tags); + return uniqueTags.stream().toList(); + } } diff --git a/openmetadata-spec/src/main/resources/json/schema/api/addGlossaryToAssetsRequest.json b/openmetadata-spec/src/main/resources/json/schema/api/addGlossaryToAssetsRequest.json new file mode 100644 index 000000000000..72a8865e4b6c --- /dev/null +++ b/openmetadata-spec/src/main/resources/json/schema/api/addGlossaryToAssetsRequest.json @@ -0,0 +1,29 @@ +{ + "$id": "https://open-metadata.org/schema/api/createBot.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "AddGlossaryToAssetsRequest", + "description": "Create Request for adding a glossary to assets", + "type": "object", + "javaType": "org.openmetadata.schema.api.AddGlossaryToAssetsRequest", + "properties": { + "dryRun": { + "description": "If true, the request will be validated but no changes will be made", + "type": "boolean", + "default": true + }, + "glossaryTags": { + "description": "Glossary Tags to be added", + "type": "array", + "items": { + "$ref": "../type/tagLabel.json" + }, + "default": null + }, + "assets": { + "description": "List of assets to be created against which the glossary needs to be added.", + "$ref": "../type/entityReferenceList.json" + } + }, + "required": ["glossaryTags, assets"], + "additionalProperties": false +} From deb3e320fd74f44307f7fe943a23f35a60d616db Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Fri, 1 Dec 2023 19:28:05 +0530 Subject: [PATCH 03/16] Update Status to ApiStatus --- .../src/main/java/org/openmetadata/csv/EntityCsv.java | 10 +++++----- .../service/jdbi3/GlossaryTermRepository.java | 9 +++++---- .../test/java/org/openmetadata/csv/EntityCsvTest.java | 6 +++--- .../service/resources/EntityResourceTest.java | 3 ++- .../resources/glossary/GlossaryResourceTest.java | 7 ++++--- .../service/resources/teams/TeamResourceTest.java | 11 ++++++----- .../service/resources/teams/UserResourceTest.java | 7 ++++--- .../src/main/resources/json/schema/type/basic.json | 1 + 8 files changed, 30 insertions(+), 24 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 774c580a362b..a76c9052bf5a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -38,11 +38,11 @@ import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.common.utils.CommonUtil; import org.openmetadata.schema.EntityInterface; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TagLabel.TagSource; -import org.openmetadata.schema.type.api.BulkOperationResult.Status; import org.openmetadata.schema.type.csv.CsvDocumentation; import org.openmetadata.schema.type.csv.CsvErrorType; import org.openmetadata.schema.type.csv.CsvFile; @@ -413,7 +413,7 @@ public static String invalidBoolean(int field, String fieldValue) { } private void documentFailure(String error) { - importResult.withStatus(Status.ABORTED); + importResult.withStatus(ApiStatus.ABORTED); importResult.withAbortReason(error); } @@ -435,11 +435,11 @@ protected void importFailure(CSVPrinter printer, String failedReason, CSVRecord } private void setFinalStatus() { - Status status = Status.FAILURE; + ApiStatus status = ApiStatus.FAILURE; if (importResult.getNumberOfRowsPassed().equals(importResult.getNumberOfRowsProcessed())) { - status = Status.SUCCESS; + status = ApiStatus.SUCCESS; } else if (importResult.getNumberOfRowsPassed() > 1) { - status = Status.PARTIAL_SUCCESS; + status = ApiStatus.PARTIAL_SUCCESS; } importResult.setStatus(status); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 349e660f8ec3..a2c3ab60d79c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -53,6 +53,7 @@ import org.openmetadata.schema.entity.data.GlossaryTerm; import org.openmetadata.schema.entity.data.GlossaryTerm.Status; import org.openmetadata.schema.entity.feed.Thread; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.ProviderType; @@ -220,7 +221,7 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( if (dryRun && (CommonUtil.nullOrEmpty(request.getGlossaryTags()) || CommonUtil.nullOrEmpty(request.getAssets()))) { // Nothing to Validate - return result.withStatus(BulkOperationResult.Status.SUCCESS).withSuccessRequest("Nothing to Validate."); + return result.withStatus(ApiStatus.SUCCESS).withSuccessRequest("Nothing to Validate."); } // Validation for entityReferences @@ -273,11 +274,11 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( // Set Final Status if (result.getNumberOfRowsPassed().equals(result.getNumberOfRowsProcessed())) { - result.withStatus(BulkOperationResult.Status.SUCCESS); + result.withStatus(ApiStatus.SUCCESS); } else if (result.getNumberOfRowsPassed() > 1) { - result.withStatus(BulkOperationResult.Status.PARTIAL_SUCCESS); + result.withStatus(ApiStatus.PARTIAL_SUCCESS); } else { - result.withStatus(BulkOperationResult.Status.FAILURE); + result.withStatus(ApiStatus.FAILURE); } return result; diff --git a/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java b/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java index 46406cd5ba5d..2be797df8fb8 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/csv/EntityCsvTest.java @@ -18,10 +18,10 @@ import org.mockito.Mockito; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.csv.CsvFile; import org.openmetadata.schema.type.csv.CsvHeader; import org.openmetadata.schema.type.csv.CsvImportResult; -import org.openmetadata.schema.type.csv.CsvImportResult.Status; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.TableRepository; @@ -56,14 +56,14 @@ void test_validateCsvInvalidHeader() throws IOException { String csv = ",h2,h3" + LINE_SEPARATOR; // Header h1 is missing in the CSV file TestCsv testCsv = new TestCsv(); CsvImportResult importResult = testCsv.importCsv(csv, true); - assertSummary(importResult, Status.ABORTED, 1, 0, 1); + assertSummary(importResult, ApiStatus.ABORTED, 1, 0, 1); assertNull(importResult.getImportResultsCsv()); assertEquals(TestCsv.invalidHeader("h1*,h2,h3", ",h2,h3"), importResult.getAbortReason()); } public static void assertSummary( CsvImportResult importResult, - Status expectedStatus, + ApiStatus expectedStatus, int expectedRowsProcessed, int expectedRowsPassed, int expectedRowsFailed) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java index a697da19a823..6f2ffadc667d 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java @@ -150,6 +150,7 @@ import org.openmetadata.schema.tests.TestSuite; import org.openmetadata.schema.type.AccessDetails; import org.openmetadata.schema.type.AnnouncementDetails; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.Column; @@ -2906,7 +2907,7 @@ protected void importCsvAndValidate( // Validate the imported result summary - it should include both created and updated records int totalRows = 1 + createRecords.size() + updateRecords.size(); - assertSummary(dryRunResult, CsvImportResult.Status.SUCCESS, totalRows, totalRows, 0); + assertSummary(dryRunResult, ApiStatus.SUCCESS, totalRows, totalRows, 0); String expectedResultsCsv = EntityCsvTest.createCsvResult(csvHeaders, createRecords, updateRecords); assertEquals(expectedResultsCsv, dryRunResult.getImportResultsCsv()); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index 5bf53487c279..002519a2a4ff 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -63,6 +63,7 @@ import org.openmetadata.schema.entity.data.Glossary; import org.openmetadata.schema.entity.data.GlossaryTerm; import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.Column; import org.openmetadata.schema.type.ColumnDataType; @@ -348,7 +349,7 @@ void testImportInvalidCsv() { String csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(glossaryName, csv, false); Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); String[] expectedRows = { resultsHeader, getFailedRecord(record, "[name must match \"\"^(?U)[\\w'\\- .&()%]+$\"\"]") }; @@ -359,7 +360,7 @@ record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,," csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); result = importCsv(glossaryName, csv, false); Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, entityNotFound(0, "invalidParent"))}; assertRows(result, expectedRows); @@ -367,7 +368,7 @@ record = "invalidParent,g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tier.Tier1,,," record = ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,Tag.invalidTag,,,"; csv = createCsv(GlossaryCsv.HEADERS, listOf(record), null); result = importCsv(glossaryName, csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, entityNotFound(7, "Tag.invalidTag"))}; assertRows(result, expectedRows); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java index c94a520055b0..728e7b1544b3 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/TeamResourceTest.java @@ -91,6 +91,7 @@ import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.TeamHierarchy; import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.ImageList; @@ -735,7 +736,7 @@ void testImportInvalidCsv() throws IOException { String record = getRecord(1, GROUP, team.getName(), "", false, "", "invalidPolicy"); String csv = createCsv(TeamCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); String[] expectedRows = {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(8, "invalidPolicy"))}; assertRows(result, expectedRows); @@ -743,7 +744,7 @@ void testImportInvalidCsv() throws IOException { record = getRecord(1, GROUP, team.getName(), "", false, "invalidRole", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(7, "invalidRole"))}; assertRows(result, expectedRows); @@ -751,7 +752,7 @@ record = getRecord(1, GROUP, team.getName(), "", false, "invalidRole", ""); record = getRecord(1, GROUP, team.getName(), "invalidOwner", false, "", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(5, "invalidOwner"))}; assertRows(result, expectedRows); @@ -759,7 +760,7 @@ record = getRecord(1, GROUP, team.getName(), "invalidOwner", false, "", ""); record = getRecord(1, GROUP, "invalidParent", "", false, "", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(4, "invalidParent"))}; assertRows(result, expectedRows); @@ -767,7 +768,7 @@ record = getRecord(1, GROUP, "invalidParent", "", false, "", ""); record = getRecord(1, GROUP, TEAM21.getName(), "", false, "", ""); csv = createCsv(TeamCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] { resultsHeader, getFailedRecord(record, TeamCsv.invalidTeam(4, team.getName(), "x1", TEAM21.getName())) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java index aaa2c6a36ed1..85666fd4c3e3 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/teams/UserResourceTest.java @@ -111,6 +111,7 @@ import org.openmetadata.schema.entity.teams.Team; import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.security.client.GoogleSSOClientConfig; +import org.openmetadata.schema.type.ApiStatus; import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.ImageList; @@ -991,7 +992,7 @@ void testImportInvalidCsv() throws IOException { String record = "invalid::User,,,user@domain.com,,,team-invalidCsv,"; String csv = createCsv(UserCsv.HEADERS, listOf(record), null); CsvImportResult result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); String[] expectedRows = {resultsHeader, getFailedRecord(record, "[name must match \"\"^(?U)[\\w\\-.]+$\"\"]")}; assertRows(result, expectedRows); @@ -1000,7 +1001,7 @@ void testImportInvalidCsv() throws IOException { record = "user,,,user@domain.com,,,invalidTeam,"; csv = createCsv(UserCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(6, "invalidTeam"))}; assertRows(result, expectedRows); @@ -1008,7 +1009,7 @@ record = "user,,,user@domain.com,,,invalidTeam,"; record = "user,,,user@domain.com,,,team-invalidCsv,invalidRole"; csv = createCsv(UserCsv.HEADERS, listOf(record), null); result = importCsv(team.getName(), csv, false); - assertSummary(result, CsvImportResult.Status.FAILURE, 2, 1, 1); + assertSummary(result, ApiStatus.FAILURE, 2, 1, 1); expectedRows = new String[] {resultsHeader, getFailedRecord(record, EntityCsv.entityNotFound(7, "invalidRole"))}; assertRows(result, expectedRows); } diff --git a/openmetadata-spec/src/main/resources/json/schema/type/basic.json b/openmetadata-spec/src/main/resources/json/schema/type/basic.json index 9080afc37592..fa13e00d5e65 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/basic.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/basic.json @@ -148,6 +148,7 @@ } }, "status" : { + "javaType": "org.openmetadata.schema.type.ApiStatus", "description": "State of an action over API.", "type" : "string", "enum" : ["success", "failure", "aborted", "partialSuccess"] From 67367381cf49eb380f7c8082364f11fead749d3b Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Fri, 1 Dec 2023 20:41:55 +0530 Subject: [PATCH 04/16] Unique Tags to be listed --- .../service/jdbi3/GlossaryTermRepository.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index a2c3ab60d79c..d7380e1b81c8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -33,11 +33,11 @@ import static org.openmetadata.service.util.EntityUtil.termReferenceMatch; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TreeSet; import java.util.UUID; import javax.json.JsonPatch; import lombok.extern.slf4j.Slf4j; @@ -261,12 +261,12 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( } // Apply the tags of glossary to the glossary term - if (!dryRun && CommonUtil.nullOrEmpty(result.getFailedRequest())) { - if (!(term.getTags().isEmpty() && request.getGlossaryTags().isEmpty())) { - // Remove current entity tags in the database. It will be added back later from the merged tag list. - daoCollection.tagUsageDAO().deleteTagsByTarget(term.getFullyQualifiedName()); - applyTags(getUniqueTags(request.getGlossaryTags()), term.getFullyQualifiedName()); - } + if (!dryRun + && CommonUtil.nullOrEmpty(result.getFailedRequest()) + && (!(term.getTags().isEmpty() && request.getGlossaryTags().isEmpty()))) { + // Remove current entity tags in the database. It will be added back later from the merged tag list. + daoCollection.tagUsageDAO().deleteTagsByTarget(term.getFullyQualifiedName()); + applyTags(getUniqueTags(request.getGlossaryTags()), term.getFullyQualifiedName()); } // Add Failed And Suceess Request @@ -479,7 +479,8 @@ protected void updateTags(String fqn, String fieldName, List origTags, } // Get the list of tags that are used by - Set entityTags = new HashSet<>(daoCollection.tagUsageDAO().getEntityTagsFromTag(fqn)); + Set entityTags = new TreeSet<>(compareTagLabel); + entityTags.addAll(daoCollection.tagUsageDAO().getEntityTagsFromTag(fqn)); entityTags.addAll(updatedTags); // Check if the tags are mutually exclusive From 4b46fd4521aa1bd839847bd0bbf2e9305410198a Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Fri, 1 Dec 2023 21:19:23 +0530 Subject: [PATCH 05/16] Fix in tag additon --- .../openmetadata/service/jdbi3/GlossaryTermRepository.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index d7380e1b81c8..bcb788b06188 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -243,9 +243,10 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( List allAssetTags = addDerivedTags(asset.getTags()); try { - allAssetTags.addAll(request.getGlossaryTags()); + List tempList = new ArrayList<>(allAssetTags); + tempList.addAll(request.getGlossaryTags()); // Check Mutually Exclusive - checkMutuallyExclusive(getUniqueTags(allAssetTags)); + checkMutuallyExclusive(getUniqueTags(tempList)); success.add(ref); result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); } catch (Exception ex) { From 474d90dd6cfb0e78c9a302aa02bb383d6f5dd406 Mon Sep 17 00:00:00 2001 From: karanh37 <33024356+karanh37@users.noreply.github.com> Date: Fri, 1 Dec 2023 22:03:41 +0530 Subject: [PATCH 06/16] fix: asset selection modal improvements (#14152) * fix: assets fqn issue (#14140) * fix: assets fqn issue * fix: unit tests * fix: escape fqn in remaining areas * fix: domain cypress for adding assets * fix: add assets to data products cypress * update imports * add cypress for glossary term * fix: loading in asset modal and add cypress * fix: unit tests * fix: reintroduce filters in asset selection modal --- .../ui/cypress/common/DomainUtils.js | 146 +++++++++++++++++- .../ui/cypress/constants/constants.js | 59 +++++++ .../ui/cypress/e2e/Pages/Domains.spec.js | 29 +++- .../ui/cypress/e2e/Pages/Glossary.spec.js | 86 +++++++++++ .../AssetSelectionModal.tsx | 35 ++++- .../Auth/AuthProviders/AuthProvider.tsx | 5 +- .../DataProductsDetailsPage.component.tsx | 7 +- .../DomainDetailsPage.component.tsx | 9 +- .../DataProductsTab.component.tsx | 8 +- .../ExploreSearchCard/ExploreSearchCard.tsx | 8 +- .../GlossaryHeader.component.tsx | 1 + .../GlossaryTermsV1.component.tsx | 7 +- .../GlossaryTerms/GlossaryTermsV1.test.tsx | 2 +- .../tabs/AssetsTabs.component.tsx | 17 +- .../SearchedData/SearchedData.test.tsx | 17 +- .../TableDataCardV2/TableDataCardV2.test.tsx | 3 +- .../TableDataCardV2/TableDataCardV2.tsx | 2 +- .../resources/ui/src/utils/DomainUtils.tsx | 11 ++ 18 files changed, 420 insertions(+), 32 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js b/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js index 6e6ba1d6fb75..14e31cfe1834 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/common/DomainUtils.js @@ -65,13 +65,13 @@ const checkDisplayName = (displayName) => { const checkDataProductsCount = (dataProductsCount) => { cy.get('[data-testid="data_products"] [data-testid="count"]') .scrollIntoView() - .eq(dataProductsCount); + .should('have.text', dataProductsCount); }; const checkAssetsCount = (assetsCount) => { cy.get('[data-testid="assets"] [data-testid="count"]') .scrollIntoView() - .eq(assetsCount); + .should('have.text', assetsCount); }; const updateOwner = (newOwner) => { @@ -106,6 +106,13 @@ const goToAssetsTab = (domainObj) => { cy.get('.ant-tabs-tab-active').contains('Assets').should('be.visible'); }; +const goToDataProductsTab = (domainObj) => { + cy.get('[data-testid="domain-left-panel"]').contains(domainObj.name).click(); + checkDisplayName(domainObj.name); + cy.get('[data-testid="data_products"]').click(); + cy.get('.ant-tabs-tab-active').contains('Data Products').should('be.visible'); +}; + export const updateAssets = (domainObj) => { interceptURL( 'GET', @@ -361,7 +368,7 @@ export const createDataProducts = (dataProduct, domainObj) => { cy.wait('@createDataProducts').then(({ request }) => { expect(request.body.name).equals(dataProduct.name); - expect(request.body.domain).equals(domainObj.name); + expect(request.body.domain).equals(domainObj.fullyQualifiedName); expect(request.body.description).equals(dataProduct.description); expect(request.body.experts).has.length(1); }); @@ -385,3 +392,136 @@ export const renameDomain = (domainObj) => { checkDisplayName(domainObj.updatedDisplayName); }; + +export const addAssetsToDomain = (domainObj) => { + goToAssetsTab(domainObj); + checkAssetsCount(0); + cy.contains('Adding a new Asset is easy, just give it a spin!').should( + 'be.visible' + ); + + cy.get('[data-testid="domain-details-add-button"]').click(); + cy.get('.ant-dropdown-menu .ant-dropdown-menu-title-content') + .contains('Assets') + .click(); + + cy.get('[data-testid="asset-selection-modal"] .ant-modal-title').should( + 'contain', + 'Add Assets' + ); + + domainObj.assets.forEach((asset) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + cy.get('[data-testid="asset-selection-modal"] [data-testid="searchbar"]') + .click() + .clear() + .type(asset.name); + + verifyResponseStatusCode('@searchAssets', 200); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"] input[type="checkbox"]` + ).click(); + }); + + cy.get('[data-testid="save-btn"]').click(); + + checkAssetsCount(domainObj.assets.length); +}; + +export const removeAssetsFromDomain = (domainObj) => { + goToAssetsTab(domainObj); + checkAssetsCount(domainObj.assets.length); + + domainObj.assets.forEach((asset, index) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"]` + ).within(() => { + cy.get('.explore-card-actions').invoke('show'); + cy.get('.explore-card-actions').within(() => { + cy.get('[data-testid="delete-tag"]').click(); + }); + }); + + cy.get("[data-testid='save-button']").click(); + + goToDataProductsTab(domainObj); + + interceptURL('GET', '/api/v1/search/query*', 'assetTab'); + // go assets tab + goToAssetsTab(domainObj); + verifyResponseStatusCode('@assetTab', 200); + + checkAssetsCount(domainObj.assets.length - (index + 1)); + }); +}; + +export const addAssetsToDataProduct = (dataProductObj, domainObj) => { + interceptURL('GET', `/api/v1/search/query**`, 'getDataProductAssets'); + + goToDataProductsTab(domainObj); + cy.get(`[data-testid="explore-card-${dataProductObj.name}"]`).click(); + + cy.get('[data-testid="assets"]').should('be.visible').click(); + cy.get('.ant-tabs-tab-active').contains('Assets').should('be.visible'); + + verifyResponseStatusCode('@getDataProductAssets', 200); + + cy.contains('Adding a new Asset is easy, just give it a spin!').should( + 'be.visible' + ); + + cy.get('[data-testid="data-product-details-add-button"]').click(); + + cy.get('[data-testid="asset-selection-modal"] .ant-modal-title').should( + 'contain', + 'Add Assets' + ); + + dataProductObj.assets.forEach((asset) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + cy.get('[data-testid="asset-selection-modal"] [data-testid="searchbar"]') + .click() + .clear() + .type(asset.name); + + verifyResponseStatusCode('@searchAssets', 200); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"] input[type="checkbox"]` + ).click(); + }); + + cy.get('[data-testid="save-btn"]').click(); + + checkAssetsCount(dataProductObj.assets.length); +}; + +export const removeAssetsFromDataProduct = (dataProductObj, domainObj) => { + goToDataProductsTab(domainObj); + cy.get(`[data-testid="explore-card-${dataProductObj.name}"]`).click(); + + cy.get('[data-testid="assets"]').should('be.visible').click(); + cy.get('.ant-tabs-tab-active').contains('Assets').should('be.visible'); + + checkAssetsCount(dataProductObj.assets.length); + + dataProductObj.assets.forEach((asset, index) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"]` + ).within(() => { + cy.get('.explore-card-actions').invoke('show'); + cy.get('.explore-card-actions').within(() => { + cy.get('[data-testid="delete-tag"]').click(); + }); + }); + + cy.get("[data-testid='save-button']").click(); + + checkAssetsCount(domainObj.assets.length - (index + 1)); + }); +}; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js b/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js index 2a16c0613554..3c62dde6f00c 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/constants/constants.js @@ -327,6 +327,26 @@ export const NEW_GLOSSARY_TERMS = { synonyms: 'give,disposal,deal', fullyQualifiedName: 'Cypress Glossary.CypressSales', }, + term_3: { + name: 'Cypress Space', + description: 'This is the Cypress with space', + synonyms: 'tea,coffee,water', + fullyQualifiedName: 'Cypress Glossary.Cypress Space', + assets: [ + { + name: 'dim_customer', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address', + }, + { + name: 'raw_order', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.raw_order', + }, + { + name: 'presto_etl', + fullyQualifiedName: 'sample_airflow.presto_etl', + }, + ], + }, }; export const GLOSSARY_TERM_WITH_DETAILS = { name: 'Accounts', @@ -518,6 +538,7 @@ export const NAME_MAX_LENGTH_VALIDATION_ERROR = export const DOMAIN_1 = { name: 'Cypress%Domain', updatedName: 'Cypress_Domain_Name', + fullyQualifiedName: 'Cypress%Domain', updatedDisplayName: 'Cypress_Domain_Display_Name', description: 'This is the Cypress for testing domain creation with percent and dot', @@ -549,8 +570,46 @@ export const DOMAIN_2 = { name: 'Cypress.Domain.New', updatedName: 'Cypress.Domain.New', updatedDisplayName: 'Cypress.Domain.New', + fullyQualifiedName: '"Cypress.Domain.New"', description: 'This is the Cypress for testing domain creation', experts: 'Alex Pollard', owner: 'Alex Pollard', domainType: 'Source-aligned', + dataProducts: [ + { + name: 'Cypress DataProduct Assets', + description: + 'This is the data product description for Cypress DataProduct Assets', + experts: 'Aaron Johnson', + owner: 'Aaron Johnson', + assets: [ + { + name: 'dim_customer', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address', + }, + { + name: 'raw_order', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.raw_order', + }, + { + name: 'presto_etl', + fullyQualifiedName: 'sample_airflow.presto_etl', + }, + ], + }, + ], + assets: [ + { + name: 'dim_customer', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address', + }, + { + name: 'raw_order', + fullyQualifiedName: 'sample_data.ecommerce_db.shopify.raw_order', + }, + { + name: 'presto_etl', + fullyQualifiedName: 'sample_airflow.presto_etl', + }, + ], }; diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js index b833dda73f04..48a23ce5eacb 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Domains.spec.js @@ -14,10 +14,14 @@ /// import { + addAssetsToDataProduct, + addAssetsToDomain, createDataProducts, createDomain, deleteDomain, removeAssets, + removeAssetsFromDataProduct, + removeAssetsFromDomain, renameDomain, updateAssets, updateDomainDetails, @@ -44,6 +48,10 @@ describe('Domain page should work properly', () => { verifyDomain(DOMAIN_2); }); + it('Add assets to domain using asset selection modal should work properly', () => { + addAssetsToDomain(DOMAIN_2); + }); + it('Create new data product should work properly', () => { DOMAIN_1.dataProducts.forEach((dataProduct) => { createDataProducts(dataProduct, DOMAIN_1); @@ -53,15 +61,34 @@ describe('Domain page should work properly', () => { }); }); + it('Add data product assets using asset selection modal should work properly', () => { + DOMAIN_2.dataProducts.forEach((dp) => { + createDataProducts(dp, DOMAIN_2); + cy.get('[data-testid="app-bar-item-domain"]') + .should('be.visible') + .click({ force: true }); + }); + + addAssetsToDataProduct(DOMAIN_2.dataProducts[0], DOMAIN_2); + }); + + it('Remove data product assets using asset selection modal should work properly', () => { + removeAssetsFromDataProduct(DOMAIN_2.dataProducts[0], DOMAIN_2); + }); + it('Update domain details should work properly', () => { updateDomainDetails(DOMAIN_1); }); + it('Remove assets to domain using asset selection modal should work properly', () => { + removeAssetsFromDomain(DOMAIN_2); + }); + it('Assets Tab should work properly', () => { updateAssets(DOMAIN_1); }); - it.skip('Remove Domain from entity should work properly', () => { + it('Remove Domain from entity should work properly', () => { removeAssets(DOMAIN_1); }); diff --git a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js index c491a5b921ae..990c19557c27 100644 --- a/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js +++ b/openmetadata-ui/src/main/resources/ui/cypress/e2e/Pages/Glossary.spec.js @@ -88,6 +88,12 @@ const checkDisplayName = (displayName) => { }); }; +const checkAssetsCount = (assetsCount) => { + cy.get('[data-testid="assets"] [data-testid="count"]') + .scrollIntoView() + .should('have.text', assetsCount); +}; + const validateForm = () => { // error messages cy.get('#name_help') @@ -967,6 +973,86 @@ describe('Glossary page should work properly', () => { .should('be.visible'); }); + it('Add asset to glossary term using asset modal', () => { + selectActiveGlossary(NEW_GLOSSARY.name); + goToAssetsTab( + NEW_GLOSSARY_TERMS.term_3.name, + NEW_GLOSSARY_TERMS.term_3.fullyQualifiedName, + true + ); + + checkAssetsCount(0); + cy.contains('Adding a new Asset is easy, just give it a spin!').should( + 'be.visible' + ); + + cy.get('[data-testid="glossary-term-add-button-menu"]').click(); + cy.get('.ant-dropdown-menu .ant-dropdown-menu-title-content') + .contains('Assets') + .click(); + + cy.get('[data-testid="asset-selection-modal"] .ant-modal-title').should( + 'contain', + 'Add Assets' + ); + + NEW_GLOSSARY_TERMS.term_3.assets.forEach((asset) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + cy.get('[data-testid="asset-selection-modal"] [data-testid="searchbar"]') + .click() + .clear() + .type(asset.name); + + verifyResponseStatusCode('@searchAssets', 200); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"] input[type="checkbox"]` + ).click(); + }); + + cy.get('[data-testid="save-btn"]').click(); + + checkAssetsCount(NEW_GLOSSARY_TERMS.term_3.assets); + }); + + it('Remove asset from glossary term using asset modal', () => { + selectActiveGlossary(NEW_GLOSSARY.name); + goToAssetsTab( + NEW_GLOSSARY_TERMS.term_3.name, + NEW_GLOSSARY_TERMS.term_3.fullyQualifiedName, + true + ); + + checkAssetsCount(NEW_GLOSSARY_TERMS.term_3.assets.length); + NEW_GLOSSARY_TERMS.term_3.assets.assets.forEach((asset, index) => { + interceptURL('GET', '/api/v1/search/query*', 'searchAssets'); + + cy.get( + `[data-testid="table-data-card_${asset.fullyQualifiedName}"]` + ).within(() => { + cy.get('.explore-card-actions').invoke('show'); + cy.get('.explore-card-actions').within(() => { + cy.get('[data-testid="delete-tag"]').click(); + }); + }); + + cy.get("[data-testid='save-button']").click(); + + selectActiveGlossary(NEW_GLOSSARY.name); + + interceptURL('GET', '/api/v1/search/query*', 'assetTab'); + // go assets tab + goToAssetsTab( + NEW_GLOSSARY_TERMS.term_3.name, + NEW_GLOSSARY_TERMS.term_3.fullyQualifiedName, + true + ); + verifyResponseStatusCode('@assetTab', 200); + + checkAssetsCount(NEW_GLOSSARY_TERMS.term_3.assets.length - (index + 1)); + }); + }); + it('Remove Glossary term from entity should work properly', () => { const glossaryName = NEW_GLOSSARY_1.name; const { name, fullyQualifiedName } = NEW_GLOSSARY_1_TERMS.term_1; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx index be0d2026ccb2..57bd7d833c21 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx @@ -10,9 +10,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Button, List, Modal, Space, Typography } from 'antd'; +import { Button, List, Modal, Select, Space, Typography } from 'antd'; import { AxiosError } from 'axios'; import { compare } from 'fast-json-patch'; +import { map, startCase } from 'lodash'; import { EntityDetailUnion } from 'Models'; import VirtualList from 'rc-virtual-list'; import { @@ -20,6 +21,7 @@ import { UIEventHandler, useCallback, useEffect, + useMemo, useState, } from 'react'; import { useTranslation } from 'react-i18next'; @@ -37,6 +39,7 @@ import { searchQuery } from '../../../rest/searchAPI'; import { getAPIfromSource, getAssetsFields, + getAssetsSearchIndex, getEntityAPIfromSource, } from '../../../utils/Assets/AssetsUtils'; import { getEntityReferenceFromEntity } from '../../../utils/EntityUtils'; @@ -358,6 +361,10 @@ export const AssetSelectionModal = ({ } }, [type, handleSave, domainAndDataProductsSave]); + const mapAssetsSearchIndex = useMemo(() => { + return getAssetsSearchIndex(type); + }, [type]); + const onScroll: UIEventHandler = useCallback( (e) => { const scrollHeight = @@ -392,10 +399,14 @@ export const AssetSelectionModal = ({ destroyOnClose closable={false} closeIcon={null} + data-testid="asset-selection-modal" footer={ <> - + - - +
+
+ {selectedItems && selectedItems.size > 1 && ( + + {selectedItems.size} {t('label.selected-lowercase')} + + )} +
+ +
+ + +
+
} open={open} style={{ top: 40 }} title={t('label.add-entity', { entity: t('label.asset-plural') })} - width={750} + width={675} onCancel={onCancel}> ({ - label: startCase(key), - value: value, - }))} - style={{ minWidth: '100px' }} - value={activeFilter} - onChange={setActiveFilter} - /> - ), - }} placeholder={t('label.search-entity', { entity: t('label.asset-plural'), })} @@ -446,27 +429,39 @@ export const AssetSelectionModal = ({ /> {items.length > 0 && ( - - - {({ _source: item }) => ( - - )} - - +
+ onSelectAll(e.target.checked)}> + {t('label.select-field', { + field: t('label.all'), + })} + + + + {({ _source: item }) => ( + + )} + + +
)} + {!isLoading && items.length === 0 && ( {emptyPlaceHolderText && ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/asset-selection-model.style.less b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/asset-selection-model.style.less index 46fac30167de..5b725ff3682a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/asset-selection-model.style.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/asset-selection-model.style.less @@ -24,15 +24,23 @@ // CheckBox .ant-checkbox-inner { border-width: 2px; - height: 18px; - width: 18px; + height: 20px; + width: 20px; border-color: @primary-color; } .ant-checkbox-inner::after { - top: 48%; + top: 44%; left: 18.5%; - width: 6.25px; - height: 10px; + width: 6.5px; + height: 11px; + } + + .data-asset-info-row { + gap: 16px; + } + + .entity-header-display-name { + font-weight: 500 !important; } } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeader/EntityHeader.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeader/EntityHeader.component.tsx index 4b2bff18416d..ddded7b20f2c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeader/EntityHeader.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeader/EntityHeader.component.tsx @@ -35,6 +35,7 @@ interface Props { serviceName: string; titleColor?: string; badge?: React.ReactNode; + showName?: boolean; } export const EntityHeader = ({ @@ -48,6 +49,7 @@ export const EntityHeader = ({ serviceName, badge, titleColor, + showName = true, }: Props) => { return (
@@ -74,6 +76,7 @@ export const EntityHeader = ({ name={entityData.name} openEntityInNewPage={openEntityInNewPage} serviceName={serviceName} + showName={showName} />
); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.component.tsx index 35fc94e0876a..f13bff49575d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.component.tsx @@ -34,6 +34,7 @@ const EntityHeaderTitle = ({ isDisabled, className, color, + showName = true, }: EntityHeaderTitleProps) => { const { t } = useTranslation(); const location = useLocation(); @@ -51,9 +52,9 @@ const EntityHeaderTitle = ({ gutter={12} wrap={false}> {icon} - + {/* If we do not have displayName name only be shown in the bold from the below code */} - {!isEmpty(displayName) ? ( + {!isEmpty(displayName) && showName ? ( @@ -98,7 +99,7 @@ const EntityHeaderTitle = ({ return link && !isTourRoute ? ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.interface.ts index 69057e78699e..02462f5fbed1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityHeaderTitle/EntityHeaderTitle.interface.ts @@ -22,4 +22,5 @@ export interface EntityHeaderTitleProps { serviceName: string; badge?: React.ReactNode; isDisabled?: boolean; + showName?: boolean; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts index d165e95e7016..472c6f5baec1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/ExploreQuickFilters.interface.ts @@ -20,7 +20,7 @@ export interface ExploreQuickFiltersProps { fields: Array; aggregations?: Aggregations; onFieldValueSelect: (field: ExploreQuickFilterField) => void; - onAdvanceSearch: () => void; + onAdvanceSearch?: () => void; showDeleted?: boolean; onChangeShowDeleted?: (showDeleted: boolean) => void; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.interface.ts index 79539a50c3ba..1dba6ed64737 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.interface.ts @@ -32,4 +32,5 @@ export interface ExploreSearchCardProps { openEntityInNewPage?: boolean; hideBreadcrumbs?: boolean; actionPopoverContent?: React.ReactNode; + onCheckboxChange?: (checked: boolean) => void; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx index a182b32a8467..26031a466257 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/ExploreSearchCard/ExploreSearchCard.tsx @@ -10,7 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Button, Col, Row, Space, Typography } from 'antd'; +import { Button, Checkbox, Col, Row, Space, Typography } from 'antd'; import classNames from 'classnames'; import { isString, startCase, uniqueId } from 'lodash'; import { ExtraInfo } from 'Models'; @@ -59,6 +59,9 @@ const ExploreSearchCard: React.FC = forwardRef< openEntityInNewPage, hideBreadcrumbs = false, actionPopoverContent, + showCheckboxes = false, + checked = false, + onCheckboxChange, }, ref ) => { @@ -163,8 +166,17 @@ const ExploreSearchCard: React.FC = forwardRef< const header = useMemo(() => { return ( + {showCheckboxes && ( + + onCheckboxChange?.(e.target.checked)} + /> + + )} {!hideBreadcrumbs && ( - +
{serviceIcon}
@@ -194,7 +206,7 @@ const ExploreSearchCard: React.FC = forwardRef< {entityIcon} = forwardRef< ); - }, [breadcrumbs, source, hideBreadcrumbs]); + }, [breadcrumbs, source, hideBreadcrumbs, showCheckboxes, checked]); return (
>( {} as Record ); + const [api, contextHolder] = notification.useNotification(); + const [activeFilter, setActiveFilter] = useState([]); const { fqn } = useParams<{ fqn: string }>(); const [isLoading, setIsLoading] = useState(true); @@ -149,6 +171,25 @@ const AssetsTabs = forwardRef( Domain | DataProduct | GlossaryTerm >(); + const [selectedItems, setSelectedItems] = + useState>(); + const [aggregations, setAggregations] = useState(); + const [selectedQuickFilters, setSelectedQuickFilters] = useState< + ExploreQuickFilterField[] + >([] as ExploreQuickFilterField[]); + + const parsedQueryString = Qs.parse( + location.search.startsWith('?') + ? location.search.substr(1) + : location.search + ); + + const searchQuery = isString(parsedQueryString.search) + ? parsedQueryString.search + : ''; + + const [searchValue, setSearchValue] = useState(searchQuery); + const queryParam = useMemo(() => { switch (type) { case AssetsOfEntity.DOMAIN: @@ -184,7 +225,7 @@ const AssetsTabs = forwardRef( try { setIsLoading(true); const res = await searchData( - '', + searchValue, page, pageSize, queryParam, @@ -193,6 +234,8 @@ const AssetsTabs = forwardRef( index ); + setAggregations(getAggregations(res?.data.aggregations)); + // Extract useful details from the Response const totalCount = res?.data?.hits?.total.value ?? 0; const hits = res?.data?.hits?.hits; @@ -219,7 +262,7 @@ const AssetsTabs = forwardRef( setIsLoading(false); } }, - [activeFilter, currentPage, pageSize] + [activeFilter, currentPage, pageSize, searchValue] ); const onOpenChange: MenuProps['onOpenChange'] = (keys) => { const latestOpenKey = keys.find( @@ -298,6 +341,27 @@ const AssetsTabs = forwardRef( [itemCount] ); + const items: ItemType[] = [ + { + label: ( + } + id="delete-button" + name={t('label.delete')} + /> + ), + key: 'delete-button', + onClick: () => { + if (selectedCard) { + onExploreCardDelete(selectedCard); + } + }, + }, + ]; + const getOptions = useCallback( ( option: { @@ -377,6 +441,22 @@ const AssetsTabs = forwardRef( } }, [type]); + const handleCheckboxChange = ( + selected: boolean, + source: EntityDetailUnion + ) => { + setSelectedItems((prevItems) => { + const selectedItemMap = new Map(prevItems ?? []); + if (selected && source.id) { + selectedItemMap.set(source.id, source); + } else if (source.id) { + selectedItemMap.delete(source.id); + } + + return selectedItemMap; + }); + }; + const subMenuItems = useMemo(() => { return filteredAssetMenus.map((option) => ({ ...getOptions(option), @@ -415,6 +495,12 @@ const AssetsTabs = forwardRef( } }; + const deleteSelectedItems = useCallback(() => { + selectedItems?.forEach((item) => { + onAssetRemove(item); + }); + }, [selectedItems]); + useEffect(() => { fetchCountsByEntity(); @@ -429,6 +515,15 @@ const AssetsTabs = forwardRef( } }, [entityFqn]); + useEffect(() => { + if (selectedItems) { + notification.close('asset-tab-notification-key'); + if (selectedItems.size > 1) { + openNotification(); + } + } + }, [selectedItems]); + const assetErrorPlaceHolder = useMemo(() => { if (!isEmpty(activeFilter)) { return ( @@ -437,7 +532,7 @@ const AssetsTabs = forwardRef( type={ERROR_PLACEHOLDER_TYPE.FILTER} /> ); - } else if (noDataPlaceholder) { + } else if (noDataPlaceholder || searchValue) { return ( {isObject(noDataPlaceholder) && ( @@ -495,37 +590,46 @@ const AssetsTabs = forwardRef( } }, [ activeFilter, + searchValue, noDataPlaceholder, permissions, onAddAsset, isEntityDeleted, ]); + const renderDropdownContainer = useCallback((menus) => { + return
{menus}
; + }, []); + const assetListing = useMemo( () => data.length ? (
{data.map(({ _source, _id = '' }) => ( - } - size="small" - type="text" - onClick={() => onExploreCardDelete(_source)} - /> + + + ) : null } + checked={selectedItems?.has(_source.id ?? '')} className={classNames( 'm-b-sm cursor-pointer', selectedCard?.id === _source.id ? 'highlight-card' : '' @@ -535,6 +639,9 @@ const AssetsTabs = forwardRef( key={'assets_' + _id} showTags={false} source={_source} + onCheckboxChange={(selected) => + handleCheckboxChange(selected, _source) + } /> ))} {showPagination && ( @@ -561,10 +668,12 @@ const AssetsTabs = forwardRef( currentPage, selectedCard, assetErrorPlaceHolder, + selectedItems, setSelectedCard, handlePageChange, showPagination, handlePageSizeChange, + handleCheckboxChange, ] ); @@ -637,71 +746,108 @@ const AssetsTabs = forwardRef( ); }, [assetsHeader, assetListing, selectedCard]); - const onAssetRemove = useCallback(async () => { - if (!activeEntity) { - return; - } + const onAssetRemove = useCallback( + async (data: SourceType) => { + if (!activeEntity) { + return; + } - try { - let updatedEntity; - switch (type) { - case AssetsOfEntity.DATA_PRODUCT: - const updatedAssets = ( - (activeEntity as DataProduct)?.assets ?? [] - ).filter((asset) => asset.id !== assetToDelete?.id); - updatedEntity = { - ...activeEntity, - assets: updatedAssets, - }; - const jsonPatch = compare(activeEntity, updatedEntity); - const res = await patchDataProduct( - (activeEntity as DataProduct).id, - jsonPatch - ); - setActiveEntity(res); - - break; - - case AssetsOfEntity.DOMAIN: - case AssetsOfEntity.GLOSSARY: - const selectedItemMap = new Map(); - selectedItemMap.set(assetToDelete?.id, assetToDelete); - - if (type === AssetsOfEntity.DOMAIN) { - await updateDomainAssets(undefined, type, selectedItemMap); - } else if (type === AssetsOfEntity.GLOSSARY) { - await removeGlossaryTermAssets( - entityFqn ?? '', - type, - selectedItemMap + try { + let updatedEntity; + switch (type) { + case AssetsOfEntity.DATA_PRODUCT: + const updatedAssets = ( + (activeEntity as DataProduct)?.assets ?? [] + ).filter((asset) => asset.id !== data?.id); + updatedEntity = { + ...activeEntity, + assets: updatedAssets, + }; + const jsonPatch = compare(activeEntity, updatedEntity); + const res = await patchDataProduct( + (activeEntity as DataProduct).id, + jsonPatch ); - } + setActiveEntity(res); + + break; + + case AssetsOfEntity.DOMAIN: + case AssetsOfEntity.GLOSSARY: + const selectedItemMap = new Map(); + selectedItemMap.set(data?.id, data); + + if (type === AssetsOfEntity.DOMAIN) { + await updateDomainAssets(undefined, type, selectedItemMap); + } else if (type === AssetsOfEntity.GLOSSARY) { + await removeGlossaryTermAssets( + entityFqn ?? '', + type, + selectedItemMap + ); + } + + break; + default: + // Handle other entity types here + break; + } - break; - default: - // Handle other entity types here - break; + await new Promise((resolve) => { + setTimeout(() => { + resolve(''); + }, ES_UPDATE_DELAY); + }); + } catch (err) { + showErrorToast(err as AxiosError); + } finally { + setShowDeleteModal(false); + onRemoveAsset?.(); } + }, + [type, activeEntity, entityFqn] + ); - await new Promise((resolve) => { - setTimeout(() => { - resolve(''); - }, ES_UPDATE_DELAY); - }); - } catch (err) { - showErrorToast(err as AxiosError); - } finally { - setShowDeleteModal(false); - onRemoveAsset?.(); - } - }, [type, activeEntity, assetToDelete, entityFqn]); + const openNotification = () => { + api.warning({ + key: 'asset-tab-notification-key', + message: ( +
+ {selectedItems && selectedItems.size > 1 && ( + + {selectedItems.size} {t('label.selected-lowercase')} + + )} + +
+ ), + placement: 'bottom', + className: 'asset-tab-delete-notification', + duration: 0, + }); + }; useEffect(() => { fetchAssets({ index: isEmpty(activeFilter) ? [SearchIndex.ALL] : activeFilter, page: currentPage, }); - }, [activeFilter, currentPage, pageSize]); + }, [activeFilter, currentPage, pageSize, searchValue]); + + useEffect(() => { + const dropdownItems = getAssetsPageQuickFilters(); + + setSelectedQuickFilters( + dropdownItems.map((item) => ({ + ...item, + value: getSelectedValuesFromQuickFilter( + item, + dropdownItems, + undefined // pass in state variable + ), + })) + ); + }, [type]); useImperativeHandle(ref, () => ({ refreshAssets() { @@ -728,39 +874,64 @@ const AssetsTabs = forwardRef( } }, [isSummaryPanelOpen]); - if (isLoading || isCountLoading) { - return ( - - - - - - - - - ); - } - return ( -
- {layout} - setShowDeleteModal(false)} - onConfirm={onAssetRemove} - /> -
+ + {contextHolder} +
+ + + + + + + + + + {isLoading || isCountLoading ? ( + + + + + + + + + ) : ( + layout + )} + + setShowDeleteModal(false)} + onConfirm={() => onAssetRemove(assetToDelete as SourceType)} + /> +
+ ); } ); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/assets-tabs.less b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/assets-tabs.less index 6cc55f21491b..489d473ffbff 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/assets-tabs.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/assets-tabs.less @@ -71,3 +71,11 @@ padding: 2px 6px; } } + +.assets-tab-container { + .explore-search-card { + .service-icon { + height: 16px; + } + } +} diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.tsx index 11c9cc5e30de..2cbf68fb3cfc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/TableDataCardV2/TableDataCardV2.tsx @@ -18,6 +18,7 @@ import { ExtraInfo } from 'Models'; import React, { forwardRef, useMemo } from 'react'; import { useTranslation } from 'react-i18next'; import { useParams } from 'react-router-dom'; +import { PRIMERY_COLOR } from '../../../constants/constants'; import { EntityType } from '../../../enums/entity.enum'; import { OwnerType } from '../../../enums/user.enum'; import { EntityReference } from '../../../generated/entity/type'; @@ -51,6 +52,8 @@ export interface TableDataCardPropsV2 { checked?: boolean; showCheckboxes?: boolean; openEntityInNewPage?: boolean; + showBody?: boolean; + showName?: boolean; } /** @@ -68,6 +71,8 @@ const TableDataCardV2: React.FC = forwardRef< matches, handleSummaryPanelDisplay, showCheckboxes, + showBody = true, + showName = true, checked, openEntityInNewPage, }, @@ -152,7 +157,12 @@ const TableDataCardV2: React.FC = forwardRef< onClick={() => { handleSummaryPanelDisplay && handleSummaryPanelDisplay(source, tab); }}> - + + {showCheckboxes && ( + + + + )} = forwardRef< icon={serviceIcon} openEntityInNewPage={openEntityInNewPage} serviceName={source?.service?.name ?? ''} + showName={showName} + titleColor={PRIMERY_COLOR} /> - {showCheckboxes && ( - - - - )} -
- -
+ {showBody && ( +
+ +
+ )} + {matches && matches.length > 0 ? (
{`${t('label.matches')}:`} diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json index 76190e921420..9221488ba761 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Get the Producers and Consumers of data in a single platform and accelerate productivity. Collaboration just gets better with people and data from multiple tools in a centralized location.", "define-custom-property-for-entity": "Define custom properties for {{entity}} to serve your organizational needs.", "delete-action-description": "Deleting this {{entityType}} will permanently remove its metadata from OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Once you delete this {{entityType}}, it will be removed permanently.", "delete-entity-type-action-description": "Deleting this {{entityType}} will permanently remove its metadata from OpenMetadata.", "delete-message-question-mark": "Delete Message?", diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts index a455d5c0a9ec..8857809c5ab1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts @@ -17,7 +17,11 @@ import { PagingResponse } from 'Models'; import { VotingDataProps } from '../components/Voting/voting.interface'; import { CreateGlossary } from '../generated/api/data/createGlossary'; import { CreateGlossaryTerm } from '../generated/api/data/createGlossaryTerm'; -import { Glossary } from '../generated/entity/data/glossary'; +import { + EntityReference, + Glossary, + TagLabel, +} from '../generated/entity/data/glossary'; import { GlossaryTerm } from '../generated/entity/data/glossaryTerm'; import { CSVImportResult } from '../generated/type/csvImportResult'; import { EntityHistory } from '../generated/type/entityHistory'; @@ -249,3 +253,27 @@ export const updateGlossaryTermVotes = async ( return response.data; }; + +type AssetsData = { + assets: EntityReference[]; + dryRun: boolean; + glossaryTags: TagLabel[]; +}; + +export const addAssetsToGlossaryTerm = async ( + glossaryTerm: GlossaryTerm, + assets: EntityReference[] +) => { + const data = { + assets: assets, + dryRun: false, + glossaryTags: glossaryTerm.tags ?? [], + }; + + const response = await APIClient.put>( + `/glossaryTerms/${glossaryTerm.id}/assets/add`, + data + ); + + return response.data; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx index ecfe96665815..b42c531201ef 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchUtils.tsx @@ -88,6 +88,11 @@ export const getDropDownItems = (index: string) => { } }; +export const getAssetsPageQuickFilters = () => { + // TODO: Add more quick filters + return [...COMMON_DROPDOWN_ITEMS]; +}; + export const getAdvancedField = (field: string) => { switch (field) { case 'columns.name': diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts index c634eac2af69..817d012074a2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/Explore.utils.ts @@ -18,6 +18,7 @@ import { SearchHitCounts, } from '../components/Explore/ExplorePage.interface'; import { SearchDropdownOption } from '../components/SearchDropdown/SearchDropdown.interface'; +import { Aggregations } from '../interface/search.interface'; import { QueryFieldInterface, QueryFieldValueInterface, @@ -103,3 +104,12 @@ export const findActiveSearchIndex = ( return filteredKeys.length > 0 ? filteredKeys[0] : null; }; + +export const getAggregations = (data: Aggregations) => { + return Object.fromEntries( + Object.entries(data).map(([key, value]) => [ + key.replace('sterms#', ''), + value, + ]) + ) as Aggregations; +}; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx index ba434930dd49..18eef834806e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx @@ -304,22 +304,38 @@ export const getEntityLink = ( export const getServiceIcon = (source: SourceType) => { if (source.entityType === EntityType.GLOSSARY_TERM) { - return ; + return ( + + ); } else if (source.entityType === EntityType.TAG) { return ( - + ); } else if (source.entityType === EntityType.DATA_PRODUCT) { return ( - + ); } else if (source.entityType === EntityType.DOMAIN) { - return ; + return ( + + ); } else { return ( service-icon ); From 3b8a5cb75d59d46cdd6229900de55fa2cfe23fb9 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Sat, 2 Dec 2023 14:41:31 +0530 Subject: [PATCH 08/16] localization --- .../src/main/resources/ui/src/locale/languages/de-de.json | 1 + .../src/main/resources/ui/src/locale/languages/es-es.json | 1 + .../src/main/resources/ui/src/locale/languages/fr-fr.json | 1 + .../src/main/resources/ui/src/locale/languages/ja-jp.json | 1 + .../src/main/resources/ui/src/locale/languages/pt-br.json | 1 + .../src/main/resources/ui/src/locale/languages/ru-ru.json | 1 + .../src/main/resources/ui/src/locale/languages/zh-cn.json | 1 + 7 files changed, 7 insertions(+) diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json index 5f6224c99bd4..95451a8fad05 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Erhalten Sie die Produzenten und Verbraucher von Daten auf einer einzigen Plattform und steigern Sie die Produktivität. Die Zusammenarbeit wird besser mit Menschen und Daten aus verschiedenen Tools an zentraler Stelle.", "define-custom-property-for-entity": "Definieren Sie benutzerdefinierte Eigenschaften für {{entity}}, um Ihren organisatorischen Anforderungen gerecht zu werden.", "delete-action-description": "Das Löschen dieser {{entityType}} entfernt ihre Metadaten dauerhaft aus OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Nach dem Löschen dieser {{entityType}} wird sie dauerhaft entfernt.", "delete-entity-type-action-description": "Das Löschen dieser {{entityType}} entfernt ihre Metadaten dauerhaft aus OpenMetadata.", "delete-message-question-mark": "Nachricht löschen?", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json index 177118b152b0..c331cf073a7e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Comprenda profundamente las relaciones de tabla; gracias al linaje a nivel de columna.", "define-custom-property-for-entity": "Defina propiedades personalizadas para {{entity}} para satisfacer las necesidades de su organización.", "delete-action-description": "Eliminar esta {{entityType}} eliminará permanentemente sus metadatos de OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Una vez que elimine esta {{entityType}}, se eliminará permanentemente.", "delete-entity-type-action-description": "Eliminar esta {{entityType}} eliminará permanentemente sua metadatoa de OpenMetadata.", "delete-message-question-mark": "¿Eliminar mensaje?", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json index d36612da1e15..a3be02c3e0f9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Comprenez en profondeur la relation entre vos tables avec la traçabilité au niveau de la colonne.", "define-custom-property-for-entity": "Définir les propriétés personnalisées pour {{entity}} pour répondre à vos besoins organisationnels.", "delete-action-description": "Supprimer cette {{entityType}} supprimera de manière permanente les métadonnées dans OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Une fois que vous supprimez {{entityType}}, il sera supprimé de manière permanente", "delete-entity-type-action-description": "Supprimer cette {{entityType}} supprimera définitivement ses métadonnées d'OpenMetadata.", "delete-message-question-mark": "Supprimer le Message ?", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json index 0c4450efb829..0fb2fbfbd2b0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "カラムレベルのリネージによりテーブルの関係をより深く理解できます。", "define-custom-property-for-entity": "Define custom properties for {{entity}} to serve your organizational needs.", "delete-action-description": "{{entityType}}の削除によってOpenMetadataからこのメタデータが削除されます。", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "この{{entityType}}を1度削除すると、これは永久に取り除かれます。", "delete-entity-type-action-description": "{{entityType}}を削除するとOpenMetadataからこのメタデータが永久に削除されます。", "delete-message-question-mark": "このメッセージを削除しますか？", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json index f1ea743ea5ce..4dd9123e9800 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Obtenha os Produtores e Consumidores de dados em uma única plataforma e acelere a produtividade. A colaboração fica ainda melhor com pessoas e dados de múltiplas ferramentas em um local centralizado.", "define-custom-property-for-entity": "Defina propriedades personalizadas para {{entity}} para atender às necessidades da sua organização.", "delete-action-description": "Excluir este {{entityType}} removerá permanentemente seus metadados do OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Uma vez que você exclua este {{entityType}}, ele será removido permanentemente.", "delete-entity-type-action-description": "Excluir este {{entityType}} removerá permanentemente seus metadados do OpenMetadata.", "delete-message-question-mark": "Excluir Mensagem?", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json index 544f9fba990c..c2ce93dc265f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "Глубокое понимание связей между таблицами; благодаря отслеживанию происхождения на уровне столбцов.", "define-custom-property-for-entity": "Определите настраиваемые свойства для {{entity}} в соответствии с потребностями вашей организации.", "delete-action-description": "Удаление этого {{entityType}} навсегда удалит его метаданные из OpenMetadata.", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "Как только вы удалите этот {{entityType}}, он будет удален навсегда.", "delete-entity-type-action-description": "Удаление этого {{entityType}} навсегда удалит его метаданные из OpenMetadata.", "delete-message-question-mark": "Удалить сообщение?", diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json index da68eb4cb1bd..a543995d24ec 100644 --- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json +++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json @@ -1268,6 +1268,7 @@ "deeply-understand-table-relations-message": "通过列级血缘关系图，深入了解数据表之间的关系", "define-custom-property-for-entity": "为{{entity}}添加自定义属性以满足您的组织需求", "delete-action-description": "删除此{{entityType}}将永久删除其在 OpenMetadata 中的元数据", + "delete-asset-from-entity-type": "Deleting this will remove {{entityType}} from the entity.", "delete-entity-permanently": "一旦您删除此{{entityType}}，它将被永久删除", "delete-entity-type-action-description": "删除此{{entityType}}将永久删除其在 OpenMetadata 中的元数据", "delete-message-question-mark": "删除消息?", From d4a0316af1a50d1f3c37cd779a2dcaf3aa2eac43 Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Sat, 2 Dec 2023 14:55:11 +0530 Subject: [PATCH 09/16] Fixes and Refactor redundant code --- .../java/org/openmetadata/service/Entity.java | 2 +- .../service/jdbi3/CollectionDAO.java | 17 +++-- .../service/jdbi3/EntityRepository.java | 44 ++---------- .../service/jdbi3/GlossaryTermRepository.java | 39 ++++++----- .../service/jdbi3/MlModelRepository.java | 1 + .../service/jdbi3/PipelineRepository.java | 2 + .../service/jdbi3/SearchIndexRepository.java | 2 + .../service/jdbi3/TopicRepository.java | 2 + .../service/resources/tags/TagLabelUtil.java | 69 ++++++++++++++++++- 9 files changed, 113 insertions(+), 65 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index cfc5ecf79cdd..317c7e263fc1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -540,7 +540,7 @@ public static void populateEntityFieldTags( List flattenedFields = getFlattenedEntityField(fields); // Fetch All tags belonging to Prefix - Map> allTags = repository.getTagsByPrefix(fqnPrefix); + Map> allTags = repository.getTagsByPrefix(fqnPrefix, ".%"); for (T c : listOrEmpty(flattenedFields)) { if (setTags) { List columnTag = allTags.get(FullyQualifiedName.buildHash(c.getFullyQualifiedName())); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 9a9bfebad5a1..ced755969dfc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -2195,9 +2195,11 @@ default List getTags(String targetFQN) { return tags; } - default Map> getTagsByPrefix(String targetFQNPrefix) { + default Map> getTagsByPrefix( + String targetFQNPrefix, String postfix, boolean requiresFqnHash) { + String fqnHash = requiresFqnHash ? FullyQualifiedName.buildHash(targetFQNPrefix) : targetFQNPrefix; Map> resultSet = new LinkedHashMap<>(); - List> tags = getTagsInternalByPrefix(targetFQNPrefix); + List> tags = getTagsInternalByPrefix(fqnHash, postfix); tags.forEach( pair -> { String targetHash = pair.getLeft(); @@ -2234,7 +2236,7 @@ default Map> getTagsByPrefix(String targetFQNPrefix) { + " ON ta.fqnHash = tu.tagFQNHash " + " WHERE tu.source = 0 " + ") AS combined_data " - + "WHERE combined_data.targetFQNHash LIKE CONCAT(:targetFQNHashPrefix, '.%')", + + "WHERE combined_data.targetFQNHash LIKE CONCAT(:targetFQNHashPrefix, :postfix)", connectionType = MYSQL) @ConnectionAwareSqlQuery( value = @@ -2250,10 +2252,11 @@ default Map> getTagsByPrefix(String targetFQNPrefix) { + " JOIN tag_usage AS tu ON ta.fqnHash = tu.tagFQNHash " + " WHERE tu.source = 0 " + ") AS combined_data " - + "WHERE combined_data.targetFQNHash LIKE CONCAT(:targetFQNHashPrefix, '.%')", + + "WHERE combined_data.targetFQNHash LIKE CONCAT(:targetFQNHashPrefix, :postfix)", connectionType = POSTGRES) @RegisterRowMapper(TagLabelRowMapperWithTargetFqnHash.class) - List> getTagsInternalByPrefix(@BindFQN("targetFQNHashPrefix") String targetFQNHashPrefix); + List> getTagsInternalByPrefix( + @Bind("targetFQNHashPrefix") String targetFQNHashPrefix, @Bind("postfix") String postfix); @SqlQuery("SELECT * FROM tag_usage") @Deprecated(since = "Release 1.1") @@ -2338,6 +2341,10 @@ void renameInternal( @RegisterRowMapper(TagLabelMapper.class) List getEntityTagsFromTag(@BindFQN("tagFQNHash") String tagFQNHash); + @SqlQuery("select targetFQNHash FROM tag_usage where tagFQNHash = :tagFQNHash") + @RegisterRowMapper(TagLabelMapper.class) + List getTargetFQNHashForTag(@BindFQN("tagFQNHash") String tagFQNHash); + class TagLabelMapper implements RowMapper { @Override public TagLabel map(ResultSet r, StatementContext ctx) throws SQLException { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index bead76e40fae..a51fa1bc90d6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -42,6 +42,8 @@ import static org.openmetadata.service.Entity.getEntityFields; import static org.openmetadata.service.exception.CatalogExceptionMessage.csvNotSupported; import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import static org.openmetadata.service.util.EntityUtil.compareTagLabel; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.service.util.EntityUtil.fieldAdded; @@ -69,7 +71,6 @@ import java.net.URI; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -1227,31 +1228,6 @@ public Object getExtension(T entity) { return objectNode; } - /** Validate given list of tags and add derived tags to it */ - public final List addDerivedTags(List tagLabels) { - if (nullOrEmpty(tagLabels)) { - return tagLabels; - } - - List updatedTagLabels = new ArrayList<>(); - EntityUtil.mergeTags(updatedTagLabels, tagLabels); - for (TagLabel tagLabel : tagLabels) { - EntityUtil.mergeTags(updatedTagLabels, getDerivedTags(tagLabel)); - } - updatedTagLabels.sort(compareTagLabel); - return updatedTagLabels; - } - - /** Get tags associated with a given set of tags */ - private List getDerivedTags(TagLabel tagLabel) { - if (tagLabel.getSource() == TagLabel.TagSource.GLOSSARY) { // Related tags are only supported for Glossary - List derivedTags = daoCollection.tagUsageDAO().getTags(tagLabel.getTagFQN()); - derivedTags.forEach(tag -> tag.setLabelType(TagLabel.LabelType.DERIVED)); - return derivedTags; - } - return Collections.emptyList(); - } - protected void applyColumnTags(List columns) { // Add column level tags by adding tag to column relationship for (Column column : columns) { @@ -1290,18 +1266,6 @@ public void applyTags(List tagLabels, String targetFQN) { } } - void checkMutuallyExclusive(List tagLabels) { - Map map = new HashMap<>(); - for (TagLabel tagLabel : listOrEmpty(tagLabels)) { - // When two tags have the same parent that is mutuallyExclusive, then throw an error - String parentFqn = FullyQualifiedName.getParentFQN(tagLabel.getTagFQN()); - TagLabel stored = map.put(parentFqn, tagLabel); - if (stored != null && TagLabelUtil.mutuallyExclusive(tagLabel)) { - throw new IllegalArgumentException(CatalogExceptionMessage.mutuallyExclusiveLabels(tagLabel, stored)); - } - } - } - protected List getTags(T entity) { return !supportsTags ? null : getTags(entity.getFullyQualifiedName()); } @@ -1315,8 +1279,8 @@ protected List getTags(String fqn) { return addDerivedTags(daoCollection.tagUsageDAO().getTags(fqn)); } - public Map> getTagsByPrefix(String prefix) { - return !supportsTags ? null : daoCollection.tagUsageDAO().getTagsByPrefix(prefix); + public Map> getTagsByPrefix(String prefix, String postfix) { + return !supportsTags ? null : daoCollection.tagUsageDAO().getTagsByPrefix(prefix, postfix, true); } protected List getFollowers(T entity) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index bcb788b06188..def39cc70741 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -24,6 +24,8 @@ import static org.openmetadata.service.Entity.getEntity; import static org.openmetadata.service.exception.CatalogExceptionMessage.invalidGlossaryTermMove; import static org.openmetadata.service.exception.CatalogExceptionMessage.notReviewer; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusiveForParentAndSubField; import static org.openmetadata.service.resources.tags.TagLabelUtil.getUniqueTags; import static org.openmetadata.service.util.EntityUtil.compareTagLabel; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; @@ -34,10 +36,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; -import java.util.TreeSet; import java.util.UUID; import javax.json.JsonPatch; import lombok.extern.slf4j.Slf4j; @@ -219,9 +220,9 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( List failures = new ArrayList<>(); List success = new ArrayList<>(); - if (dryRun && (CommonUtil.nullOrEmpty(request.getGlossaryTags()) || CommonUtil.nullOrEmpty(request.getAssets()))) { + if (CommonUtil.nullOrEmpty(request.getGlossaryTags()) || CommonUtil.nullOrEmpty(request.getAssets())) { // Nothing to Validate - return result.withStatus(ApiStatus.SUCCESS).withSuccessRequest("Nothing to Validate."); + return result.withStatus(ApiStatus.SUCCESS).withSuccessRequest("Nothing to Add or Validate."); } // Validation for entityReferences @@ -240,13 +241,14 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( EntityRepository entityRepository = Entity.getEntityRepository(ref.getType()); EntityInterface asset = entityRepository.get(null, ref.getId(), entityRepository.getFields("tags")); - List allAssetTags = addDerivedTags(asset.getTags()); - try { - List tempList = new ArrayList<>(allAssetTags); - tempList.addAll(request.getGlossaryTags()); - // Check Mutually Exclusive - checkMutuallyExclusive(getUniqueTags(tempList)); + Map> allAssetTags = + daoCollection.tagUsageDAO().getTagsByPrefix(asset.getFullyQualifiedName(), "%", true); + checkMutuallyExclusiveForParentAndSubField( + asset.getFullyQualifiedName(), + FullyQualifiedName.buildHash(asset.getFullyQualifiedName()), + allAssetTags, + request.getGlossaryTags()); success.add(ref); result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); } catch (Exception ex) { @@ -255,9 +257,10 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( } // Validate and Store Tags if (!dryRun && CommonUtil.nullOrEmpty(result.getFailedRequest())) { - allAssetTags.add(tagLabel); + List tempList = new ArrayList<>(asset.getTags()); + tempList.add(tagLabel); // Apply Tags to Entities - entityRepository.applyTags(getUniqueTags(allAssetTags), asset.getFullyQualifiedName()); + entityRepository.applyTags(getUniqueTags(tempList), asset.getFullyQualifiedName()); } } @@ -479,13 +482,13 @@ protected void updateTags(String fqn, String fieldName, List origTags, return; // Nothing to update } - // Get the list of tags that are used by - Set entityTags = new TreeSet<>(compareTagLabel); - entityTags.addAll(daoCollection.tagUsageDAO().getEntityTagsFromTag(fqn)); - entityTags.addAll(updatedTags); + List targetFQNHashes = daoCollection.tagUsageDAO().getTargetFQNHashForTag(fqn); + for (String fqnHash : targetFQNHashes) { + Map> allAssetTags = daoCollection.tagUsageDAO().getTagsByPrefix(fqnHash, "%", false); - // Check if the tags are mutually exclusive - checkMutuallyExclusive(entityTags.stream().toList()); + // Assets FQN is not available / we can use fqnHash for now + checkMutuallyExclusiveForParentAndSubField("", fqnHash, allAssetTags, updatedTags); + } // Remove current entity tags in the database. It will be added back later from the merged tag list. daoCollection.tagUsageDAO().deleteTagsByTarget(fqn); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java index 5cebb786d1e6..9c7fb6a8717c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MlModelRepository.java @@ -17,6 +17,7 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.service.Entity.DASHBOARD; import static org.openmetadata.service.Entity.MLMODEL; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch; import static org.openmetadata.service.util.EntityUtil.mlFeatureMatch; import static org.openmetadata.service.util.EntityUtil.mlHyperParameterMatch; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java index 3852a85daf01..486fc474b92b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PipelineRepository.java @@ -19,6 +19,8 @@ import static org.openmetadata.schema.type.Include.NON_DELETED; import static org.openmetadata.service.Entity.CONTAINER; import static org.openmetadata.service.Entity.FIELD_TAGS; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import static org.openmetadata.service.util.EntityUtil.taskMatch; import java.util.ArrayList; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java index 3b01d8ab9692..26372ada07e0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SearchIndexRepository.java @@ -21,6 +21,8 @@ import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_FOLLOWERS; import static org.openmetadata.service.Entity.FIELD_TAGS; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import static org.openmetadata.service.util.EntityUtil.getSearchIndexField; import java.util.ArrayList; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java index 5d5628982cfa..4236cbe4a473 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TopicRepository.java @@ -21,6 +21,8 @@ import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME; import static org.openmetadata.service.Entity.FIELD_TAGS; import static org.openmetadata.service.Entity.populateEntityFieldTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags; +import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; import java.util.ArrayList; import java.util.HashSet; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java index 52950ccab31a..2fbb8ef32f1c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java @@ -13,15 +13,19 @@ package org.openmetadata.service.resources.tags; +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.schema.type.Include.NON_DELETED; import static org.openmetadata.service.util.EntityUtil.compareTagLabel; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.classification.Classification; import org.openmetadata.schema.entity.classification.Tag; @@ -30,6 +34,7 @@ import org.openmetadata.schema.type.TagLabel; import org.openmetadata.schema.type.TagLabel.TagSource; import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.FullyQualifiedName; @@ -91,7 +96,6 @@ public static boolean mutuallyExclusive(TagLabel label) { } } - // TODO: Below two methods :addDerivedTags and :getDerivedTags can be removed from Entity Repository public static List addDerivedTags(List tagLabels) { if (nullOrEmpty(tagLabels)) { return tagLabels; @@ -120,4 +124,67 @@ public static List getUniqueTags(List tags) { uniqueTags.addAll(tags); return uniqueTags.stream().toList(); } + + public static void checkMutuallyExclusive(List tagLabels) { + Map map = new HashMap<>(); + for (TagLabel tagLabel : listOrEmpty(tagLabels)) { + // When two tags have the same parent that is mutuallyExclusive, then throw an error + String parentFqn = FullyQualifiedName.getParentFQN(tagLabel.getTagFQN()); + TagLabel stored = map.put(parentFqn, tagLabel); + if (stored != null && TagLabelUtil.mutuallyExclusive(tagLabel)) { + throw new IllegalArgumentException(CatalogExceptionMessage.mutuallyExclusiveLabels(tagLabel, stored)); + } + } + } + + public static void checkMutuallyExclusiveForParentAndSubField( + String assetFqn, String assetFqnHash, Map> allAssetTags, List glossaryTags) { + boolean failed = false; + StringBuilder errorMessage = new StringBuilder(); + + Map> filteredTags = + allAssetTags.entrySet().stream() + .filter(entry -> entry.getKey().startsWith(assetFqnHash)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + // Check Parent Tags + List parentTags = filteredTags.remove(assetFqnHash); + + if (parentTags != null) { + List tempList = new ArrayList<>(addDerivedTags(parentTags)); + tempList.addAll(glossaryTags); + try { + checkMutuallyExclusive(getUniqueTags(tempList)); + } catch (IllegalArgumentException ex) { + failed = true; + errorMessage.append( + String.format( + "Asset %s has a tag %s which is mutually exclusive with the one of the glossary tags %s. %n", + assetFqn, converTagLabelArrayToString(tempList), converTagLabelArrayToString(glossaryTags))); + } + } + + // Check SubFields Tags + Set subFieldTags = filteredTags.values().stream().flatMap(List::stream).collect(Collectors.toSet()); + List tempList = new ArrayList<>(addDerivedTags(subFieldTags.stream().toList())); + tempList.addAll(glossaryTags); + try { + checkMutuallyExclusive(getUniqueTags(tempList)); + } catch (IllegalArgumentException ex) { + failed = true; + errorMessage.append( + String.format( + "Asset %s has a Subfield Column/Schema/Field containing tags %s which is mutually exclusive with the one of the glossary tags %s", + assetFqn, converTagLabelArrayToString(tempList), converTagLabelArrayToString(glossaryTags))); + } + + // Throw Exception if failed + if (failed) { + throw new IllegalArgumentException(errorMessage.toString()); + } + } + + public static String converTagLabelArrayToString(List tags) { + return String.format("[%s]", tags.stream().map(TagLabel::getTagFQN).collect(Collectors.joining(", "))); + } } From 0d5bf7a4e5f34fd3b9eb1b06955a2e96e9193f60 Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Sat, 2 Dec 2023 14:59:21 +0530 Subject: [PATCH 10/16] Revert --- .../openmetadata/service/jdbi3/GlossaryTermRepository.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 4795f1abb2d4..fd305d039c1c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -21,7 +21,6 @@ import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.service.Entity.GLOSSARY; import static org.openmetadata.service.Entity.GLOSSARY_TERM; -import static org.openmetadata.service.Entity.getEntity; import static org.openmetadata.service.exception.CatalogExceptionMessage.invalidGlossaryTermMove; import static org.openmetadata.service.exception.CatalogExceptionMessage.notReviewer; import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive; @@ -134,7 +133,7 @@ public void prepare(GlossaryTerm entity, boolean update) { // Validate parent term GlossaryTerm parentTerm = entity.getParent() != null - ? getEntity(entity.getParent().withType(GLOSSARY_TERM), "owner,reviewers", Include.NON_DELETED) + ? Entity.getEntity(entity.getParent().withType(GLOSSARY_TERM), "owner,reviewers", Include.NON_DELETED) : null; if (parentTerm != null) { parentReviewers = parentTerm.getReviewers(); @@ -142,7 +141,7 @@ public void prepare(GlossaryTerm entity, boolean update) { } // Validate glossary - Glossary glossary = getEntity(entity.getGlossary(), "reviewers", Include.NON_DELETED); + Glossary glossary = Entity.getEntity(entity.getGlossary(), "reviewers", Include.NON_DELETED); entity.setGlossary(glossary.getEntityReference()); parentReviewers = parentReviewers != null ? parentReviewers : glossary.getReviewers(); From f734cb4083b754c8a51e027b8d7fddba01cd8949 Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Sat, 2 Dec 2023 15:26:35 +0530 Subject: [PATCH 11/16] Add Delete API for removing tags from assets --- .../service/jdbi3/CollectionDAO.java | 5 +++ .../service/jdbi3/GlossaryTermRepository.java | 26 +++++++++++++ .../glossary/GlossaryTermResource.java | 38 +++++++++++++------ .../api/addGlossaryToAssetsRequest.json | 1 - 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index ced755969dfc..94a574c353e5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -2272,6 +2272,11 @@ List> getTagsInternalByPrefix( @SqlUpdate("DELETE FROM tag_usage where targetFQNHash = :targetFQNHash") void deleteTagsByTarget(@BindFQN("targetFQNHash") String targetFQNHash); + @SqlUpdate( + "DELETE FROM tag_usage where tagFQNHash = :tagFqnHash AND targetFQNHash LIKE CONCAT(:targetFQNHash, '%')") + void deleteTagsByTagAndTargetEntity( + @BindFQN("tagFqnHash") String tagFqnHash, @BindFQN("targetFQNHash") String targetFQNHash); + @SqlUpdate("DELETE FROM tag_usage where tagFQNHash = :tagFQNHash AND source = :source") void deleteTagLabels(@Bind("source") int source, @BindFQN("tagFQNHash") String tagFQNHash); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index fd305d039c1c..53a662262093 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -287,6 +287,32 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( return result; } + public BulkOperationResult bulkRemoveGlossaryToAssets(UUID glossaryTermId, AddGlossaryToAssetsRequest request) { + GlossaryTerm term = this.get(null, glossaryTermId, getFields("id,tags")); + + BulkOperationResult result = new BulkOperationResult().withStatus(ApiStatus.SUCCESS).withDryRun(false); + List success = new ArrayList<>(); + + // Validation for entityReferences + EntityUtil.populateEntityReferences(request.getAssets()); + + for (EntityReference ref : request.getAssets()) { + // Update Result Processed + result.setNumberOfRowsProcessed(result.getNumberOfRowsProcessed() + 1); + + EntityRepository entityRepository = Entity.getEntityRepository(ref.getType()); + EntityInterface asset = entityRepository.get(null, ref.getId(), entityRepository.getFields("id")); + + daoCollection + .tagUsageDAO() + .deleteTagsByTagAndTargetEntity(term.getFullyQualifiedName(), asset.getFullyQualifiedName()); + success.add(ref); + result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); + } + + return result.withSuccessRequest(success); + } + protected EntityReference getGlossary(GlossaryTerm term) { Relationship relationship = term.getParent() != null ? Relationship.HAS : Relationship.CONTAINS; return term.getGlossary() != null diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java index 2b01e4753f94..c6e79e4114eb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java @@ -61,6 +61,7 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; +import org.openmetadata.schema.type.api.BulkOperationResult; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.CatalogExceptionMessage; @@ -419,31 +420,46 @@ public Response updateVote( @PUT @Path("/{id}/assets/add") @Operation( - operationId = "bulkAddGlossaryToAssets", - summary = "Bulk Add Glossary to Assets", - description = "Bulk Add Glossary to Assets", + operationId = "bulkAddGlossaryTermToAssets", + summary = "Bulk Add Glossary Term to Assets", + description = "Bulk Add Glossary Term to Assets", responses = { @ApiResponse( responseCode = "200", description = "OK", - content = @Content(mediaType = "application/json", schema = @Schema(implementation = ChangeEvent.class))), + content = + @Content(mediaType = "application/json", schema = @Schema(implementation = BulkOperationResult.class))), @ApiResponse(responseCode = "404", description = "model for instance {id} is not found") }) public Response bulkAddGlossaryToAssets( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Parameter(description = "Id of the Entity", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, - @Parameter( - description = - "Dry-run when true is used for validating the glossary without really applying it. (default=true)", - schema = @Schema(type = "boolean")) - @DefaultValue("true") - @QueryParam("dryRun") - boolean dryRun, @Valid AddGlossaryToAssetsRequest request) { return Response.ok().entity(repository.bulkAddAndValidateGlossaryToAssets(id, request)).build(); } + @PUT + @Path("/{id}/assets/remove") + @Operation( + operationId = "bulkRemoveGlossaryTermFromAssets", + summary = "Bulk Remove Glossary Term from Assets", + description = "Bulk Remove Glossary Term from Assets", + responses = { + @ApiResponse( + responseCode = "200", + description = "OK", + content = @Content(mediaType = "application/json", schema = @Schema(implementation = ChangeEvent.class))), + @ApiResponse(responseCode = "404", description = "model for instance {id} is not found") + }) + public Response bulkRemoveGlossaryFromAssets( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the Entity", schema = @Schema(type = "UUID")) @PathParam("id") UUID id, + @Valid AddGlossaryToAssetsRequest request) { + return Response.ok().entity(repository.bulkRemoveGlossaryToAssets(id, request)).build(); + } + @DELETE @Path("/{id}") @Operation( diff --git a/openmetadata-spec/src/main/resources/json/schema/api/addGlossaryToAssetsRequest.json b/openmetadata-spec/src/main/resources/json/schema/api/addGlossaryToAssetsRequest.json index 72a8865e4b6c..4e10674b10e8 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/addGlossaryToAssetsRequest.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/addGlossaryToAssetsRequest.json @@ -24,6 +24,5 @@ "$ref": "../type/entityReferenceList.json" } }, - "required": ["glossaryTags, assets"], "additionalProperties": false } From fc268f5554c6e5ad177396e6b44dd20217019867 Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Sat, 2 Dec 2023 15:46:35 +0530 Subject: [PATCH 12/16] ADd ES calls --- .../service/jdbi3/GlossaryTermRepository.java | 11 +++++++++-- .../openmetadata/service/search/SearchRepository.java | 8 ++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 53a662262093..886a284b5ed7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -219,9 +219,9 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( List failures = new ArrayList<>(); List success = new ArrayList<>(); - if (CommonUtil.nullOrEmpty(request.getGlossaryTags()) || CommonUtil.nullOrEmpty(request.getAssets())) { + if (dryRun && (CommonUtil.nullOrEmpty(request.getGlossaryTags()) || CommonUtil.nullOrEmpty(request.getAssets()))) { // Nothing to Validate - return result.withStatus(ApiStatus.SUCCESS).withSuccessRequest("Nothing to Add or Validate."); + return result.withStatus(ApiStatus.SUCCESS).withSuccessRequest("Nothing to Validate."); } // Validation for entityReferences @@ -260,6 +260,8 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( tempList.add(tagLabel); // Apply Tags to Entities entityRepository.applyTags(getUniqueTags(tempList), asset.getFullyQualifiedName()); + + searchRepository.updateEntity(ref); } } @@ -270,6 +272,8 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( // Remove current entity tags in the database. It will be added back later from the merged tag list. daoCollection.tagUsageDAO().deleteTagsByTarget(term.getFullyQualifiedName()); applyTags(getUniqueTags(request.getGlossaryTags()), term.getFullyQualifiedName()); + + searchRepository.updateEntity(term.getEntityReference()); } // Add Failed And Suceess Request @@ -308,6 +312,9 @@ public BulkOperationResult bulkRemoveGlossaryToAssets(UUID glossaryTermId, AddGl .deleteTagsByTagAndTargetEntity(term.getFullyQualifiedName(), asset.getFullyQualifiedName()); success.add(ref); result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); + + // Update ES + searchRepository.updateEntity(ref); } return result.withSuccessRequest(success); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 9e32fd90645e..bf8fc4d0955c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -52,6 +52,7 @@ import org.openmetadata.schema.type.UsageDetails; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.UnhandledServerException; +import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.search.elasticsearch.ElasticSearchClient; import org.openmetadata.service.search.indexes.SearchIndex; import org.openmetadata.service.search.models.IndexMapping; @@ -261,6 +262,13 @@ public void updateEntity(EntityInterface entity) { } } + public void updateEntity(EntityReference entityReference) { + EntityRepository entityRepository = Entity.getEntityRepository(entityReference.getType()); + EntityInterface entity = entityRepository.get(null, entityReference.getId(), entityRepository.getFields("*")); + // Update Entity + updateEntity(entity); + } + public void propagateInheritedFieldsToChildren( String entityType, String entityId, ChangeDescription changeDescription, IndexMapping indexMapping) { if (changeDescription != null) { From a34421fd40468e1aceba3d632dc54ecfc0adb92f Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Sat, 2 Dec 2023 15:55:05 +0530 Subject: [PATCH 13/16] Fix Assests added in case of errors --- .../org/openmetadata/service/jdbi3/GlossaryTermRepository.java | 1 + 1 file changed, 1 insertion(+) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 886a284b5ed7..646259d4bdef 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -252,6 +252,7 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); } catch (Exception ex) { failures.add(new FailureRequest().withRequest(ref).withError(ex.getMessage())); + result.withFailedRequest(failures); result.setNumberOfRowsFailed(result.getNumberOfRowsFailed() + 1); } // Validate and Store Tags From aebc29f0af19999696e32c9d088099c9d4751798 Mon Sep 17 00:00:00 2001 From: karanh37 Date: Sat, 2 Dec 2023 16:49:59 +0530 Subject: [PATCH 14/16] ui updates --- .../GlossaryDetails.component.tsx | 14 +- .../GlossaryTermsV1.component.tsx | 9 +- .../tabs/AssetsTabs.component.tsx | 207 +++++++++++------- .../main/resources/ui/src/rest/glossaryAPI.ts | 37 ++-- 4 files changed, 166 insertions(+), 101 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx index 7099151e4d61..4a7cc361dfda 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetails/GlossaryDetails.component.tsx @@ -19,6 +19,7 @@ import { useHistory, useParams } from 'react-router-dom'; import { getGlossaryTermDetailsPath } from '../../../constants/constants'; import { EntityField } from '../../../constants/Feeds.constants'; import { EntityType } from '../../../enums/entity.enum'; +import { Glossary } from '../../../generated/entity/data/glossary'; import { ChangeDescription } from '../../../generated/entity/type'; import { getFeedCounts } from '../../../utils/CommonUtils'; import { getEntityName } from '../../../utils/EntityUtils'; @@ -56,13 +57,18 @@ const GlossaryDetails = ({ const [isDescriptionEditable, setIsDescriptionEditable] = useState(false); + const handleGlossaryUpdate = async (updatedGlossary: Glossary) => { + await updateGlossary(updatedGlossary); + getEntityFeedCount(); + }; + const onDescriptionUpdate = async (updatedHTML: string) => { if (glossary.description !== updatedHTML) { - const updatedTableDetails = { + const updatedGlossaryDetails = { ...glossary, description: updatedHTML, }; - updateGlossary(updatedTableDetails); + handleGlossaryUpdate(updatedGlossaryDetails); setIsDescriptionEditable(false); } else { setIsDescriptionEditable(false); @@ -165,7 +171,7 @@ const GlossaryDetails = ({ permissions={permissions} selectedData={glossary} onThreadLinkSelect={onThreadLinkSelect} - onUpdate={updateGlossary} + onUpdate={(data) => handleGlossaryUpdate(data as Glossary)} /> @@ -243,7 +249,7 @@ const GlossaryDetails = ({ updateVote={updateVote} onAddGlossaryTerm={onAddGlossaryTerm} onDelete={handleGlossaryDelete} - onUpdate={updateGlossary} + onUpdate={(data) => handleGlossaryUpdate(data as Glossary)} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx index 2ddccb388de8..b46716c938c1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx @@ -121,6 +121,11 @@ const GlossaryTermsV1 = ({ [glossaryTerm, handleGlossaryTermUpdate] ); + const onTermUpdate = async (data: GlossaryTerm) => { + await handleGlossaryTermUpdate(data); + getEntityFeedCount(); + }; + const tabItems = useMemo(() => { const items = [ { @@ -133,7 +138,7 @@ const GlossaryTermsV1 = ({ permissions={permissions} selectedData={glossaryTerm} onThreadLinkSelect={onThreadLinkSelect} - onUpdate={(data) => handleGlossaryTermUpdate(data as GlossaryTerm)} + onUpdate={(data) => onTermUpdate(data as GlossaryTerm)} /> ), }, @@ -316,7 +321,7 @@ const GlossaryTermsV1 = ({ onAddGlossaryTerm={onAddGlossaryTerm} onAssetAdd={() => setAssetModelVisible(true)} onDelete={handleGlossaryTermDelete} - onUpdate={(data) => handleGlossaryTermUpdate(data as GlossaryTerm)} + onUpdate={(data) => onTermUpdate(data as GlossaryTerm)} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx index 4dd84c4dce77..a584093712fb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/tabs/AssetsTabs.component.tsx @@ -13,7 +13,6 @@ */ import { FilterOutlined, PlusOutlined } from '@ant-design/icons'; -import Context from '@ant-design/icons/lib/components/Context'; import { Badge, Button, @@ -73,15 +72,18 @@ import { patchDataProduct, } from '../../../../rest/dataProductAPI'; import { getDomainByName } from '../../../../rest/domainAPI'; -import { getGlossaryTermByFQN } from '../../../../rest/glossaryAPI'; +import { + getGlossaryTermByFQN, + removeAssetsFromGlossaryTerm, +} from '../../../../rest/glossaryAPI'; import { searchData } from '../../../../rest/miscAPI'; import { getAssetsPageQuickFilters } from '../../../../utils/AdvancedSearchUtils'; -import { - removeGlossaryTermAssets, - updateDomainAssets, -} from '../../../../utils/Assets/AssetsUtils'; +import { updateDomainAssets } from '../../../../utils/Assets/AssetsUtils'; import { getCountBadge, Transi18next } from '../../../../utils/CommonUtils'; -import { getEntityName } from '../../../../utils/EntityUtils'; +import { + getEntityName, + getEntityReferenceFromEntity, +} from '../../../../utils/EntityUtils'; import { getAggregations, getSelectedValuesFromQuickFilter, @@ -135,7 +137,6 @@ const AssetsTabs = forwardRef( const [itemCount, setItemCount] = useState>( {} as Record ); - const [api, contextHolder] = notification.useNotification(); const [activeFilter, setActiveFilter] = useState([]); const { fqn } = useParams<{ fqn: string }>(); @@ -264,6 +265,11 @@ const AssetsTabs = forwardRef( }, [activeFilter, currentPage, pageSize, searchValue] ); + + const hideNotification = () => { + notification.close('asset-tab-notification-key'); + }; + const onOpenChange: MenuProps['onOpenChange'] = (keys) => { const latestOpenKey = keys.find( (key) => openKeys.indexOf(key as EntityType) === -1 @@ -496,9 +502,9 @@ const AssetsTabs = forwardRef( }; const deleteSelectedItems = useCallback(() => { - selectedItems?.forEach((item) => { - onAssetRemove(item); - }); + if (selectedItems) { + onAssetRemove(Array.from(selectedItems.values())); + } }, [selectedItems]); useEffect(() => { @@ -517,7 +523,7 @@ const AssetsTabs = forwardRef( useEffect(() => { if (selectedItems) { - notification.close('asset-tab-notification-key'); + hideNotification(); if (selectedItems.size > 1) { openNotification(); } @@ -677,9 +683,39 @@ const AssetsTabs = forwardRef( ] ); + const onSelectAll = (selectAll: boolean) => { + setSelectedItems((prevItems) => { + const selectedItemMap = new Map(prevItems ?? []); + + if (selectAll) { + data.forEach(({ _source }) => { + const id = _source.id; + if (id) { + selectedItemMap.set(id, _source); + } + }); + } else { + // Clear selection + selectedItemMap.clear(); + } + + return selectedItemMap; + }); + }; + const assetsHeader = useMemo(() => { return ( -
+
+ {data.length > 0 && ( + onSelectAll(e.target.checked)}> + {t('label.select-field', { + field: t('label.all'), + })} + + )} + { @@ -747,7 +784,7 @@ const AssetsTabs = forwardRef( }, [assetsHeader, assetListing, selectedCard]); const onAssetRemove = useCallback( - async (data: SourceType) => { + async (assetsData: SourceType[]) => { if (!activeEntity) { return; } @@ -758,7 +795,10 @@ const AssetsTabs = forwardRef( case AssetsOfEntity.DATA_PRODUCT: const updatedAssets = ( (activeEntity as DataProduct)?.assets ?? [] - ).filter((asset) => asset.id !== data?.id); + ).filter( + (asset) => !assetsData.some((item) => item.id === asset.id) + ); + updatedEntity = { ...activeEntity, assets: updatedAssets, @@ -772,20 +812,26 @@ const AssetsTabs = forwardRef( break; - case AssetsOfEntity.DOMAIN: case AssetsOfEntity.GLOSSARY: - const selectedItemMap = new Map(); - selectedItemMap.set(data?.id, data); - - if (type === AssetsOfEntity.DOMAIN) { - await updateDomainAssets(undefined, type, selectedItemMap); - } else if (type === AssetsOfEntity.GLOSSARY) { - await removeGlossaryTermAssets( - entityFqn ?? '', - type, - selectedItemMap + const entities = [...(assetsData?.values() ?? [])].map((item) => { + return getEntityReferenceFromEntity( + item, + (item as EntityDetailUnion).entityType ); - } + }); + await removeAssetsFromGlossaryTerm( + activeEntity as GlossaryTerm, + entities + ); + + break; + + case AssetsOfEntity.DOMAIN: + const selectedItemMap = new Map(); + assetsData.forEach((item) => { + selectedItemMap.set(item.id, item); + }); + await updateDomainAssets(undefined, type, selectedItemMap); break; default: @@ -803,13 +849,15 @@ const AssetsTabs = forwardRef( } finally { setShowDeleteModal(false); onRemoveAsset?.(); + hideNotification(); + setSelectedItems(new Map()); // Reset selected items } }, [type, activeEntity, entityFqn] ); const openNotification = () => { - api.warning({ + notification.warning({ key: 'asset-tab-notification-key', message: (
@@ -875,63 +923,60 @@ const AssetsTabs = forwardRef( }, [isSummaryPanelOpen]); return ( - - {contextHolder} -
- +
+ + + + + + + + + + {isLoading || isCountLoading ? ( + - + - - + + - - {isLoading || isCountLoading ? ( - - - - - - - - - ) : ( - layout - )} - - setShowDeleteModal(false)} - onConfirm={() => onAssetRemove(assetToDelete as SourceType)} - /> -
- + ) : ( + layout + )} + + setShowDeleteModal(false)} + onConfirm={() => onAssetRemove(assetToDelete ? [assetToDelete] : [])} + /> +
); } ); diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts index 8857809c5ab1..f78c93c5c18a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/glossaryAPI.ts @@ -15,13 +15,10 @@ import { AxiosResponse } from 'axios'; import { Operation } from 'fast-json-patch'; import { PagingResponse } from 'Models'; import { VotingDataProps } from '../components/Voting/voting.interface'; +import { AddGlossaryToAssetsRequest } from '../generated/api/addGlossaryToAssetsRequest'; import { CreateGlossary } from '../generated/api/data/createGlossary'; import { CreateGlossaryTerm } from '../generated/api/data/createGlossaryTerm'; -import { - EntityReference, - Glossary, - TagLabel, -} from '../generated/entity/data/glossary'; +import { EntityReference, Glossary } from '../generated/entity/data/glossary'; import { GlossaryTerm } from '../generated/entity/data/glossaryTerm'; import { CSVImportResult } from '../generated/type/csvImportResult'; import { EntityHistory } from '../generated/type/entityHistory'; @@ -254,13 +251,25 @@ export const updateGlossaryTermVotes = async ( return response.data; }; -type AssetsData = { - assets: EntityReference[]; - dryRun: boolean; - glossaryTags: TagLabel[]; +export const addAssetsToGlossaryTerm = async ( + glossaryTerm: GlossaryTerm, + assets: EntityReference[] +) => { + const data = { + assets: assets, + dryRun: false, + glossaryTags: glossaryTerm.tags ?? [], + }; + + const response = await APIClient.put< + AddGlossaryToAssetsRequest, + AxiosResponse + >(`/glossaryTerms/${glossaryTerm.id}/assets/add`, data); + + return response.data; }; -export const addAssetsToGlossaryTerm = async ( +export const removeAssetsFromGlossaryTerm = async ( glossaryTerm: GlossaryTerm, assets: EntityReference[] ) => { @@ -270,10 +279,10 @@ export const addAssetsToGlossaryTerm = async ( glossaryTags: glossaryTerm.tags ?? [], }; - const response = await APIClient.put>( - `/glossaryTerms/${glossaryTerm.id}/assets/add`, - data - ); + const response = await APIClient.put< + AddGlossaryToAssetsRequest, + AxiosResponse + >(`/glossaryTerms/${glossaryTerm.id}/assets/remove`, data); return response.data; }; From 622d627e56377e36e7f980604b61b6ca8f44234a Mon Sep 17 00:00:00 2001 From: karanh37 Date: Sat, 2 Dec 2023 17:56:07 +0530 Subject: [PATCH 15/16] ui changes --- .../AssetsSelectionModal/AssetSelectionModal.tsx | 1 + .../asset-selection-model.style.less | 7 +++++++ .../data-products-details-page.less | 4 ---- .../ExploreSearchCard/ExploreSearchCard.tsx | 5 +++++ .../ui/src/components/ExploreV1/exploreV1.less | 7 ++++--- .../EntitySummaryDetails/EntitySummaryDetails.tsx | 15 +++++++++++++-- 6 files changed, 30 insertions(+), 9 deletions(-) diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx index 14f2360c34db..473e45d9df7c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Assets/AssetsSelectionModal/AssetSelectionModal.tsx @@ -384,6 +384,7 @@ export const AssetSelectionModal = ({ return ( = forwardRef< isLink: true, openInNewTab: false, }); + } else { + _otherDetails.push({ + key: 'Domain', + value: '', + }); } if ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/exploreV1.less b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/exploreV1.less index 9b44fdcad38b..57252d0c7d36 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/exploreV1.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/ExploreV1/exploreV1.less @@ -27,9 +27,7 @@ border-top: 0; } } -.filters-row { - border-bottom: 1px solid @border-color; -} + .quick-filter-dropdown-trigger-btn { .ant-typography { color: @text-grey-muted; @@ -63,6 +61,9 @@ .sorting-dropdown-container { line-height: 24px; } + .filters-row { + border-bottom: 1px solid @border-color; + } } .explore-page { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx index 6ae6155972f6..65d59938efc1 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx @@ -14,7 +14,14 @@ import { Button, Space } from 'antd'; import Tooltip, { RenderFunction } from 'antd/lib/tooltip'; import classNames from 'classnames'; -import { isString, isUndefined, lowerCase, noop, toLower } from 'lodash'; +import { + isEmpty, + isString, + isUndefined, + lowerCase, + noop, + toLower, +} from 'lodash'; import { ExtraInfo } from 'Models'; import React, { useMemo } from 'react'; import { useTranslation } from 'react-i18next'; @@ -189,7 +196,7 @@ const EntitySummaryDetails = ({ case 'Domain': { - retVal = ( + retVal = !isEmpty(displayVal) ? ( + ) : ( + + {t('label.no-entity', { entity: t('label.domain') })} + ); } From 9e016b4f8dc6ca233c1df6e7b757a0d32081b4b4 Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Sat, 2 Dec 2023 23:51:24 +0530 Subject: [PATCH 16/16] Add Conditional Validate for SubFields --- .../service/jdbi3/GlossaryTermRepository.java | 5 +-- .../service/resources/tags/TagLabelUtil.java | 32 +++++++++++-------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java index 646259d4bdef..39ff4a50c538 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/GlossaryTermRepository.java @@ -247,7 +247,8 @@ public BulkOperationResult bulkAddAndValidateGlossaryToAssets( asset.getFullyQualifiedName(), FullyQualifiedName.buildHash(asset.getFullyQualifiedName()), allAssetTags, - request.getGlossaryTags()); + request.getGlossaryTags(), + false); success.add(ref); result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1); } catch (Exception ex) { @@ -520,7 +521,7 @@ protected void updateTags(String fqn, String fieldName, List origTags, Map> allAssetTags = daoCollection.tagUsageDAO().getTagsByPrefix(fqnHash, "%", false); // Assets FQN is not available / we can use fqnHash for now - checkMutuallyExclusiveForParentAndSubField("", fqnHash, allAssetTags, updatedTags); + checkMutuallyExclusiveForParentAndSubField("", fqnHash, allAssetTags, updatedTags, true); } // Remove current entity tags in the database. It will be added back later from the merged tag list. diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java index 2fbb8ef32f1c..b335ac73f87f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java @@ -138,7 +138,11 @@ public static void checkMutuallyExclusive(List tagLabels) { } public static void checkMutuallyExclusiveForParentAndSubField( - String assetFqn, String assetFqnHash, Map> allAssetTags, List glossaryTags) { + String assetFqn, + String assetFqnHash, + Map> allAssetTags, + List glossaryTags, + boolean validateSubFields) { boolean failed = false; StringBuilder errorMessage = new StringBuilder(); @@ -164,18 +168,20 @@ public static void checkMutuallyExclusiveForParentAndSubField( } } - // Check SubFields Tags - Set subFieldTags = filteredTags.values().stream().flatMap(List::stream).collect(Collectors.toSet()); - List tempList = new ArrayList<>(addDerivedTags(subFieldTags.stream().toList())); - tempList.addAll(glossaryTags); - try { - checkMutuallyExclusive(getUniqueTags(tempList)); - } catch (IllegalArgumentException ex) { - failed = true; - errorMessage.append( - String.format( - "Asset %s has a Subfield Column/Schema/Field containing tags %s which is mutually exclusive with the one of the glossary tags %s", - assetFqn, converTagLabelArrayToString(tempList), converTagLabelArrayToString(glossaryTags))); + if (validateSubFields) { + // Check SubFields Tags + Set subFieldTags = filteredTags.values().stream().flatMap(List::stream).collect(Collectors.toSet()); + List tempList = new ArrayList<>(addDerivedTags(subFieldTags.stream().toList())); + tempList.addAll(glossaryTags); + try { + checkMutuallyExclusive(getUniqueTags(tempList)); + } catch (IllegalArgumentException ex) { + failed = true; + errorMessage.append( + String.format( + "Asset %s has a Subfield Column/Schema/Field containing tags %s which is mutually exclusive with the one of the glossary tags %s", + assetFqn, converTagLabelArrayToString(tempList), converTagLabelArrayToString(glossaryTags))); + } } // Throw Exception if failed