GEN-1983 : Feat - Support Tag Asset Page (#18505)

* Feat : Support bulk adding tag to Assets * remove warnings * add apis for assets remove * Fix: Add tag page (#18461) * add tag page which all Assets * update as per feedbacks * update as per feedbacks * add divider in header badge * remove styling * update permission and refactoring code * updated as per comments * fix sonar cloud issues * add delete asset functionality * refactor entityTypeString * made the top bar fixed to top * add tests for tag page * fix check failures issue * fix tag page check failure * fix flaky test issue * add tag page tests * update the add asset test * update playwright tests * add right panel in tag page * updated as per feedbacks * remove usage test * updated as per feedbacks --------- * Backend: make bulkAssets api async * Backend: limit bulkAssets api async to Tag Assets page * Update Tag page Assets API (#18622) * create branch * update add asset API * add websocket on tags page --------- Co-authored-by: Karan Hotchandani <33024356+karanh37@users.noreply.github.com> Co-authored-by: karanh37 <karanh37@gmail.com> * add failed case for socket operation --------- Co-authored-by: Karan Hotchandani <33024356+karanh37@users.noreply.github.com> Co-authored-by: Sweta Agarwalla <105535990+sweta1308@users.noreply.github.com> Co-authored-by: karanh37 <karanh37@gmail.com>
open-metadata · Nov 18, 2024 · 853bbc2 · 853bbc2
1 parent 66c253e
commit 853bbc2
Show file tree

Hide file tree

Showing 36 changed files with 1,740 additions and 102 deletions.
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
@@ -119,6 +119,7 @@
 import org.apache.commons.lang3.tuple.Pair;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.common.utils.CommonUtil;
+import org.openmetadata.schema.BulkAssetsRequestInterface;
 import org.openmetadata.schema.CreateEntity;
 import org.openmetadata.schema.EntityInterface;
 import org.openmetadata.schema.api.VoteRequest;
@@ -2511,6 +2512,16 @@ protected void validateColumnTags(List<Column> columns) {
     }
   }
 
+  public BulkOperationResult bulkAddAndValidateTagsToAssets(
+      UUID glossaryTermId, BulkAssetsRequestInterface request) {
+    throw new UnsupportedOperationException("Bulk Add tags to Asset operation not supported");
+  }
+
+  public BulkOperationResult bulkRemoveAndValidateTagsToAssets(
+      UUID glossaryTermId, BulkAssetsRequestInterface request) {
+    throw new UnsupportedOperationException("Bulk Remove tags to Asset operation not supported");
+  }
+
   public enum Operation {
     PUT,
     PATCH,

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java
@@ -17,25 +17,39 @@
 import static org.openmetadata.schema.type.Include.NON_DELETED;
 import static org.openmetadata.service.Entity.CLASSIFICATION;
 import static org.openmetadata.service.Entity.TAG;
+import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusiveForParentAndSubField;
+import static org.openmetadata.service.resources.tags.TagLabelUtil.getUniqueTags;
 import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch;
 import static org.openmetadata.service.util.EntityUtil.getId;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.UUID;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
+import org.openmetadata.common.utils.CommonUtil;
+import org.openmetadata.schema.BulkAssetsRequestInterface;
+import org.openmetadata.schema.EntityInterface;
+import org.openmetadata.schema.api.AddTagToAssetsRequest;
 import org.openmetadata.schema.entity.classification.Classification;
 import org.openmetadata.schema.entity.classification.Tag;
+import org.openmetadata.schema.type.ApiStatus;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.ProviderType;
 import org.openmetadata.schema.type.Relationship;
+import org.openmetadata.schema.type.TagLabel;
 import org.openmetadata.schema.type.TagLabel.TagSource;
+import org.openmetadata.schema.type.api.BulkOperationResult;
+import org.openmetadata.schema.type.api.BulkResponse;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.exception.CatalogExceptionMessage;
 import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord;
 import org.openmetadata.service.resources.tags.TagResource;
+import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.EntityUtil.Fields;
 import org.openmetadata.service.util.FullyQualifiedName;
 
@@ -109,6 +123,118 @@ public void setFullyQualifiedName(Tag tag) {
     }
   }
 
+  @Override
+  public BulkOperationResult bulkAddAndValidateTagsToAssets(
+      UUID classificationTagId, BulkAssetsRequestInterface request) {
+    AddTagToAssetsRequest addTagToAssetsRequest = (AddTagToAssetsRequest) request;
+    boolean dryRun = Boolean.TRUE.equals(addTagToAssetsRequest.getDryRun());
+
+    Tag tag = this.get(null, classificationTagId, getFields("id"));
+
+    BulkOperationResult result = new BulkOperationResult().withDryRun(dryRun);
+    List<BulkResponse> failures = new ArrayList<>();
+    List<BulkResponse> success = new ArrayList<>();
+
+    if (dryRun || CommonUtil.nullOrEmpty(request.getAssets())) {
+      // Nothing to Validate
+      return result
+          .withStatus(ApiStatus.SUCCESS)
+          .withSuccessRequest(List.of(new BulkResponse().withMessage("Nothing to Validate.")));
+    }
+
+    // Validation for entityReferences
+    EntityUtil.populateEntityReferences(request.getAssets());
+
+    TagLabel tagLabel =
+        new TagLabel()
+            .withTagFQN(tag.getFullyQualifiedName())
+            .withSource(TagSource.CLASSIFICATION)
+            .withLabelType(TagLabel.LabelType.MANUAL);
+
+    for (EntityReference ref : request.getAssets()) {
+      // Update Result Processed
+      result.setNumberOfRowsProcessed(result.getNumberOfRowsProcessed() + 1);
+
+      EntityRepository<?> entityRepository = Entity.getEntityRepository(ref.getType());
+      EntityInterface asset =
+          entityRepository.get(null, ref.getId(), entityRepository.getFields("tags"));
+
+      try {
+        Map<String, List<TagLabel>> allAssetTags =
+            daoCollection.tagUsageDAO().getTagsByPrefix(asset.getFullyQualifiedName(), "%", true);
+        checkMutuallyExclusiveForParentAndSubField(
+            asset.getFullyQualifiedName(),
+            FullyQualifiedName.buildHash(asset.getFullyQualifiedName()),
+            allAssetTags,
+            new ArrayList<>(Collections.singleton(tagLabel)),
+            false);
+        success.add(new BulkResponse().withRequest(ref));
+        result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1);
+      } catch (Exception ex) {
+        failures.add(new BulkResponse().withRequest(ref).withMessage(ex.getMessage()));
+        result.withFailedRequest(failures);
+        result.setNumberOfRowsFailed(result.getNumberOfRowsFailed() + 1);
+      }
+      // Validate and Store Tags
+      if (CommonUtil.nullOrEmpty(result.getFailedRequest())) {
+        List<TagLabel> tempList = new ArrayList<>(asset.getTags());
+        tempList.add(tagLabel);
+        // Apply Tags to Entities
+        entityRepository.applyTags(getUniqueTags(tempList), asset.getFullyQualifiedName());
+
+        searchRepository.updateEntity(ref);
+      }
+    }
+
+    // Add Failed And Suceess Request
+    result.withFailedRequest(failures).withSuccessRequest(success);
+
+    // Set Final Status
+    if (result.getNumberOfRowsPassed().equals(result.getNumberOfRowsProcessed())) {
+      result.withStatus(ApiStatus.SUCCESS);
+    } else if (result.getNumberOfRowsPassed() > 1) {
+      result.withStatus(ApiStatus.PARTIAL_SUCCESS);
+    } else {
+      result.withStatus(ApiStatus.FAILURE);
+    }
+
+    return result;
+  }
+
+  @Override
+  public BulkOperationResult bulkRemoveAndValidateTagsToAssets(
+      UUID classificationTagId, BulkAssetsRequestInterface request) {
+    Tag tag = this.get(null, classificationTagId, getFields("id"));
+
+    BulkOperationResult result =
+        new BulkOperationResult().withStatus(ApiStatus.SUCCESS).withDryRun(false);
+    List<BulkResponse> success = new ArrayList<>();
+
+    // Validation for entityReferences
+    EntityUtil.populateEntityReferences(request.getAssets());
+
+    for (EntityReference ref : request.getAssets()) {
+      // Update Result Processed
+      result.setNumberOfRowsProcessed(result.getNumberOfRowsProcessed() + 1);
+
+      EntityRepository<?> entityRepository = Entity.getEntityRepository(ref.getType());
+      EntityInterface asset =
+          entityRepository.get(null, ref.getId(), entityRepository.getFields("id"));
+
+      daoCollection
+          .tagUsageDAO()
+          .deleteTagsByTagAndTargetEntity(
+              tag.getFullyQualifiedName(), asset.getFullyQualifiedName());
+      success.add(new BulkResponse().withRequest(ref));
+      result.setNumberOfRowsPassed(result.getNumberOfRowsPassed() + 1);
+
+      // Update ES
+      searchRepository.updateEntity(ref);
+    }
+
+    return result.withSuccessRequest(success);
+  }
+
   @Override
   public EntityRepository<Tag>.EntityUpdater getUpdater(
       Tag original, Tag updated, Operation operation) {

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java
@@ -36,11 +36,13 @@
 import javax.ws.rs.core.UriInfo;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import org.openmetadata.schema.BulkAssetsRequestInterface;
 import org.openmetadata.schema.EntityInterface;
 import org.openmetadata.schema.type.EntityHistory;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.schema.type.MetadataOperation;
+import org.openmetadata.schema.type.api.BulkOperationResult;
 import org.openmetadata.schema.type.csv.CsvImportResult;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
@@ -56,6 +58,7 @@
 import org.openmetadata.service.security.policyevaluator.ResourceContext;
 import org.openmetadata.service.security.policyevaluator.ResourceContextInterface;
 import org.openmetadata.service.util.AsyncService;
+import org.openmetadata.service.util.BulkAssetsOperationResponse;
 import org.openmetadata.service.util.CSVExportResponse;
 import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.EntityUtil.Fields;
@@ -412,6 +415,56 @@ public Response exportCsvInternalAsync(SecurityContext securityContext, String n
     return Response.accepted().entity(response).type(MediaType.APPLICATION_JSON).build();
   }
 
+  public Response bulkAddToAssetsAsync(
+      SecurityContext securityContext, UUID entityId, BulkAssetsRequestInterface request) {
+    OperationContext operationContext =
+        new OperationContext(entityType, MetadataOperation.EDIT_ALL);
+    authorizer.authorize(securityContext, operationContext, getResourceContextById(entityId));
+    String jobId = UUID.randomUUID().toString();
+    ExecutorService executorService = AsyncService.getInstance().getExecutorService();
+    executorService.submit(
+        () -> {
+          try {
+            BulkOperationResult result =
+                repository.bulkAddAndValidateTagsToAssets(entityId, request);
+            WebsocketNotificationHandler.bulkAssetsOperationCompleteNotification(
+                jobId, securityContext, result);
+          } catch (Exception e) {
+            WebsocketNotificationHandler.bulkAssetsOperationFailedNotification(
+                jobId, securityContext, e.getMessage());
+          }
+        });
+    BulkAssetsOperationResponse response =
+        new BulkAssetsOperationResponse(
+            jobId, "Bulk Add tags to Asset operation initiated successfully.");
+    return Response.ok().entity(response).type(MediaType.APPLICATION_JSON).build();
+  }
+
+  public Response bulkRemoveFromAssetsAsync(
+      SecurityContext securityContext, UUID entityId, BulkAssetsRequestInterface request) {
+    OperationContext operationContext =
+        new OperationContext(entityType, MetadataOperation.EDIT_ALL);
+    authorizer.authorize(securityContext, operationContext, getResourceContextById(entityId));
+    String jobId = UUID.randomUUID().toString();
+    ExecutorService executorService = AsyncService.getInstance().getExecutorService();
+    executorService.submit(
+        () -> {
+          try {
+            BulkOperationResult result =
+                repository.bulkRemoveAndValidateTagsToAssets(entityId, request);
+            WebsocketNotificationHandler.bulkAssetsOperationCompleteNotification(
+                jobId, securityContext, result);
+          } catch (Exception e) {
+            WebsocketNotificationHandler.bulkAssetsOperationFailedNotification(
+                jobId, securityContext, e.getMessage());
+          }
+        });
+    BulkAssetsOperationResponse response =
+        new BulkAssetsOperationResponse(
+            jobId, "Bulk Remove tags to Asset operation initiated successfully.");
+    return Response.ok().entity(response).type(MediaType.APPLICATION_JSON).build();
+  }
+
   public String exportCsvInternal(SecurityContext securityContext, String name) throws IOException {
     OperationContext operationContext =
         new OperationContext(entityType, MetadataOperation.VIEW_ALL);

diff --git a/...service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java b/...service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java
@@ -355,7 +355,7 @@ public Response bulkAddAssets(
                     schema = @Schema(implementation = ChangeEvent.class))),
         @ApiResponse(responseCode = "404", description = "model for instance {id} is not found")
       })
-  public Response bulkRemoveGlossaryFromAssets(
+  public Response bulkRemoveAssets(
       @Context UriInfo uriInfo,
       @Context SecurityContext securityContext,
       @Parameter(description = "Name of the Data Product", schema = @Schema(type = "string"))

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java
@@ -179,6 +179,7 @@ public static void checkMutuallyExclusiveForParentAndSubField(
         checkMutuallyExclusive(getUniqueTags(tempList));
       } catch (IllegalArgumentException ex) {
         failed = true;
+        tempList.removeAll(glossaryTags);
         errorMessage.append(
             String.format(
                 "Asset %s has a tag %s which is mutually exclusive with the one of the glossary tags %s. %n",

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java
@@ -50,14 +50,17 @@
 import javax.ws.rs.core.SecurityContext;
 import javax.ws.rs.core.UriInfo;
 import lombok.extern.slf4j.Slf4j;
+import org.openmetadata.schema.api.AddTagToAssetsRequest;
 import org.openmetadata.schema.api.classification.CreateTag;
 import org.openmetadata.schema.api.classification.LoadTags;
 import org.openmetadata.schema.api.data.RestoreEntity;
 import org.openmetadata.schema.entity.classification.Classification;
 import org.openmetadata.schema.entity.classification.Tag;
+import org.openmetadata.schema.type.ChangeEvent;
 import org.openmetadata.schema.type.EntityHistory;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.schema.type.MetadataOperation;
+import org.openmetadata.schema.type.api.BulkOperationResult;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
 import org.openmetadata.service.jdbi3.ClassificationRepository;
@@ -503,6 +506,56 @@ public Response restore(
     return restoreEntity(uriInfo, securityContext, restore.getId());
   }
 
+  @PUT
+  @Path("/{id}/assets/add")
+  @Operation(
+      operationId = "bulkAddTagToAssets",
+      summary = "Bulk Add Classification Tag to Assets",
+      description = "Bulk Add Classification Tag to Assets",
+      responses = {
+        @ApiResponse(
+            responseCode = "200",
+            description = "OK",
+            content =
+                @Content(
+                    mediaType = "application/json",
+                    schema = @Schema(implementation = BulkOperationResult.class))),
+        @ApiResponse(responseCode = "404", description = "model for instance {id} is not found")
+      })
+  public Response bulkAddTagToAssets(
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @Parameter(description = "Id of the Entity", schema = @Schema(type = "UUID")) @PathParam("id")
+          UUID id,
+      @Valid AddTagToAssetsRequest request) {
+    return bulkAddToAssetsAsync(securityContext, id, request);
+  }
+
+  @PUT
+  @Path("/{id}/assets/remove")
+  @Operation(
+      operationId = "bulkRemoveTagFromAssets",
+      summary = "Bulk Remove Tag from Assets",
+      description = "Bulk Remove Tag from Assets",
+      responses = {
+        @ApiResponse(
+            responseCode = "200",
+            description = "OK",
+            content =
+                @Content(
+                    mediaType = "application/json",
+                    schema = @Schema(implementation = ChangeEvent.class))),
+        @ApiResponse(responseCode = "404", description = "model for instance {id} is not found")
+      })
+  public Response bulkRemoveTagFromAssets(
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @Parameter(description = "Id of the Entity", schema = @Schema(type = "UUID")) @PathParam("id")
+          UUID id,
+      @Valid AddTagToAssetsRequest request) {
+    return bulkRemoveFromAssetsAsync(securityContext, id, request);
+  }
+
   @Override
   public Tag addHref(UriInfo uriInfo, Tag tag) {
     super.addHref(uriInfo, tag);

diff --git a/...metadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java b/...metadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java
@@ -444,7 +444,7 @@ public Response bulkAddAssets(
                     schema = @Schema(implementation = ChangeEvent.class))),
         @ApiResponse(responseCode = "404", description = "model for instance {id} is not found")
       })
-  public Response bulkRemoveGlossaryFromAssets(
+  public Response bulkRemoveAssets(
       @Context UriInfo uriInfo,
       @Context SecurityContext securityContext,
       @Parameter(description = "Name of the Team", schema = @Schema(type = "string"))

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/socket/WebSocketManager.java
@@ -30,6 +30,8 @@ public class WebSocketManager {
   public static final String ANNOUNCEMENT_CHANNEL = "announcementChannel";
   public static final String CSV_EXPORT_CHANNEL = "csvExportChannel";
 
+  public static final String BULK_ASSETS_CHANNEL = "bulkAssetsChannel";
+
   @Getter
   private final Map<UUID, Map<String, SocketIoSocket>> activityFeedEndpoints =
       new ConcurrentHashMap<>();

diff --git a/...adata-service/src/main/java/org/openmetadata/service/util/BulkAssetsOperationMessage.java b/...adata-service/src/main/java/org/openmetadata/service/util/BulkAssetsOperationMessage.java
@@ -0,0 +1,22 @@
+package org.openmetadata.service.util;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import org.openmetadata.schema.type.api.BulkOperationResult;
+
+@NoArgsConstructor
+public class BulkAssetsOperationMessage {
+  @Getter @Setter private String jobId;
+  @Getter @Setter private String status;
+  @Getter @Setter private BulkOperationResult result;
+  @Getter @Setter private String error;
+
+  public BulkAssetsOperationMessage(
+      String jobId, String status, BulkOperationResult result, String error) {
+    this.jobId = jobId;
+    this.status = status;
+    this.result = result;
+    this.error = error;
+  }
+}