Improve Search Relevancy for tier, usage and use stemming to generate…

… ngrams (#17871) * Improve Search Relevancy for tier, usage and use stemming to generate ngrams * Improve Search Relevancy for tier, usage and use stemming to generate ngrams * add displayName using name --------- Co-authored-by: Sachin Chaurasiya <sachinchaurasiyachotey87@gmail.com>
open-metadata · Sep 18, 2024 · 4d0339b · 4d0339b
1 parent 91a1479
commit 4d0339b
Show file tree

Hide file tree

Showing 54 changed files with 830 additions and 474 deletions.
diff --git a/...adata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java b/...adata-service/src/main/java/org/openmetadata/service/resources/search/SearchResource.java
@@ -171,7 +171,13 @@ public Response search(
                   "Fetch search results in hierarchical order of children elements. By default hierarchy is not fetched. Currently only supported for glossary_term_search_index.")
           @DefaultValue("false")
           @QueryParam("getHierarchy")
-          boolean getHierarchy)
+          boolean getHierarchy,
+      @Parameter(
+              description =
+                  "Explain the results of the query. Defaults to false. Only for debugging purposes.")
+          @DefaultValue("false")
+          @QueryParam("explain")
+          boolean explain)
       throws IOException {
 
     if (nullOrEmpty(query)) {
@@ -202,6 +208,7 @@ public Response search(
             .applyDomainFilter(
                 !subjectContext.isAdmin() && subjectContext.hasAnyRole(DOMAIN_ONLY_ACCESS_ROLE))
             .searchAfter(searchAfter)
+            .explain(explain)
             .build();
     return searchRepository.search(request, subjectContext);
   }

diff --git a/...metadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java b/...metadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java
@@ -9,6 +9,9 @@ public class EntityBuilderConstant {
   public static final String ES_MESSAGE_SCHEMA_FIELD = "messageSchema.schemaFields.name";
   public static final String ES_MESSAGE_SCHEMA_FIELD_KEYWORD =
       "messageSchema.schemaFields.name.keyword";
+  public static final String API_RESPONSE_SCHEMA_FIELD = "responseSchema.schemaFields.name";
+  public static final String API_RESPONSE_SCHEMA_FIELD_KEYWORD =
+      "responseSchema.schemaFields.name.keyword";
   public static final String ES_TAG_FQN_FIELD = "tags.tagFQN";
 
   public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword";

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRequest.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRequest.java
@@ -29,6 +29,7 @@ public class SearchRequest {
   private final List<String> domains;
   private final boolean getHierarchy;
   private final Object[] searchAfter;
+  private final boolean explain;
 
   public SearchRequest(ElasticSearchRequestBuilder builder) {
     this.query = builder.query;
@@ -48,6 +49,7 @@ public SearchRequest(ElasticSearchRequestBuilder builder) {
     this.domains = builder.domains;
     this.applyDomainFilter = builder.applyDomainFilter;
     this.searchAfter = builder.searchAfter;
+    this.explain = builder.explain;
   }
 
   // Builder class for ElasticSearchRequest
@@ -70,6 +72,7 @@ public static class ElasticSearchRequestBuilder {
     private boolean applyDomainFilter;
     private List<String> domains;
     private Object[] searchAfter;
+    private boolean explain;
 
     public ElasticSearchRequestBuilder(String query, int size, String index) {
       this.query = query;
@@ -153,6 +156,11 @@ public ElasticSearchRequestBuilder searchAfter(String searchAfter) {
       return this;
     }
 
+    public ElasticSearchRequestBuilder explain(boolean explain) {
+      this.explain = explain;
+      return this;
+    }
+
     public SearchRequest build() {
       return new SearchRequest(this);
     }

diff --git a/...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
diff --git a/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java b/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java
@@ -57,9 +57,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
 
     Map<String, Object> commonAttributes = getCommonAttributesMap(container, Entity.CONTAINER);
     doc.putAll(commonAttributes);
-    doc.put(
-        "displayName",
-        container.getDisplayName() != null ? container.getDisplayName() : container.getName());
     doc.put("tags", flattenedTagList);
     doc.put("tier", parseTags.getTierTag());
     doc.put("service_suggest", serviceSuggest);

diff --git a/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java b/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java
@@ -48,10 +48,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
     }
     Map<String, Object> commonAttributes = getCommonAttributesMap(dashboard, Entity.DASHBOARD);
     doc.putAll(commonAttributes);
-
-    doc.put(
-        "displayName",
-        dashboard.getDisplayName() != null ? dashboard.getDisplayName() : dashboard.getName());
     doc.put("tags", parseTags.getTags());
     doc.put("tier", parseTags.getTierTag());
     doc.put("chart_suggest", chartSuggest);

diff --git a/...service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java b/...service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java
@@ -60,11 +60,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
         ingestionPipeline.getName() != null
             ? ingestionPipeline.getName()
             : ingestionPipeline.getDisplayName());
-    doc.put(
-        "displayName",
-        ingestionPipeline.getDisplayName() != null
-            ? ingestionPipeline.getDisplayName()
-            : ingestionPipeline.getName());
     doc.put("tags", parseTags.getTags());
     doc.put("tier", parseTags.getTierTag());
     doc.put("service_suggest", serviceSuggest);

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java
@@ -32,9 +32,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.MLMODEL, mlModel));
     Map<String, Object> commonAttributes = getCommonAttributesMap(mlModel, Entity.MLMODEL);
     doc.putAll(commonAttributes);
-    doc.put(
-        "displayName",
-        mlModel.getDisplayName() != null ? mlModel.getDisplayName() : mlModel.getName());
     doc.put("tags", parseTags.getTags());
     doc.put("tier", parseTags.getTierTag());
     doc.put("serviceType", mlModel.getServiceType());

diff --git a/...metadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java b/...metadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java
@@ -46,9 +46,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
     Map<String, Object> commonAttributes = getCommonAttributesMap(pipeline, Entity.PIPELINE);
     doc.putAll(commonAttributes);
     doc.put("name", pipeline.getName() != null ? pipeline.getName() : pipeline.getDisplayName());
-    doc.put(
-        "displayName",
-        pipeline.getDisplayName() != null ? pipeline.getDisplayName() : pipeline.getName());
     doc.put("tags", parseTags.getTags());
     doc.put("tier", parseTags.getTierTag());
     doc.put("task_suggest", taskSuggest);

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/QueryIndex.java
@@ -36,7 +36,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.QUERY, query));
     Map<String, Object> commonAttributes = getCommonAttributesMap(query, Entity.QUERY);
     doc.putAll(commonAttributes);
-    doc.put("displayName", query.getDisplayName() != null ? query.getDisplayName() : "");
     doc.put("tags", parseTags.getTags());
     doc.put("tier", parseTags.getTierTag());
     return doc;

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java
@@ -3,14 +3,11 @@
 import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
 import static org.openmetadata.service.Entity.FIELD_DESCRIPTION;
 import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
-import static org.openmetadata.service.Entity.FIELD_NAME;
 import static org.openmetadata.service.jdbi3.LineageRepository.buildRelationshipDetailsMap;
 import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
 import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
-import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
 import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME;
 import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
-import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -68,6 +65,9 @@ default List<SearchSuggest> getSuggest() {
   default Map<String, Object> getCommonAttributesMap(EntityInterface entity, String entityType) {
     Map<String, Object> map = new HashMap<>();
     List<SearchSuggest> suggest = getSuggest();
+    map.put(
+        "displayName",
+        entity.getDisplayName() != null ? entity.getDisplayName() : entity.getName());
     map.put("entityType", entityType);
     map.put("owners", getEntitiesWithDisplayName(entity.getOwners()));
     map.put("domain", getEntityWithDisplayName(entity.getDomain()));
@@ -164,12 +164,9 @@ static List<Map<String, Object>> getLineageData(EntityReference entity) {
 
   static Map<String, Float> getDefaultFields() {
     Map<String, Float> fields = new HashMap<>();
-    fields.put(FIELD_DISPLAY_NAME, 10.0f);
+    fields.put(DISPLAY_NAME_KEYWORD, 10.0f);
     fields.put(FIELD_DISPLAY_NAME_NGRAM, 1.0f);
-    fields.put(FIELD_NAME, 10.0f);
-    fields.put(FIELD_NAME_NGRAM, 1.0f);
-    fields.put(DISPLAY_NAME_KEYWORD, 8.0f);
-    fields.put(NAME_KEYWORD, 8.0f);
+    fields.put(FIELD_DISPLAY_NAME, 10.0f);
     fields.put(FIELD_DESCRIPTION, 2.0f);
     fields.put(FULLY_QUALIFIED_NAME, 5.0f);
     fields.put(FULLY_QUALIFIED_NAME_PARTS, 5.0f);
@@ -179,6 +176,7 @@ static Map<String, Float> getDefaultFields() {
   static Map<String, Float> getAllFields() {
     Map<String, Float> fields = getDefaultFields();
     fields.putAll(TableIndex.getFields());
+    fields.putAll(StoredProcedureIndex.getFields());
     fields.putAll(DashboardIndex.getFields());
     fields.putAll(DashboardDataModelIndex.getFields());
     fields.putAll(PipelineIndex.getFields());

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java
@@ -91,8 +91,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
             .collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
     Map<String, Object> commonAttributes = getCommonAttributesMap(table, Entity.TABLE);
     doc.putAll(commonAttributes);
-    doc.put(
-        "displayName", table.getDisplayName() != null ? table.getDisplayName() : table.getName());
     doc.put("tags", flattenedTagList);
     doc.put("tier", parseTags.getTierTag());
     doc.put("service_suggest", serviceSuggest);
@@ -113,7 +111,7 @@ public static Map<String, Float> getFields() {
     fields.put("columns.name", 5.0f);
     fields.put("columns.displayName", 5.0f);
     fields.put("columns.description", 2.0f);
-    fields.put("columns.children.name", 5.0f);
+    fields.put("columns.children.name", 3.0f);
     return fields;
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TeamIndex.java
@@ -4,7 +4,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.teams.Team;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.models.SearchSuggest;
@@ -39,9 +38,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
     Map<String, Object> commonAttributes = getCommonAttributesMap(team, Entity.TEAM);
     doc.putAll(commonAttributes);
     doc.put("isBot", false);
-    doc.put(
-        "displayName",
-        CommonUtil.nullOrEmpty(team.getDisplayName()) ? team.getName() : team.getDisplayName());
     return doc;
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java
@@ -76,8 +76,6 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
             .collect(ArrayList::new, ArrayList::add, ArrayList::addAll);
     Map<String, Object> commonAttributes = getCommonAttributesMap(topic, Entity.TOPIC);
     doc.putAll(commonAttributes);
-    doc.put(
-        "displayName", topic.getDisplayName() != null ? topic.getDisplayName() : topic.getName());
     doc.put("tags", flattenedTagList);
     doc.put("tier", parseTags.getTierTag());
     doc.put("field_suggest", fieldSuggest);

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java
@@ -4,7 +4,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.models.SearchSuggest;
@@ -38,9 +37,6 @@ public Set<String> getExcludedFields() {
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     Map<String, Object> commonAttributes = getCommonAttributesMap(user, Entity.USER);
     doc.putAll(commonAttributes);
-    doc.put(
-        "displayName",
-        CommonUtil.nullOrEmpty(user.getDisplayName()) ? user.getName() : user.getDisplayName());
     if (user.getIsBot() == null) {
       doc.put("isBot", false);
     }