Fix open-metadata#14308: Search is not working for column/schema fiel…

…d level (open-metadata#14310)
MrVinegar · Dec 12, 2023 · a26fd5a · a26fd5a
1 parent 9f89e44
commit a26fd5a
Show file tree

Hide file tree

Showing 17 changed files with 121 additions and 42 deletions.
diff --git a/...metadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java b/...metadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java
@@ -7,6 +7,7 @@ public class EntityBuilderConstant {
   public static final String FIELD_TAGS = "tags";
   public static final String UNIFIED = "unified";
   public static final String ES_MESSAGE_SCHEMA_FIELD = "messageSchema.schemaFields.name";
+  public static final String ES_MESSAGE_SCHEMA_FIELD_KEYWORD = "messageSchema.schemaFields.name.keyword";
   public static final String ES_TAG_FQN_FIELD = "tags.tagFQN";
 
   public static final String COLUMNS_NAME_KEYWORD = "columns.name.keyword";

diff --git a/...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
@@ -9,7 +9,7 @@
 import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD;
 import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD;
 import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD;
-import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD;
+import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD_KEYWORD;
 import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD;
 import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES;
 import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE;
@@ -545,7 +545,7 @@ private static SearchSourceBuilder buildTopicSearchBuilder(String query, int fro
     hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED));
     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size);
     searchSourceBuilder
-        .aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD))
+        .aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD_KEYWORD).field(ES_MESSAGE_SCHEMA_FIELD_KEYWORD))
         .aggregation(AggregationBuilders.terms(SCHEMA_FIELD_NAMES).field(SCHEMA_FIELD_NAMES));
     return addAggregation(searchSourceBuilder);
   }
@@ -883,7 +883,11 @@ private static QueryStringQueryBuilder buildSearchQueryBuilder(String query, Map
 
   private static SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, int size) {
     QueryStringQueryBuilder queryBuilder =
-        QueryBuilders.queryStringQuery(query).fields(SearchIndex.getDefaultFields()).lenient(true);
+        QueryBuilders.queryStringQuery(query)
+            .fields(SearchIndex.getAllFields())
+            .defaultOperator(Operator.AND)
+            .fuzziness(Fuzziness.AUTO);
+    ;
     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
     return addAggregation(searchSourceBuilder);
   }

diff --git a/...data-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java b/...data-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java
@@ -45,6 +45,7 @@ public Map<String, Object> buildESDoc() {
   public static Map<String, Float> getFields() {
     Map<String, Float> fields = SearchIndex.getDefaultFields();
     fields.put("fields.name", 2.0f);
+    fields.put("fields.name.keyword", 2.0f);
     fields.put("fields.children.description", 1.0f);
     fields.put("fields.children.name", 2.0f);
     return fields;

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java
@@ -58,4 +58,20 @@ static Map<String, Float> getDefaultFields() {
     fields.put(FULLY_QUALIFIED_NAME_PARTS, 10.0f);
     return fields;
   }
+
+  static Map<String, Float> getAllFields() {
+    Map<String, Float> fields = getDefaultFields();
+    fields.putAll(TableIndex.getFields());
+    fields.putAll(DashboardIndex.getFields());
+    fields.putAll(DashboardDataModelIndex.getFields());
+    fields.putAll(PipelineIndex.getFields());
+    fields.putAll(TopicIndex.getFields());
+    fields.putAll(MlModelIndex.getFields());
+    fields.putAll(ContainerIndex.getFields());
+    fields.putAll(SearchEntityIndex.getFields());
+    fields.putAll(GlossaryTermIndex.getFields());
+    fields.putAll(TagIndex.getFields());
+    fields.putAll(DataProductIndex.getFields());
+    return fields;
+  }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java
@@ -110,8 +110,11 @@ private void parseSchemaFields(
   public static Map<String, Float> getFields() {
     Map<String, Float> fields = SearchIndex.getDefaultFields();
     fields.put(ES_MESSAGE_SCHEMA_FIELD, 2.0f);
+    fields.put("messageSchema.schemaFields.name.keyword", 2.0f);
+    fields.put("messageSchema.schemaFields.name.ngram", 1.0f);
     fields.put("messageSchema.schemaFields.description", 1.0f);
     fields.put("messageSchema.schemaFields.children.name", 2.0f);
+    fields.put("messageSchema.schemaFields.children.keyword", 2.0f);
     return fields;
   }
 }
diff --git a/...ta-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/...ta-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
@@ -9,7 +9,7 @@
 import static org.openmetadata.service.search.EntityBuilderConstant.COLUMNS_NAME_KEYWORD;
 import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD;
 import static org.openmetadata.service.search.EntityBuilderConstant.DOMAIN_DISPLAY_NAME_KEYWORD;
-import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD;
+import static org.openmetadata.service.search.EntityBuilderConstant.ES_MESSAGE_SCHEMA_FIELD_KEYWORD;
 import static org.openmetadata.service.search.EntityBuilderConstant.ES_TAG_FQN_FIELD;
 import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_COLUMN_NAMES;
 import static org.openmetadata.service.search.EntityBuilderConstant.MAX_AGGREGATE_SIZE;
@@ -553,7 +553,7 @@ private static SearchSourceBuilder buildTopicSearchBuilder(String query, int fro
     hb.field(new HighlightBuilder.Field("messageSchema.schemaFields.children.name").highlighterType(UNIFIED));
     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size);
     searchSourceBuilder
-        .aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD).field(ES_MESSAGE_SCHEMA_FIELD))
+        .aggregation(AggregationBuilders.terms(ES_MESSAGE_SCHEMA_FIELD_KEYWORD).field(ES_MESSAGE_SCHEMA_FIELD_KEYWORD))
         .aggregation(AggregationBuilders.terms(SCHEMA_FIELD_NAMES).field(SCHEMA_FIELD_NAMES));
     return addAggregation(searchSourceBuilder);
   }
@@ -871,7 +871,10 @@ private static QueryStringQueryBuilder buildSearchQueryBuilder(String query, Map
 
   private static SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, int size) {
     QueryStringQueryBuilder queryBuilder =
-        QueryBuilders.queryStringQuery(query).fields(SearchIndex.getDefaultFields()).lenient(true);
+        QueryBuilders.queryStringQuery(query)
+            .fields(SearchIndex.getAllFields())
+            .defaultOperator(Operator.AND)
+            .fuzziness(Fuzziness.AUTO);
     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
     return addAggregation(searchSourceBuilder);
   }

diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json
@@ -42,11 +42,16 @@
       },
       "name": {
         "type": "text",
+        "analyzer": "om_analyzer",
         "fields": {
           "keyword": {
             "type": "keyword",
             "ignore_above": 256,
             "normalizer": "lowercase_normalizer"
+          },
+          "ngram": {
+            "type": "text",
+            "analyzer": "om_ngram"
           }
         }
       },
@@ -134,13 +139,17 @@
           "columns": {
             "properties": {
               "name": {
-                "type": "keyword",
-                "normalizer": "lowercase_normalizer",
+                "type": "text",
+                "analyzer": "om_analyzer",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
-                    "normalizer": "lowercase_normalizer",
-                    "ignore_above": 256
+                    "ignore_above": 256,
+                    "normalizer": "lowercase_normalizer"
+                  },
+                  "ngram": {
+                    "type": "text",
+                    "analyzer": "om_ngram"
                   }
                 }
               },

diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/search_entity_index_mapping.json
@@ -103,13 +103,17 @@
       "fields": {
         "properties": {
           "name": {
-            "type": "keyword",
-            "normalizer": "lowercase_normalizer",
+            "type": "text",
+            "analyzer": "om_analyzer",
             "fields": {
               "keyword": {
                 "type": "keyword",
-                "normalizer": "lowercase_normalizer",
-                "ignore_above": 256
+                "ignore_above": 256,
+                "normalizer": "lowercase_normalizer"
+              },
+              "ngram": {
+                "type": "text",
+                "analyzer": "om_ngram"
               }
             }
           },

diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/table_index_mapping.json
@@ -100,13 +100,17 @@
       "columns": {
         "properties": {
           "name": {
-            "type": "keyword",
-            "normalizer": "lowercase_normalizer",
+            "type": "text",
+            "analyzer": "om_analyzer",
             "fields": {
               "keyword": {
                 "type": "keyword",
-                "normalizer": "lowercase_normalizer",
-                "ignore_above": 256
+                "ignore_above": 256,
+                "normalizer": "lowercase_normalizer"
+              },
+              "ngram": {
+                "type": "text",
+                "analyzer": "om_ngram"
               }
             }
           },

diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/topic_index_mapping.json
@@ -201,12 +201,17 @@
           "schemaFields": {
             "properties": {
               "name": {
-                "type": "keyword",
-                "normalizer": "lowercase_normalizer",
+                "type": "text",
+                "analyzer": "om_analyzer",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
-                    "ignore_above": 256
+                    "ignore_above": 256,
+                    "normalizer": "lowercase_normalizer"
+                  },
+                  "ngram": {
+                    "type": "text",
+                    "analyzer": "om_ngram"
                   }
                 }
               },

diff --git a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json
@@ -15,13 +15,13 @@
     "indexName": "database_schema_search_index",
     "indexMappingFile": "/elasticsearch/%s/database_schema_index_mapping.json",
     "alias": "databaseSchema",
-    "parentAliases": ["database", "databaseService", "all"]
+    "parentAliases": ["database", "databaseService", "all", "dataAsset"]
   },
   "database": {
     "indexName": "database_search_index",
     "alias": "database",
     "indexMappingFile": "/elasticsearch/%s/database_index_mapping.json",
-    "parentAliases": ["databaseService", "all"]
+    "parentAliases": ["databaseService", "all", "dataAsset"]
   },
   "chart": {
     "indexName": "chart_search_index",
@@ -93,7 +93,7 @@
     "indexName": "data_product_search_index",
     "indexMappingFile": "/elasticsearch/%s/data_products_index_mapping.json",
     "alias": "dataProduct",
-    "parentAliases": ["domain","all"]
+    "parentAliases": ["domain","all", "dataAsset"]
   },
   "domain": {
     "indexName": "domain_search_index",

diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/container_index_mapping.json
@@ -45,11 +45,16 @@
       },
       "name": {
         "type": "text",
+        "analyzer": "om_analyzer_jp",
         "fields": {
           "keyword": {
             "type": "keyword",
             "ignore_above": 256,
             "normalizer": "lowercase_normalizer"
+          },
+          "ngram": {
+            "type": "text",
+            "analyzer": "om_ngram"
           }
         }
       },
@@ -143,12 +148,17 @@
             "type": "keyword"
           },
           "name": {
-            "type": "keyword",
-            "normalizer": "lowercase_normalizer",
+            "type": "text",
+            "analyzer": "om_analyzer_jp",
             "fields": {
               "keyword": {
                 "type": "keyword",
-                "ignore_above": 256
+                "ignore_above": 256,
+                "normalizer": "lowercase_normalizer"
+              },
+              "ngram": {
+                "type": "text",
+                "analyzer": "om_ngram"
               }
             }
           },

diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json
@@ -105,12 +105,17 @@
       "fields": {
         "properties": {
           "name": {
-            "type": "keyword",
-            "normalizer": "lowercase_normalizer",
+            "type": "text",
+            "analyzer": "om_analyzer_jp",
             "fields": {
               "keyword": {
                 "type": "keyword",
-                "ignore_above": 256
+                "ignore_above": 256,
+                "normalizer": "lowercase_normalizer"
+              },
+              "ngram": {
+                "type": "text",
+                "analyzer": "om_ngram"
               }
             }
           },

diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/table_index_mapping.json
@@ -112,12 +112,17 @@
       "columns": {
         "properties": {
           "name": {
-            "type": "keyword",
-            "normalizer": "lowercase_normalizer",
+            "type": "text",
+            "analyzer": "om_analyzer_jp",
             "fields": {
               "keyword": {
                 "type": "keyword",
-                "ignore_above": 256
+                "ignore_above": 256,
+                "normalizer": "lowercase_normalizer"
+              },
+              "ngram": {
+                "type": "text",
+                "analyzer": "om_ngram"
               }
             }
           },

diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/topic_index_mapping.json
@@ -107,12 +107,17 @@
           "schemaFields": {
             "properties": {
               "name": {
-                "type": "keyword",
-                "normalizer": "lowercase_normalizer",
+                "type": "text",
+                "analyzer": "om_analyzer_jp",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
-                    "ignore_above": 256
+                    "ignore_above": 256,
+                    "normalizer": "lowercase_normalizer"
+                  },
+                  "ngram": {
+                    "type": "text",
+                    "analyzer": "om_ngram"
                   }
                 }
               },

diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/table_index_mapping.json
@@ -83,12 +83,14 @@
       "columns": {
         "properties": {
           "name": {
-            "type": "keyword",
-            "normalizer": "lowercase_normalizer",
+            "type": "text",
+            "analyzer": "ik_max_word",
+            "search_analyzer": "ik_smart",
             "fields": {
               "keyword": {
                 "type": "keyword",
-                "ignore_above": 256
+                "ignore_above": 256,
+                "normalizer": "lowercase_normalizer"
               }
             }
           },

diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/topic_index_mapping.json
@@ -169,12 +169,14 @@
           "schemaFields": {
             "properties": {
               "name": {
-                "type": "keyword",
-                "normalizer": "lowercase_normalizer",
+                "type": "text",
+                "analyzer": "ik_max_word",
+                "search_analyzer": "ik_smart",
                 "fields": {
                   "keyword": {
                     "type": "keyword",
-                    "ignore_above": 256
+                    "ignore_above": 256,
+                    "normalizer": "lowercase_normalizer"
                   }
                 }
               },