Minor: Remove ngram from the indexes

open-metadata · Jan 15, 2024 · 5bf2cae · 5bf2cae
1 parent 7a652a7
commit 5bf2cae
Show file tree

Hide file tree

Showing 44 changed files with 12 additions and 489 deletions.
diff --git a/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java b/...etadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java
@@ -7,7 +7,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 import org.openmetadata.schema.entity.data.Container;
 import org.openmetadata.schema.type.TagLabel;
 import org.openmetadata.service.Entity;
@@ -68,7 +67,7 @@ public Map<String, Object> buildESDoc() {
         "fqnParts",
         getFQNParts(
             container.getFullyQualifiedName(),
-            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
+            suggest.stream().map(SearchSuggest::getInput).toList()));
     doc.put("owner", getEntityWithDisplayName(container.getOwner()));
     doc.put("service", getEntityWithDisplayName(container.getService()));
     doc.put("domain", getEntityWithDisplayName(container.getDomain()));
@@ -79,9 +78,7 @@ public static Map<String, Float> getFields() {
     Map<String, Float> fields = SearchIndex.getDefaultFields();
     fields.put("dataModel.columns.name", 2.0f);
     fields.put(DATA_MODEL_COLUMNS_NAME_KEYWORD, 10.0f);
-    fields.put("dataModel.columns.name.ngram", 1.0f);
     fields.put("dataModel.columns.displayName", 2.0f);
-    fields.put("dataModel.columns.displayName.ngram", 1.0f);
     fields.put("dataModel.columns.description", 1.0f);
     fields.put("dataModel.columns.children.name", 2.0f);
     return fields;

diff --git a/...ervice/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java b/...ervice/src/main/java/org/openmetadata/service/search/indexes/DashboardDataModelIndex.java
@@ -7,7 +7,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 import org.openmetadata.schema.entity.data.DashboardDataModel;
 import org.openmetadata.schema.type.TagLabel;
 import org.openmetadata.service.Entity;
@@ -62,7 +61,7 @@ public Map<String, Object> buildESDoc() {
         "fqnParts",
         getFQNParts(
             dashboardDataModel.getFullyQualifiedName(),
-            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
+            suggest.stream().map(SearchSuggest::getInput).toList()));
     doc.put("tier", parseTags.getTierTag());
     doc.put("owner", getEntityWithDisplayName(dashboardDataModel.getOwner()));
     doc.put("service", getEntityWithDisplayName(dashboardDataModel.getService()));
@@ -75,7 +74,6 @@ public static Map<String, Float> getFields() {
     Map<String, Float> fields = SearchIndex.getDefaultFields();
     fields.put(COLUMNS_NAME_KEYWORD, 10.0f);
     fields.put("columns.name", 2.0f);
-    fields.put("columns.name.ngram", 1.0f);
     fields.put("columns.displayName", 1.0f);
     fields.put("columns.description", 1.0f);
     fields.put("columns.children.name", 2.0f);

diff --git a/...adata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java b/...adata-service/src/main/java/org/openmetadata/service/search/indexes/DataProductIndex.java
@@ -3,7 +3,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import org.openmetadata.schema.entity.domains.DataProduct;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
@@ -24,7 +23,7 @@ public Map<String, Object> buildESDoc() {
         "fqnParts",
         getFQNParts(
             dataProduct.getFullyQualifiedName(),
-            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
+            suggest.stream().map(SearchSuggest::getInput).toList()));
     doc.put("entityType", Entity.DATA_PRODUCT);
     doc.put("owner", getEntityWithDisplayName(dataProduct.getOwner()));
     doc.put("domain", getEntityWithDisplayName(dataProduct.getDomain()));

diff --git a/...data-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java b/...data-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java
@@ -43,7 +43,6 @@ public static Map<String, Float> getFields() {
     fields.put("synonyms.ngram", 1.0f);
     fields.put("glossary.name", 5.0f);
     fields.put("glossary.displayName", 5.0f);
-    fields.put("glossary.displayName.ngram", 1.0f);
     return fields;
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/MlModelIndex.java
@@ -3,7 +3,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import org.openmetadata.schema.entity.data.MlModel;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.ParseTags;
@@ -41,7 +40,7 @@ public Map<String, Object> buildESDoc() {
         "fqnParts",
         getFQNParts(
             mlModel.getFullyQualifiedName(),
-            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
+            suggest.stream().map(SearchSuggest::getInput).toList()));
     doc.put("owner", getEntityWithDisplayName(mlModel.getOwner()));
     doc.put("service", getEntityWithDisplayName(mlModel.getService()));
     doc.put("domain", getEntityWithDisplayName(mlModel.getDomain()));

diff --git a/...metadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java b/...metadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java
@@ -3,7 +3,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import org.openmetadata.schema.entity.data.Pipeline;
 import org.openmetadata.schema.type.Task;
 import org.openmetadata.service.Entity;
@@ -53,7 +52,7 @@ public Map<String, Object> buildESDoc() {
         "fqnParts",
         getFQNParts(
             pipeline.getFullyQualifiedName(),
-            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
+            suggest.stream().map(SearchSuggest::getInput).toList()));
     doc.put("owner", getEntityWithDisplayName(pipeline.getOwner()));
     doc.put("service", getEntityWithDisplayName(pipeline.getService()));
     doc.put("domain", getEntityWithDisplayName(pipeline.getDomain()));

diff --git a/...data-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java b/...data-service/src/main/java/org/openmetadata/service/search/indexes/SearchEntityIndex.java
@@ -3,7 +3,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.ParseTags;
 import org.openmetadata.service.search.SearchIndexUtils;
@@ -27,7 +26,7 @@ public Map<String, Object> buildESDoc() {
         "fqnParts",
         getFQNParts(
             searchIndex.getFullyQualifiedName(),
-            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
+            suggest.stream().map(SearchSuggest::getInput).toList()));
     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.SEARCH_INDEX, searchIndex));
     doc.put("tags", parseTags.getTags());
     doc.put("tier", parseTags.getTierTag());

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java
@@ -4,8 +4,6 @@
 import static org.openmetadata.service.Entity.FIELD_DISPLAY_NAME;
 import static org.openmetadata.service.Entity.FIELD_NAME;
 import static org.openmetadata.service.search.EntityBuilderConstant.DISPLAY_NAME_KEYWORD;
-import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_DISPLAY_NAME_NGRAM;
-import static org.openmetadata.service.search.EntityBuilderConstant.FIELD_NAME_NGRAM;
 import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME;
 import static org.openmetadata.service.search.EntityBuilderConstant.FULLY_QUALIFIED_NAME_PARTS;
 import static org.openmetadata.service.search.EntityBuilderConstant.NAME_KEYWORD;
@@ -135,9 +133,7 @@ static void getLineageDataDirection(
   static Map<String, Float> getDefaultFields() {
     Map<String, Float> fields = new HashMap<>();
     fields.put(FIELD_DISPLAY_NAME, 15.0f);
-    fields.put(FIELD_DISPLAY_NAME_NGRAM, 1.0f);
     fields.put(FIELD_NAME, 15.0f);
-    fields.put(FIELD_NAME_NGRAM, 1.0f);
     fields.put(DISPLAY_NAME_KEYWORD, 25.0f);
     fields.put(NAME_KEYWORD, 25.0f);
     fields.put(FIELD_DESCRIPTION, 1.0f);

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TableIndex.java
@@ -112,7 +112,6 @@ public static Map<String, Float> getFields() {
     Map<String, Float> fields = SearchIndex.getDefaultFields();
     fields.put(COLUMNS_NAME_KEYWORD, 10.0f);
     fields.put("columns.name", 2.0f);
-    fields.put("columns.name.ngram", 1.0f);
     fields.put("columns.displayName", 1.0f);
     fields.put("columns.description", 1.0f);
     fields.put("columns.children.name", 2.0f);

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TagIndex.java
@@ -3,7 +3,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import org.openmetadata.schema.entity.classification.Tag;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.SearchIndexUtils;
@@ -22,8 +21,7 @@ public Map<String, Object> buildESDoc() {
     doc.put(
         "fqnParts",
         getFQNParts(
-            tag.getFullyQualifiedName(),
-            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
+            tag.getFullyQualifiedName(), suggest.stream().map(SearchSuggest::getInput).toList()));
     if (tag.getDisabled() != null && tag.getDisabled()) {
       doc.put("disabled", tag.getDisabled());
     } else {

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TopicIndex.java
@@ -119,7 +119,6 @@ public static Map<String, Float> getFields() {
     Map<String, Float> fields = SearchIndex.getDefaultFields();
     fields.put(ES_MESSAGE_SCHEMA_FIELD, 2.0f);
     fields.put("messageSchema.schemaFields.name.keyword", 2.0f);
-    fields.put("messageSchema.schemaFields.name.ngram", 1.0f);
     fields.put("messageSchema.schemaFields.description", 1.0f);
     fields.put("messageSchema.schemaFields.children.name", 2.0f);
     fields.put("messageSchema.schemaFields.children.keyword", 2.0f);

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/UserIndex.java
@@ -3,7 +3,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.stream.Collectors;
 import org.openmetadata.common.utils.CommonUtil;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.service.Entity;
@@ -29,8 +28,7 @@ public Map<String, Object> buildESDoc() {
     doc.put(
         "fqnParts",
         getFQNParts(
-            user.getFullyQualifiedName(),
-            suggest.stream().map(SearchSuggest::getInput).collect(Collectors.toList())));
+            user.getFullyQualifiedName(), suggest.stream().map(SearchSuggest::getInput).toList()));
     doc.put("suggest", suggest);
     doc.put("entityType", Entity.USER);
     doc.put(

diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/chart_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/chart_index_mapping.json
@@ -17,14 +17,6 @@
             "lowercase",
             "om_stemmer"
           ]
-        },
-        "om_ngram": {
-          "tokenizer": "ngram",
-          "min_gram": 3,
-          "max_gram": 10,
-          "filter": [
-            "lowercase"
-          ]
         }
       },
       "filter": {
@@ -48,21 +40,14 @@
             "type": "keyword",
             "ignore_above": 256,
             "normalizer": "lowercase_normalizer"
-          },
-          "ngram": {
-            "type": "text",
-            "analyzer": "om_ngram"
           }
         }
       },
       "displayName": {
         "type": "text",
         "analyzer": "om_analyzer",
         "fields": {
-          "ngram": {
-            "type": "text",
-            "analyzer": "om_ngram"
-          },
+        "fields": {
           "keyword": {
             "type": "keyword",
             "ignore_above": 256

diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/classification_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/classification_index_mapping.json
@@ -17,14 +17,6 @@
             "lowercase",
             "om_stemmer"
           ]
-        },
-        "om_ngram": {
-          "tokenizer": "ngram",
-          "min_gram": 3,
-          "max_gram": 10,
-          "filter": [
-            "lowercase"
-          ]
         }
       },
       "filter": {
@@ -48,10 +40,6 @@
             "type": "keyword",
             "ignore_above": 256,
             "normalizer": "lowercase_normalizer"
-          },
-          "ngram": {
-            "type": "text",
-            "analyzer": "om_ngram"
           }
         }
       },
@@ -69,10 +57,6 @@
           "keyword": {
             "type": "keyword",
             "ignore_above": 256
-          },
-          "ngram": {
-            "type": "text",
-            "analyzer": "om_ngram"
           }
         }
       },

diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/container_index_mapping.json
@@ -17,14 +17,6 @@
             "lowercase",
             "om_stemmer"
           ]
-        },
-        "om_ngram": {
-          "tokenizer": "ngram",
-          "min_gram": 3,
-          "max_gram": 10,
-          "filter": [
-            "lowercase"
-          ]
         }
       },
       "filter": {
@@ -48,10 +40,6 @@
             "type": "keyword",
             "ignore_above": 256,
             "normalizer": "lowercase_normalizer"
-          },
-          "ngram": {
-            "type": "text",
-            "analyzer": "om_ngram"
           }
         }
       },
@@ -64,13 +52,7 @@
       },
       "displayName": {
         "type": "text",
-        "analyzer": "om_analyzer",
-        "fields": {
-          "ngram": {
-            "type": "text",
-            "analyzer": "om_ngram"
-          }
-        }
+        "analyzer": "om_analyzer"
       },
       "description": {
         "type": "text",
@@ -149,10 +131,6 @@
                     "type": "keyword",
                     "ignore_above": 256,
                     "normalizer": "lowercase_normalizer"
-                  },
-                  "ngram": {
-                    "type": "text",
-                    "analyzer": "om_ngram"
                   }
                 }
               },

diff --git a/...adata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json b/...adata-service/src/main/resources/elasticsearch/en/dashboard_data_model_index_mapping.json
@@ -17,14 +17,6 @@
             "lowercase",
             "om_stemmer"
           ]
-        },
-        "om_ngram": {
-          "tokenizer": "ngram",
-          "min_gram": 3,
-          "max_gram": 10,
-          "filter": [
-            "lowercase"
-          ]
         }
       },
       "filter": {
@@ -48,21 +40,13 @@
             "type": "keyword",
             "ignore_above": 256,
             "normalizer": "lowercase_normalizer"
-          },
-          "ngram": {
-            "type": "text",
-            "analyzer": "om_ngram"
           }
         }
       },
       "displayName": {
         "type": "text",
         "analyzer": "om_analyzer",
         "fields": {
-          "ngram": {
-            "type": "text",
-            "analyzer": "om_ngram"
-          },
           "keyword": {
             "type": "keyword",
             "ignore_above": 256

diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/dashboard_index_mapping.json
@@ -17,14 +17,6 @@
             "lowercase",
             "om_stemmer"
           ]
-        },
-        "om_ngram": {
-          "tokenizer": "ngram",
-          "min_gram": 3,
-          "max_gram": 10,
-          "filter": [
-            "lowercase"
-          ]
         }
       },
       "filter": {
@@ -48,10 +40,6 @@
             "type": "keyword",
             "ignore_above": 256,
             "normalizer": "lowercase_normalizer"
-          },
-          "ngram": {
-            "type": "text",
-            "analyzer": "om_ngram"
           }
         }
       },
@@ -66,10 +54,6 @@
         "type": "text",
         "analyzer": "om_analyzer",
         "fields": {
-          "ngram": {
-            "type": "text",
-            "analyzer": "om_ngram"
-          },
           "keyword": {
             "type": "keyword",
             "ignore_above": 256