feat(search): supporting chinese glossaryterm full text retrieval(#3914)

docker/datahub-gms/env/docker.env

@@ -33,6 +33,10 @@ MCE_CONSUMER_ENABLED=true
 # METADATA_CHANGE_EVENT_NAME=MetadataChangeEvent_v4
 # FAILED_METADATA_CHANGE_EVENT_NAME=FailedMetadataChangeEvent_v4
 
+# Uncomment and Support specific language analyzer for full-text retrieval
+# Be careful Elasticsearch is required to support the tokenizer of this plug-in
+# ELASTICSEARCH_MAIN_TOKENIZER=smartcn_tokenizer
+
 # Uncomment and set these to support SSL connection to Elasticsearch
 # ELASTICSEARCH_USE_SSL=true
 # ELASTICSEARCH_SSL_PROTOCOL=TLSv1.2

metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/SettingsBuilder.java

@@ -2,6 +2,8 @@
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import org.apache.commons.lang3.StringUtils;
+
 import java.util.List;
 import java.util.Map;
 
@@ -12,21 +14,21 @@
 public class SettingsBuilder {
   private final Map<String, Object> settings;
 
-  public SettingsBuilder(List<String> urnStopWords) {
-    settings = buildSettings(urnStopWords);
+  public SettingsBuilder(List<String> urnStopWords, String mainTokenizer) {
+    settings = buildSettings(urnStopWords, mainTokenizer);
   }
 
   public Map<String, Object> getSettings() {
     return settings;
   }
 
-  private static Map<String, Object> buildSettings(List<String> urnStopWords) {
+  private static Map<String, Object> buildSettings(List<String> urnStopWords, String mainTokenizer) {
     ImmutableMap.Builder<String, Object> settings = ImmutableMap.builder();
     settings.put("max_ngram_diff", 17);
     settings.put("analysis", ImmutableMap.<String, Object>builder().put("filter", buildFilters(urnStopWords))
         .put("tokenizer", buildTokenizers())
         .put("normalizer", buildNormalizers())
-        .put("analyzer", buildAnalyzers())
+        .put("analyzer", buildAnalyzers(mainTokenizer))
         .build());
     return settings.build();
   }
@@ -79,15 +81,18 @@ private static Map<String, Object> buildNormalizers() {
   }
 
   // Analyzers turn fields into multiple tokens
-  private static Map<String, Object> buildAnalyzers() {
+  private static Map<String, Object> buildAnalyzers(String mainTokenizer) {
     ImmutableMap.Builder<String, Object> analyzers = ImmutableMap.builder();
+    // For special analysis, the substitution can be read from the configuration (chinese tokenizer: ik_smart / smartCN)
     // Analyzer for partial matching (i.e. autocomplete) - Prefix matching of each token
-    analyzers.put("partial", ImmutableMap.<String, Object>builder().put("tokenizer", "main_tokenizer")
+    analyzers.put("partial", ImmutableMap.<String, Object>builder()
+        .put("tokenizer", StringUtils.isNotBlank(mainTokenizer) ? mainTokenizer : "main_tokenizer")
         .put("filter", ImmutableList.of("custom_delimiter", "lowercase", "partial_filter"))
         .build());
 
     // Analyzer for text tokenized into words (split by spaces, periods, and slashes)
-    analyzers.put("word_delimited", ImmutableMap.<String, Object>builder().put("tokenizer", "main_tokenizer")
+    analyzers.put("word_delimited", ImmutableMap.<String, Object>builder()
+        .put("tokenizer", StringUtils.isNotBlank(mainTokenizer) ? mainTokenizer : "main_tokenizer")
         .put("filter", ImmutableList.of("custom_delimiter", "lowercase", "stop"))
         .build());
 

metadata-io/src/test/java/com/linkedin/metadata/search/SearchServiceTest.java

@@ -53,7 +53,7 @@ public void setup() {
     _entityRegistry = new SnapshotEntityRegistry(new Snapshot());
     _indexConvention = new IndexConventionImpl(null);
     _elasticsearchContainer = ElasticTestUtils.getNewElasticsearchContainer();
-    _settingsBuilder = new SettingsBuilder(Collections.emptyList());
+    _settingsBuilder = new SettingsBuilder(Collections.emptyList(), null);
     checkContainerEngine(_elasticsearchContainer.getDockerClient());
     _elasticsearchContainer.start();
     _searchClient = ElasticTestUtils.buildRestClient(_elasticsearchContainer);

metadata-io/src/test/java/com/linkedin/metadata/search/elasticsearch/ElasticSearchServiceTest.java

@@ -54,7 +54,7 @@ public void setup() {
     _entityRegistry = new SnapshotEntityRegistry(new Snapshot());
     _indexConvention = new IndexConventionImpl(null);
     _elasticsearchContainer = ElasticTestUtils.getNewElasticsearchContainer();
-    _settingsBuilder = new SettingsBuilder(Collections.emptyList());
+    _settingsBuilder = new SettingsBuilder(Collections.emptyList(), null);
     checkContainerEngine(_elasticsearchContainer.getDockerClient());
     _elasticsearchContainer.start();
     _searchClient = ElasticTestUtils.buildRestClient(_elasticsearchContainer);

metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/SettingsBuilderFactory.java

@@ -2,28 +2,35 @@
 
 import com.google.common.collect.ImmutableList;
 import com.linkedin.gms.factory.entityregistry.EntityRegistryFactory;
+import com.linkedin.gms.factory.spring.YamlPropertySourceFactory;
 import com.linkedin.metadata.models.registry.EntityRegistry;
 import com.linkedin.metadata.search.elasticsearch.indexbuilder.SettingsBuilder;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.PropertySource;
 
 
 @Configuration
 @Import(EntityRegistryFactory.class)
+@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class)
 public class SettingsBuilderFactory {
   @Autowired
   @Qualifier("entityRegistry")
   private EntityRegistry entityRegistry;
 
+  @Value("${elasticsearch.index.mainTokenizer}")
+  private String mainTokenizer;
+
   @Bean("settingsBuilder")
   protected SettingsBuilder getInstance() {
     // Filter to process URNs
     ImmutableList.Builder<String> stopWords = ImmutableList.<String>builder().add("urn").add("li");
     // Add all entity names to stop word list
     entityRegistry.getEntitySpecs().values().forEach(entitySpec -> stopWords.add(entitySpec.getName().toLowerCase()));
-    return new SettingsBuilder(stopWords.build());
+    return new SettingsBuilder(stopWords.build(), mainTokenizer);
   }
 }

metadata-service/factories/src/main/resources/application.yml

@@ -109,6 +109,7 @@ elasticsearch:
     numReplicas: ${ELASTICSEARCH_NUM_REPLICAS_PER_INDEX:1}
     numRetries: ${ELASTICSEARCH_INDEX_BUILDER_NUM_RETRIES :3}
     maxArrayLength: ${SEARCH_DOCUMENT_MAX_ARRAY_LENGTH:1000}
+    mainTokenizer: ${ELASTICSEARCH_MAIN_TOKENIZER:#{null}}
 
 # TODO: Kafka topic convention
 kafka: