[Backport 2.17] Add new cluster setting for keyword indexordocvalues query (#15637)

CHANGELOG.md

@@ -53,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508))
 - Relax the join validation for Remote State publication ([#15471](https://github.com/opensearch-project/OpenSearch/pull/15471))
 - Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131))
+- MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637))
 
 ### Dependencies
 - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))

server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

@@ -551,6 +551,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 SearchService.MAX_AGGREGATION_REWRITE_FILTERS,
                 SearchService.INDICES_MAX_CLAUSE_COUNT_SETTING,
                 SearchService.CARDINALITY_AGGREGATION_PRUNING_THRESHOLD,
+                SearchService.KEYWORD_INDEX_OR_DOC_VALUES_ENABLED,
                 CreatePitController.PIT_INIT_KEEP_ALIVE,
                 Node.WRITE_PORTS_FILE_SETTING,
                 Node.NODE_NAME_SETTING,

server/src/main/java/org/opensearch/index/IndexService.java

@@ -959,7 +959,8 @@ public QueryShardContext newQueryShardContext(
         IndexSearcher searcher,
         LongSupplier nowInMillis,
         String clusterAlias,
-        boolean validate
+        boolean validate,
+        boolean keywordIndexOrDocValuesEnabled
     ) {
         final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(
             index().getName(),
@@ -985,10 +986,27 @@ public QueryShardContext newQueryShardContext(
             indexNameMatcher,
             allowExpensiveQueries,
             valuesSourceRegistry,
-            validate
+            validate,
+            keywordIndexOrDocValuesEnabled
         );
     }
 
+    /**
+     * Creates a new QueryShardContext.
+     * <p>
+     * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make
+     * {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
+     */
+    public QueryShardContext newQueryShardContext(
+        int shardId,
+        IndexSearcher searcher,
+        LongSupplier nowInMillis,
+        String clusterAlias,
+        boolean validate
+    ) {
+        return newQueryShardContext(shardId, searcher, nowInMillis, clusterAlias, validate, false);
+    }
+
     /**
      * The {@link ThreadPool} to use for this index.
      */

server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java

@@ -392,6 +392,9 @@
             failIfNotIndexedAndNoDocValues();
             // has index and doc_values enabled
             if (isSearchable() && hasDocValues()) {
+                if (!context.keywordFieldIndexOrDocValuesEnabled()) {
+                    return super.termsQuery(values, context);
+                }
                 BytesRef[] bytesRefs = new BytesRef[values.size()];
                 for (int i = 0; i < bytesRefs.length; i++) {
                     bytesRefs[i] = indexedValueForSearch(values.get(i));
@@ -429,6 +432,9 @@
             }
             failIfNotIndexedAndNoDocValues();
             if (isSearchable() && hasDocValues()) {
+                if (!context.keywordFieldIndexOrDocValuesEnabled()) {
+                    return super.prefixQuery(value, method, caseInsensitive, context);
+                }
                 Query indexQuery = super.prefixQuery(value, method, caseInsensitive, context);
                 Query dvQuery = super.prefixQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, context);
                 return new IndexOrDocValuesQuery(indexQuery, dvQuery);
@@ -461,6 +467,9 @@
             }
             failIfNotIndexedAndNoDocValues();
             if (isSearchable() && hasDocValues()) {
+                if (!context.keywordFieldIndexOrDocValuesEnabled()) {
+                    return super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context);
+                }
                 Query indexQuery = super.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context);
                 Query dvQuery = super.regexpQuery(
                     value,
@@ -549,6 +558,9 @@
                 );
             }
             if (isSearchable() && hasDocValues()) {
+                if (!context.keywordFieldIndexOrDocValuesEnabled()) {
+                    return super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context);
+                }
                 Query indexQuery = super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context);
                 Query dvQuery = super.fuzzyQuery(
                     value,
@@ -591,6 +603,9 @@
             // wildcard
             // query text
             if (isSearchable() && hasDocValues()) {
+                if (!context.keywordFieldIndexOrDocValuesEnabled()) {
+                    return super.wildcardQuery(value, method, caseInsensitive, true, context);
+                }
                 Query indexQuery = super.wildcardQuery(value, method, caseInsensitive, true, context);
                 Query dvQuery = super.wildcardQuery(value, MultiTermQuery.DOC_VALUES_REWRITE, caseInsensitive, true, context);
                 return new IndexOrDocValuesQuery(indexQuery, dvQuery);

server/src/main/java/org/opensearch/index/query/QueryShardContext.java

@@ -126,6 +126,7 @@ public class QueryShardContext extends QueryRewriteContext {
     private final ValuesSourceRegistry valuesSourceRegistry;
     private BitSetProducer parentFilter;
     private DerivedFieldResolver derivedFieldResolver;
+    private boolean keywordIndexOrDocValuesEnabled;
 
     public QueryShardContext(
         int shardId,
@@ -209,7 +210,55 @@ public QueryShardContext(
             ),
             allowExpensiveQueries,
             valuesSourceRegistry,
-            validate
+            validate,
+            false
+        );
+    }
+
+    public QueryShardContext(
+        int shardId,
+        IndexSettings indexSettings,
+        BigArrays bigArrays,
+        BitsetFilterCache bitsetFilterCache,
+        TriFunction<MappedFieldType, String, Supplier<SearchLookup>, IndexFieldData<?>> indexFieldDataLookup,
+        MapperService mapperService,
+        SimilarityService similarityService,
+        ScriptService scriptService,
+        NamedXContentRegistry xContentRegistry,
+        NamedWriteableRegistry namedWriteableRegistry,
+        Client client,
+        IndexSearcher searcher,
+        LongSupplier nowInMillis,
+        String clusterAlias,
+        Predicate<String> indexNameMatcher,
+        BooleanSupplier allowExpensiveQueries,
+        ValuesSourceRegistry valuesSourceRegistry,
+        boolean validate,
+        boolean keywordIndexOrDocValuesEnabled
+    ) {
+        this(
+            shardId,
+            indexSettings,
+            bigArrays,
+            bitsetFilterCache,
+            indexFieldDataLookup,
+            mapperService,
+            similarityService,
+            scriptService,
+            xContentRegistry,
+            namedWriteableRegistry,
+            client,
+            searcher,
+            nowInMillis,
+            indexNameMatcher,
+            new Index(
+                RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()),
+                indexSettings.getIndex().getUUID()
+            ),
+            allowExpensiveQueries,
+            valuesSourceRegistry,
+            validate,
+            keywordIndexOrDocValuesEnabled
         );
     }
 
@@ -232,7 +281,8 @@ public QueryShardContext(QueryShardContext source) {
             source.fullyQualifiedIndex,
             source.allowExpensiveQueries,
             source.valuesSourceRegistry,
-            source.validate()
+            source.validate(),
+            source.keywordIndexOrDocValuesEnabled
         );
     }
 
@@ -254,7 +304,8 @@ private QueryShardContext(
         Index fullyQualifiedIndex,
         BooleanSupplier allowExpensiveQueries,
         ValuesSourceRegistry valuesSourceRegistry,
-        boolean validate
+        boolean validate,
+        boolean keywordIndexOrDocValuesEnabled
     ) {
         super(xContentRegistry, namedWriteableRegistry, client, nowInMillis, validate);
         this.shardId = shardId;
@@ -278,6 +329,7 @@ private QueryShardContext(
             emptyList(),
             indexSettings.isDerivedFieldAllowed()
         );
+        this.keywordIndexOrDocValuesEnabled = keywordIndexOrDocValuesEnabled;
     }
 
     private void reset() {
@@ -425,6 +477,14 @@ public MappedFieldType getDerivedFieldType(String fieldName) {
         throw new UnsupportedOperationException("Use resolveDerivedFieldType() instead.");
     }
 
+    public boolean keywordFieldIndexOrDocValuesEnabled() {
+        return keywordIndexOrDocValuesEnabled;
+    }
+
+    public void setKeywordFieldIndexOrDocValuesEnabled(boolean keywordIndexOrDocValuesEnabled) {
+        this.keywordIndexOrDocValuesEnabled = keywordIndexOrDocValuesEnabled;
+    }
+
     public void setAllowUnmappedFields(boolean allowUnmappedFields) {
         this.allowUnmappedFields = allowUnmappedFields;
     }

server/src/main/java/org/opensearch/search/DefaultSearchContext.java

@@ -121,6 +121,7 @@
 import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL;
 import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_AUTO;
 import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_NONE;
+import static org.opensearch.search.SearchService.KEYWORD_INDEX_OR_DOC_VALUES_ENABLED;
 import static org.opensearch.search.SearchService.MAX_AGGREGATION_REWRITE_FILTERS;
 
 /**
@@ -207,6 +208,7 @@ final class DefaultSearchContext extends SearchContext {
     private final SetOnce<Boolean> requestShouldUseConcurrentSearch = new SetOnce<>();
     private final int maxAggRewriteFilters;
     private final int cardinalityAggregationPruningThreshold;
+    private final boolean keywordIndexOrDocValuesEnabled;
 
     DefaultSearchContext(
         ReaderContext readerContext,
@@ -257,14 +259,16 @@ final class DefaultSearchContext extends SearchContext {
             this.searcher,
             request::nowInMillis,
             shardTarget.getClusterAlias(),
-            validate
+            validate,
+            evaluateKeywordIndexOrDocValuesEnabled()
         );
         queryBoost = request.indexBoost();
         this.lowLevelCancellation = lowLevelCancellation;
         this.requestToAggReduceContextBuilder = requestToAggReduceContextBuilder;
 
         this.maxAggRewriteFilters = evaluateFilterRewriteSetting();
         this.cardinalityAggregationPruningThreshold = evaluateCardinalityAggregationPruningThreshold();
+        this.keywordIndexOrDocValuesEnabled = evaluateKeywordIndexOrDocValuesEnabled();
         this.concurrentSearchDeciderFactories = concurrentSearchDeciderFactories;
     }
 
@@ -1125,10 +1129,22 @@ public int cardinalityAggregationPruningThreshold() {
         return cardinalityAggregationPruningThreshold;
     }
 
+    @Override
+    public boolean keywordIndexOrDocValuesEnabled() {
+        return keywordIndexOrDocValuesEnabled;
+    }
+
     private int evaluateCardinalityAggregationPruningThreshold() {
         if (clusterService != null) {
             return clusterService.getClusterSettings().get(CARDINALITY_AGGREGATION_PRUNING_THRESHOLD);
         }
         return 0;
     }
+
+    public boolean evaluateKeywordIndexOrDocValuesEnabled() {
+        if (clusterService != null) {
+            return clusterService.getClusterSettings().get(KEYWORD_INDEX_OR_DOC_VALUES_ENABLED);
+        }
+        return false;
+    }
 }

server/src/main/java/org/opensearch/search/SearchService.java

@@ -338,6 +338,13 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         Property.NodeScope
     );
 
+    public static final Setting<Boolean> KEYWORD_INDEX_OR_DOC_VALUES_ENABLED = Setting.boolSetting(
+        "search.keyword_index_or_doc_values_enabled",
+        false,
+        Property.Dynamic,
+        Property.NodeScope
+    );
+
     public static final int DEFAULT_SIZE = 10;
     public static final int DEFAULT_FROM = 0;
 
@@ -1174,6 +1181,7 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
                 context.getIndexSettings().isDerivedFieldAllowed() && allowDerivedField
             );
             context.setDerivedFieldResolver(derivedFieldResolver);
+            context.setKeywordFieldIndexOrDocValuesEnabled(searchContext.keywordIndexOrDocValuesEnabled());
             searchContext.getQueryShardContext().setDerivedFieldResolver(derivedFieldResolver);
             Rewriteable.rewrite(request.getRewriteable(), context, true);
             assert searchContext.getQueryShardContext().isCacheable();

server/src/main/java/org/opensearch/search/internal/SearchContext.java

@@ -530,4 +530,9 @@
     public int cardinalityAggregationPruningThreshold() {
         return 0;
     }
+
+    public boolean keywordIndexOrDocValuesEnabled() {
+        return false;
+    }
+
 }

server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java

@@ -136,7 +136,7 @@ public void testTermsQuery() {
             new TermInSetQuery("field", terms),
             new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, "field", terms)
         );
-        assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), null));
+        assertEquals(expected, ft.termsQuery(Arrays.asList("foo", "bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES));
 
         MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap());
         Query expectedIndex = new TermInSetQuery("field", terms);
@@ -225,7 +225,7 @@ public void testRegexpQuery() {
                 new RegexpQuery(new Term("field", "foo.*")),
                 new RegexpQuery(new Term("field", "foo.*"), 0, 0, RegexpQuery.DEFAULT_PROVIDER, 10, MultiTermQuery.DOC_VALUES_REWRITE)
             ),
-            ft.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC)
+            ft.regexpQuery("foo.*", 0, 0, 10, MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)
         );
 
         Query indexExpected = new RegexpQuery(new Term("field", "foo.*"));
@@ -267,7 +267,7 @@ public void testFuzzyQuery() {
                 new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true),
                 new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true, MultiTermQuery.DOC_VALUES_REWRITE)
             ),
-            ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC)
+            ft.fuzzyQuery("foo", Fuzziness.fromEdits(2), 1, 50, true, null, MOCK_QSC_ENABLE_INDEX_DOC_VALUES)
         );
 
         Query indexExpected = new FuzzyQuery(new Term("field", "foo"), 2, 1, 50, true);
@@ -308,7 +308,7 @@ public void testWildCardQuery() {
                 MultiTermQuery.DOC_VALUES_REWRITE
             )
         );
-        assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC));
+        assertEquals(expected, ft.wildcardQuery("foo*", MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE, MOCK_QSC_ENABLE_INDEX_DOC_VALUES));
 
         Query indexExpected = new WildcardQuery(new Term("field", new BytesRef("foo*")));
         MappedFieldType onlyIndexed = new KeywordFieldType("field", true, false, Collections.emptyMap());

server/src/test/java/org/opensearch/index/search/NestedHelperTests.java

@@ -57,6 +57,8 @@
 import java.io.IOException;
 import java.util.Collections;
 
+import static org.opensearch.index.mapper.FieldTypeTestCase.MOCK_QSC_ENABLE_INDEX_DOC_VALUES;
+
 public class NestedHelperTests extends OpenSearchSingleNodeTestCase {
 
     IndexService indexService;
@@ -132,28 +134,28 @@ public void testMatchNo() {
     }
 
     public void testTermsQuery() {
-        Query termsQuery = mapperService.fieldType("foo").termsQuery(Collections.singletonList("bar"), null);
+        Query termsQuery = mapperService.fieldType("foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES);
         assertFalse(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1"));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2"));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3"));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing"));
 
-        termsQuery = mapperService.fieldType("nested1.foo").termsQuery(Collections.singletonList("bar"), null);
+        termsQuery = mapperService.fieldType("nested1.foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES);
         assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery));
         assertFalse(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1"));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2"));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3"));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing"));
 
-        termsQuery = mapperService.fieldType("nested2.foo").termsQuery(Collections.singletonList("bar"), null);
+        termsQuery = mapperService.fieldType("nested2.foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES);
         assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1"));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2"));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested3"));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested_missing"));
 
-        termsQuery = mapperService.fieldType("nested3.foo").termsQuery(Collections.singletonList("bar"), null);
+        termsQuery = mapperService.fieldType("nested3.foo").termsQuery(Collections.singletonList("bar"), MOCK_QSC_ENABLE_INDEX_DOC_VALUES);
         assertTrue(new NestedHelper(mapperService).mightMatchNestedDocs(termsQuery));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested1"));
         assertTrue(new NestedHelper(mapperService).mightMatchNonNestedDocs(termsQuery, "nested2"));