[Bug Fix] Fix the backward compatibility regression with COMPLEMENT for Regexp queries introduced in OpenSearch 3.0 (#18640)

* FIx ~ bug

Signed-off-by: Prudhvi Godithi <pgodithi@amazon.com>

* Update changelog

Signed-off-by: Prudhvi Godithi <pgodithi@amazon.com>

---------

Signed-off-by: Prudhvi Godithi <pgodithi@amazon.com>
(cherry picked from commit 1af55d2)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: github-actions[bot]

CHANGELOG.md

@@ -15,6 +15,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Removed
 
 ### Fixed
+- Add task cancellation checks in aggregators ([#18426](https://github.com/opensearch-project/OpenSearch/pull/18426))
+- Fix concurrent timings in profiler ([#18540](https://github.com/opensearch-project/OpenSearch/pull/18540))
+- Fix regex query from query string query to work with field alias ([#18215](https://github.com/opensearch-project/OpenSearch/issues/18215))
+- [Autotagging] Fix delete rule event consumption in InMemoryRuleProcessingService ([#18628](https://github.com/opensearch-project/OpenSearch/pull/18628))
+- Cannot communicate with HTTP/2 when reactor-netty is enabled ([#18599](https://github.com/opensearch-project/OpenSearch/pull/18599))
+- Fix the visit of sub queries for HasParentQuery and HasChildQuery ([#18621](https://github.com/opensearch-project/OpenSearch/pull/18621))
+- Fix the backward compatibility regression with COMPLEMENT for Regexp queries introduced in OpenSearch 3.0 ([#18640](https://github.com/opensearch-project/OpenSearch/pull/18640))
 
 ### Security
 

server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java

@@ -49,6 +49,8 @@
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.query.ConstantScoreQueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.index.query.RegexpFlag;
+import org.opensearch.index.query.RegexpQueryBuilder;
 import org.opensearch.index.query.TermQueryBuilder;
 import org.opensearch.search.rescore.QueryRescorerBuilder;
 import org.opensearch.search.sort.SortOrder;
@@ -59,7 +61,9 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
 
 import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
@@ -763,4 +767,19 @@ private void assertRescoreWindowFails(int windowSize) {
             )
         );
     }
+
+    public void testRegexQueryWithComplementFlag() {
+        createIndex("test_regex");
+        client().prepareIndex("test_regex").setId("1").setSource("text", "abc").get();
+        client().prepareIndex("test_regex").setId("2").setSource("text", "adc").get();
+        client().prepareIndex("test_regex").setId("3").setSource("text", "acc").get();
+        refresh();
+        RegexpQueryBuilder query = new RegexpQueryBuilder("text.keyword", "a~bc");
+        query.flags(RegexpFlag.COMPLEMENT);
+        SearchResponse response = client().prepareSearch("test_regex").setQuery(query).get();
+        assertEquals("COMPLEMENT should match 2 documents", 2L, response.getHits().getTotalHits().value());
+        Set<String> matchedIds = Arrays.stream(response.getHits().getHits()).map(hit -> hit.getId()).collect(Collectors.toSet());
+        assertEquals("Should match exactly 2 documents", 2, matchedIds.size());
+        assertTrue("Should match documents 2 and 3", matchedIds.containsAll(Arrays.asList("2", "3")));
+    }
 }

server/src/main/java/org/opensearch/index/query/RegexpQueryBuilder.java

@@ -38,6 +38,7 @@
 import org.apache.lucene.search.RegexpQuery;
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
+import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.lucene.BytesRefs;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.core.ParseField;
@@ -60,6 +61,9 @@
  * @opensearch.internal
  */
 public class RegexpQueryBuilder extends AbstractQueryBuilder<RegexpQueryBuilder> implements MultiTermQueryBuilder {
+
+    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RegexpQueryBuilder.class);
+
     public static final String NAME = "regexp";
 
     public static final int DEFAULT_FLAGS_VALUE = RegexpFlag.ALL.value();
@@ -294,12 +298,25 @@ protected Query doToQuery(QueryShardContext context) throws QueryShardException,
                     + "] index level setting."
             );
         }
+
+        // Check if COMPLEMENT flag is being used
+        // The COMPLEMENT flag maps to Lucene's DEPRECATED_COMPLEMENT which is marked for removal in Lucene 11
+        // This deprecation warning helps users migrate their queries before the feature is completely removed
+        if ((syntaxFlagsValue & RegexpFlag.COMPLEMENT.value()) != 0) {
+            deprecationLogger.deprecate(
+                "regexp_complement_operator",
+                "The complement operator (~) for arbitrary patterns in regexp queries is deprecated and will be removed in a future version. "
+                    + "Consider rewriting your query to use character class negation [^...] or other query types."
+            );
+        }
+
         MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewrite, null, LoggingDeprecationHandler.INSTANCE);
 
         int matchFlagsValue = caseInsensitive ? RegExp.ASCII_CASE_INSENSITIVE : 0;
         Query query = null;
         // For BWC we mask irrelevant bits (RegExp changed ALL from 0xffff to 0xff)
-        int sanitisedSyntaxFlag = syntaxFlagsValue & RegExp.ALL;
+        // The hexadecimal for DEPRECATED_COMPLEMENT is 0x10000. The OR condition ensures COMPLEMENT ~ is preserved
+        int sanitisedSyntaxFlag = syntaxFlagsValue & (RegExp.ALL | RegExp.DEPRECATED_COMPLEMENT);
 
         MappedFieldType fieldType = context.fieldMapper(fieldName);
         if (fieldType != null) {

server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java

@@ -55,7 +55,15 @@ protected RegexpQueryBuilder doCreateTestQueryBuilder() {
             List<RegexpFlag> flags = new ArrayList<>();
             int iter = randomInt(5);
             for (int i = 0; i < iter; i++) {
-                flags.add(randomFrom(RegexpFlag.values()));
+                // Exclude COMPLEMENT from random selection to avoid deprecation warnings
+                RegexpFlag[] availableFlags = {
+                    RegexpFlag.INTERSECTION,
+                    RegexpFlag.EMPTY,
+                    RegexpFlag.ANYSTRING,
+                    RegexpFlag.INTERVAL,
+                    RegexpFlag.NONE,
+                    RegexpFlag.ALL };
+                flags.add(randomFrom(availableFlags));
             }
             query.flags(flags.toArray(new RegexpFlag[0]));
         }
@@ -162,4 +170,32 @@ public void testParseFailsWithMultipleFields() throws IOException {
         e = expectThrows(ParsingException.class, () -> parseQuery(shortJson));
         assertEquals("[regexp] query doesn't support multiple fields, found [user1] and [user2]", e.getMessage());
     }
+
+    // Test that COMPLEMENT flag triggers deprecation warning
+    public void testComplementFlagDeprecation() throws IOException {
+        RegexpQueryBuilder query = new RegexpQueryBuilder("field", "a~bc");
+        query.flags(RegexpFlag.COMPLEMENT);
+        QueryShardContext context = createShardContext();
+        Query luceneQuery = query.toQuery(context);
+        assertNotNull(luceneQuery);
+        assertThat(luceneQuery, instanceOf(RegexpQuery.class));
+        assertWarnings(
+            "The complement operator (~) for arbitrary patterns in regexp queries is deprecated "
+                + "and will be removed in a future version. Consider rewriting your query to use character class negation [^...] or other query types."
+        );
+    }
+
+    // Separate test for COMPLEMENT flag Cacheability
+    public void testComplementFlagCacheability() throws IOException {
+        RegexpQueryBuilder queryBuilder = new RegexpQueryBuilder("field", "pattern");
+        queryBuilder.flags(RegexpFlag.COMPLEMENT);
+        QueryShardContext context = createShardContext();
+        QueryBuilder rewriteQuery = rewriteQuery(queryBuilder, new QueryShardContext(context));
+        assertNotNull(rewriteQuery.toQuery(context));
+        assertTrue("query should be cacheable: " + queryBuilder, context.isCacheable());
+        assertWarnings(
+            "The complement operator (~) for arbitrary patterns in regexp queries is deprecated "
+                + "and will be removed in a future version. Consider rewriting your query to use character class negation [^...] or other query types."
+        );
+    }
 }