Merge main into merge_feature_extensions_into_main_1

Signed-off-by: Ryan Bogan <rbogan@amazon.com>
opensearch-project · Dec 9, 2022 · c4643c3 · c4643c3
2 parents b93d41f + 67977a2
commit c4643c3
Show file tree

Hide file tree

Showing 40 changed files with 1,873 additions and 140 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,3 +1 @@
-# This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @opensearch-project/opensearch-core @reta
-
+*   @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @adnapibar @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @xuezhou25
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,7 +14,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Test] Add IAE test for deprecated edgeNGram analyzer name ([#5040](https://github.com/opensearch-project/OpenSearch/pull/5040))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
 - Add feature flag for extensions ([#5211](https://github.com/opensearch-project/OpenSearch/pull/5211))
-- Merge first batch of feature/extensions into main ([#5347](https://github.com/opensearch-project/OpenSearch/pull/5347))
+- Added jackson dependency to server ([#5366] (https://github.com/opensearch-project/OpenSearch/pull/5366))
+- Added experimental extensions to main ([#5347](https://github.com/opensearch-project/OpenSearch/pull/5347))
 
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0
@@ -80,6 +81,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fixed compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
 - Reject bulk requests with invalid actions ([#5299](https://github.com/opensearch-project/OpenSearch/issues/5299))
+- Support OpenSSL Provider with default Netty allocator ([#5460](https://github.com/opensearch-project/OpenSearch/pull/5460))
 
 ### Security
 
@@ -100,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Removed
 ### Fixed
 - Fix 1.x compatibility bug with stored Tasks ([#5412](https://github.com/opensearch-project/OpenSearch/pull/5412))
+- Fix case sensitivity for wildcard queries ([#5462](https://github.com/opensearch-project/OpenSearch/pull/5462))
 ### Security
 
 [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.4...HEAD

diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -5,7 +5,6 @@
 
 | Maintainer | GitHub ID | Affiliation |
 | --------------- | --------- | ----------- |
-| Abbas Hussain | [abbashus](https://github.com/abbashus) | Amazon |
 | Anas Alkouz | [anasalkouz](https://github.com/anasalkouz) | Amazon |
 | Andrew Ross   | [andrross](https://github.com/andrross)| Amazon |
 | Andriy Redko | [reta](https://github.com/reta) | Aiven |
@@ -22,8 +21,8 @@
 | Rishikesh Pasham | [Rishikesh1159](https://github.com/Rishikesh1159) | Amazon|
 | Ryan Bogan | [ryanbogan](https://github.com/ryanbogan) | Amazon |
 | Sarat Vemulapalli | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon |
-| Shweta Thareja |[shwetathareja](https://github.com/shwetathareja) | Amazon |
-| Suraj Singh |[dreamer-89](https://github.com/dreamer-89) | Amazon |
+| Shweta Thareja | [shwetathareja](https://github.com/shwetathareja) | Amazon |
+| Suraj Singh | [dreamer-89](https://github.com/dreamer-89) | Amazon |
 | Tianli Feng | [tlfeng](https://github.com/tlfeng) | Amazon |
 | Vacha Shah | [VachaShah](https://github.com/VachaShah) | Amazon |
 | Xue Zhou | [xuezhou25](https://github.com/xuezhou25) | Amazon |
@@ -32,6 +31,7 @@
 
 | Maintainer | GitHub ID | Affiliation |
 | --------------- | --------- | ----------- |
+| Abbas Hussain | [abbashus](https://github.com/abbashus) | Amazon |
 | Megha Sai Kavikondala | [meghasaik](https://github.com/meghasaik) | Amazon |
 
 [This document](https://github.com/opensearch-project/.github/blob/main/MAINTAINERS.md) explains what maintainers do in this repo, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md).
diff --git a/...transport-netty4/src/main/java/org/opensearch/transport/Netty4NioServerSocketChannel.java b/...transport-netty4/src/main/java/org/opensearch/transport/Netty4NioServerSocketChannel.java
@@ -0,0 +1,62 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport;
+
+import io.netty.channel.socket.InternetProtocolFamily;
+import io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.netty.util.internal.SocketUtils;
+import io.netty.util.internal.logging.InternalLogger;
+import io.netty.util.internal.logging.InternalLoggerFactory;
+
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.nio.channels.spi.SelectorProvider;
+import java.util.List;
+
+public class Netty4NioServerSocketChannel extends NioServerSocketChannel {
+    private static final InternalLogger logger = InternalLoggerFactory.getInstance(Netty4NioServerSocketChannel.class);
+
+    public Netty4NioServerSocketChannel() {
+        super();
+    }
+
+    public Netty4NioServerSocketChannel(SelectorProvider provider) {
+        super(provider);
+    }
+
+    public Netty4NioServerSocketChannel(SelectorProvider provider, InternetProtocolFamily family) {
+        super(provider, family);
+    }
+
+    public Netty4NioServerSocketChannel(ServerSocketChannel channel) {
+        super(channel);
+    }
+
+    @Override
+    protected int doReadMessages(List<Object> buf) throws Exception {
+        SocketChannel ch = SocketUtils.accept(javaChannel());
+
+        try {
+            if (ch != null) {
+                buf.add(new Netty4NioSocketChannel(this, ch));
+                return 1;
+            }
+        } catch (Throwable t) {
+            logger.warn("Failed to create a new channel from an accepted socket.", t);
+
+            try {
+                ch.close();
+            } catch (Throwable t2) {
+                logger.warn("Failed to close a socket.", t2);
+            }
+        }
+
+        return 0;
+    }
+}
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/NettyAllocator.java
@@ -39,7 +39,6 @@
 import io.netty.buffer.UnpooledByteBufAllocator;
 import io.netty.channel.Channel;
 import io.netty.channel.ServerChannel;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.common.Booleans;
@@ -181,7 +180,7 @@ public static Class<? extends ServerChannel> getServerChannelType() {
         if (ALLOCATOR instanceof NoDirectBuffers) {
             return CopyBytesServerSocketChannel.class;
         } else {
-            return NioServerSocketChannel.class;
+            return Netty4NioServerSocketChannel.class;
         }
     }
 

diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
@@ -170,9 +170,9 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath && !s3EKSBu
 processYamlRestTestResources {
   Map<String, Object> expansions = [
     'permanent_bucket': s3PermanentBucket,
-    'permanent_base_path': s3PermanentBasePath + "_integration_tests",
+    'permanent_base_path': s3PermanentBasePath + "_integration_tests_" + BuildParams.testSeed,
     'temporary_bucket': s3TemporaryBucket,
-    'temporary_base_path': s3TemporaryBasePath + "_integration_tests",
+    'temporary_base_path': s3TemporaryBasePath + "_integration_tests_" + BuildParams.testSeed,
     'ec2_bucket': s3EC2Bucket,
     'ec2_base_path': s3EC2BasePath,
     'ecs_bucket': s3ECSBucket,

diff --git a/release-notes/opensearch.release-notes-2.4.1.md b/release-notes/opensearch.release-notes-2.4.1.md
@@ -0,0 +1,22 @@
+## 2022-12-07 Version 2.4.1 Release Notes
+
+### Bug Fixes
+* Fix 1.x compatibility bug with stored Tasks ([#5412](https://github.com/opensearch-project/opensearch/pull/5412)) ([#5440](https://github.com/opensearch-project/opensearch/pull/5440))
+* Use BuildParams.isCi() instead of checking env var ([#5368](https://github.com/opensearch-project/opensearch/pull/5368)) ([#5373](https://github.com/opensearch-project/opensearch/pull/5373))
+* [BUG] org.opensearch.repositories.s3.RepositoryS3ClientYamlTestSuiteIT/test {yaml=repository_s3/20_repository_permanent_credentials/Snapshot and Restore with repository-s3 using permanent credentials} flaky ([#5325](https://github.com/opensearch-project/opensearch/pull/5325)) ([#5336](https://github.com/opensearch-project/opensearch/pull/5336))
+* [BUG] Gradle Check Failed on Windows due to JDK19 pulling by gradle ([#5188](https://github.com/opensearch-project/opensearch/pull/5188)) ([#5192](https://github.com/opensearch-project/opensearch/pull/5192))
+* Fix test to use a file from another temp directory ([#5158](https://github.com/opensearch-project/opensearch/pull/5158)) ([#5163](https://github.com/opensearch-project/opensearch/pull/5163))
+* Fix boundary condition in indexing pressure test ([#5168](https://github.com/opensearch-project/opensearch/pull/5168)) ([#5182](https://github.com/opensearch-project/opensearch/pull/5182))
+* [Backport 2.x] Fix: org.opensearch.clustermanager.ClusterManagerTaskThrottlingIT is flaky. ([#5153](https://github.com/opensearch-project/opensearch/pull/5153)) ([#5171](https://github.com/opensearch-project/opensearch/pull/5171))
+* [Backport 2.4] Raise error on malformed CSV ([#5141](https://github.com/opensearch-project/opensearch/pull/5141))
+
+### Features/Enhancements
+* Change the output error message back to use OpenSearchException in the cause chain ([#5081](https://github.com/opensearch-project/opensearch/pull/5081)) ([#5085](https://github.com/opensearch-project/opensearch/pull/5085))
+* Revert changes in AbstractPointGeometryFieldMapper ([#5250](https://github.com/opensearch-project/opensearch/pull/5250))
+* Add support for skipping changelog ([#5088](https://github.com/opensearch-project/opensearch/pull/5088)) ([#5160](https://github.com/opensearch-project/opensearch/pull/5160))
+* [Backport 2.4]Revert "Cluster manager task throttling feature [Final PR] ([#5071](https://github.com/opensearch-project/opensearch/pull/5071))  ([#5203](https://github.com/opensearch-project/opensearch/pull/5203))
+
+### Maintenance
+* Update Apache Lucene to 9.4.2 ([#5354](https://github.com/opensearch-project/opensearch/pull/5354)) ([#5361](https://github.com/opensearch-project/opensearch/pull/5361))
+* Update Jackson to 2.14.1 ([#5346](https://github.com/opensearch-project/opensearch/pull/5346)) ([#5358](https://github.com/opensearch-project/opensearch/pull/5358))
+* Bump nebula-publishing-plugin from v4.4.0 to v4.6.0. ([#5127](https://github.com/opensearch-project/opensearch/pull/5127)) ([#5131](https://github.com/opensearch-project/opensearch/pull/5131))
diff --git a/scripts/build.sh b/scripts/build.sh
@@ -139,7 +139,7 @@ esac
 
 echo "Building OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE"
 
-./gradlew :distribution:$TYPE:$TARGET:assemble :distribution:$TYPE:no-jdk-$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
+./gradlew :distribution:$TYPE:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
 
 # Copy artifact to dist folder in bundle build output
 [[ "$SNAPSHOT" == "true" ]] && IDENTIFIER="-SNAPSHOT"

diff --git a/server/build.gradle b/server/build.gradle
@@ -135,6 +135,7 @@ dependencies {
   // jna
   api "net.java.dev.jna:jna:${versions.jna}"
 
+  // jackson
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 

diff --git a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java
@@ -193,34 +193,29 @@ public void testDeleteIndexWhileIndexing() throws Exception {
         String index = "deleted_while_indexing";
         createIndex(index);
         AtomicBoolean stopped = new AtomicBoolean();
-        Thread[] threads = new Thread[between(1, 4)];
         AtomicInteger docID = new AtomicInteger();
-        for (int i = 0; i < threads.length; i++) {
-            threads[i] = new Thread(() -> {
-                while (stopped.get() == false && docID.get() < 5000) {
-                    String id = Integer.toString(docID.incrementAndGet());
-                    try {
-                        IndexResponse response = client().prepareIndex(index)
-                            .setId(id)
-                            .setSource(Collections.singletonMap("f" + randomIntBetween(1, 10), randomNonNegativeLong()), XContentType.JSON)
-                            .get();
-                        assertThat(response.getResult(), is(oneOf(CREATED, UPDATED)));
-                        logger.info("--> index id={} seq_no={}", response.getId(), response.getSeqNo());
-                    } catch (OpenSearchException ignore) {
-                        logger.info("--> fail to index id={}", id);
-                    }
+        Thread thread = new Thread(() -> {
+            while (stopped.get() == false && docID.get() < 5000) {
+                String id = Integer.toString(docID.incrementAndGet());
+                try {
+                    IndexResponse response = client().prepareIndex(index)
+                        .setId(id)
+                        .setSource(Collections.singletonMap("f" + randomIntBetween(1, 10), randomNonNegativeLong()), XContentType.JSON)
+                        .get();
+                    assertThat(response.getResult(), is(oneOf(CREATED, UPDATED)));
+                    logger.info("--> index id={} seq_no={}", response.getId(), response.getSeqNo());
+                } catch (OpenSearchException ignore) {
+                    logger.info("--> fail to index id={}", id);
                 }
-            });
-            threads[i].start();
-        }
+            }
+        });
+        thread.start();
         ensureGreen(index);
         assertBusy(() -> assertThat(docID.get(), greaterThanOrEqualTo(1)));
         assertAcked(client().admin().indices().prepareDelete(index));
         stopped.set(true);
-        for (Thread thread : threads) {
-            thread.join(ReplicationRequest.DEFAULT_TIMEOUT.millis() / 2);
-            assertFalse(thread.isAlive());
-        }
+        thread.join(ReplicationRequest.DEFAULT_TIMEOUT.millis() / 2);
+        assertFalse(thread.isAlive());
     }
 
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java
@@ -216,6 +216,39 @@ public void testKeywordWithWhitespace() throws Exception {
         assertHitCount(resp, 3L);
     }
 
+    public void testRegexCaseInsensitivity() throws Exception {
+        createIndex("messages");
+        List<IndexRequestBuilder> indexRequests = new ArrayList<>();
+        indexRequests.add(client().prepareIndex("messages").setId("1").setSource("message", "message: this is a TLS handshake"));
+        indexRequests.add(client().prepareIndex("messages").setId("2").setSource("message", "message: this is a tcp handshake"));
+        indexRandom(true, false, indexRequests);
+
+        SearchResponse response = client().prepareSearch("messages").setQuery(queryStringQuery("/TLS/").defaultField("message")).get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertHits(response.getHits(), "1");
+
+        response = client().prepareSearch("messages").setQuery(queryStringQuery("/tls/").defaultField("message")).get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertHits(response.getHits(), "1");
+
+        response = client().prepareSearch("messages").setQuery(queryStringQuery("/TCP/").defaultField("message")).get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertHits(response.getHits(), "2");
+
+        response = client().prepareSearch("messages").setQuery(queryStringQuery("/tcp/").defaultField("message")).get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertHits(response.getHits(), "2");
+
+        response = client().prepareSearch("messages").setQuery(queryStringQuery("/HANDSHAKE/").defaultField("message")).get();
+        assertNoFailures(response);
+        assertHitCount(response, 2);
+        assertHits(response.getHits(), "1", "2");
+    }
+
     public void testAllFields() throws Exception {
         String indexBody = copyToStringFromClasspath("/org/opensearch/search/query/all-query-index.json");
 

diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java
@@ -89,12 +89,15 @@
 import java.time.format.DateTimeFormatter;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Random;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.regex.Pattern;
 
 import static java.util.Collections.singletonMap;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
 import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -2089,8 +2092,14 @@ public void testWildcardQueryNormalizationOnTextField() {
         refresh();
 
         {
+            // test default case insensitivity: false
             WildcardQueryBuilder wildCardQuery = wildcardQuery("field1", "Bb*");
             SearchResponse searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
+            assertHitCount(searchResponse, 0L);
+
+            // test case insensitivity set to true
+            wildCardQuery = wildcardQuery("field1", "Bb*").caseInsensitive(true);
+            searchResponse = client().prepareSearch().setQuery(wildCardQuery).get();
             assertHitCount(searchResponse, 1L);
 
             wildCardQuery = wildcardQuery("field1", "bb*");
@@ -2099,6 +2108,24 @@ public void testWildcardQueryNormalizationOnTextField() {
         }
     }
 
+    /** tests wildcard case sensitivity */
+    public void testWildcardCaseSensitivity() {
+        assertAcked(prepareCreate("test").setMapping("field", "type=text"));
+        client().prepareIndex("test").setId("1").setSource("field", "lowercase text").get();
+        refresh();
+
+        // test case sensitive
+        SearchResponse response = client().prepareSearch("test").setQuery(wildcardQuery("field", "Text").caseInsensitive(false)).get();
+        assertNoFailures(response);
+        assertHitCount(response, 0);
+
+        // test case insensitive
+        response = client().prepareSearch("test").setQuery(wildcardQuery("field", "Text").caseInsensitive(true)).get();
+        assertNoFailures(response);
+        assertHitCount(response, 1);
+        assertHits(response.getHits(), "1");
+    }
+
     /**
      * Reserved characters should be excluded when the normalization is applied for keyword fields.
      * See https://github.com/elastic/elasticsearch/issues/46300 for details.
@@ -2175,4 +2202,16 @@ public void testIssueFuzzyInsideSpanMulti() {
         SearchResponse response = client().prepareSearch("test").setQuery(query).get();
         assertHitCount(response, 1);
     }
+
+    /**
+     * asserts the search response hits include the expected ids
+     */
+    private void assertHits(SearchHits hits, String... ids) {
+        assertThat(hits.getTotalHits().value, equalTo((long) ids.length));
+        Set<String> hitIds = new HashSet<>();
+        for (SearchHit hit : hits.getHits()) {
+            hitIds.add(hit.getId());
+        }
+        assertThat(hitIds, containsInAnyOrder(ids));
+    }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/ResizeRequest.java
@@ -95,7 +95,7 @@ public ResizeRequest(StreamInput in) throws IOException {
         sourceIndex = in.readString();
         type = in.readEnum(ResizeType.class);
         copySettings = in.readOptionalBoolean();
-        if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (in.getVersion().onOrAfter(Version.V_2_5_0)) {
             maxShardSize = in.readOptionalWriteable(ByteSizeValue::new);
         }
     }
@@ -140,7 +140,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeString(sourceIndex);
         out.writeEnum(type);
         out.writeOptionalBoolean(copySettings);
-        if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
+        if (out.getVersion().onOrAfter(Version.V_2_5_0)) {
             out.writeOptionalWriteable(maxShardSize);
         }
     }