Merge branch '2.x' into backport/backport-16052-to-2.x

.ci/bwcVersions

@@ -97,3 +97,4 @@ BWC_VERSION:
   - "2.16.1"
   - "2.17.0"
   - "2.17.1"
+  - "2.17.2"

.github/workflows/delete_backport_branch.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
-    if: startsWith(github.event.pull_request.head.ref,'backport/')
+    if: github.repository == 'opensearch-project/OpenSearch' && startsWith(github.event.pull_request.head.ref,'backport/')
     steps:
     - name: Delete merged branch
       uses: actions/github-script@v7

CHANGELOG.md

@@ -5,15 +5,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- [Workload Management] Add orchestrator for wlm resiliency (QueryGroupService) ([#15925](https://github.com/opensearch-project/OpenSearch/pull/15925))
 - [Offline Nodes] Adds offline-tasks library containing various interfaces to be used for Offline Background Tasks. ([#13574](https://github.com/opensearch-project/OpenSearch/pull/13574))
-- Add path prefix support to hashed prefix snapshots ([#15664](https://github.com/opensearch-project/OpenSearch/pull/15664))
 - Add support for async deletion in S3BlobContainer ([#15621](https://github.com/opensearch-project/OpenSearch/pull/15621))
 - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651))
 - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916))
 - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967))
-- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430))
 - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424))
+- [Tiered Caching] Segmented cache changes ([#16047](https://github.com/opensearch-project/OpenSearch/pull/16047))
 - Add support for msearch API to pass search pipeline name - ([#15923](https://github.com/opensearch-project/OpenSearch/pull/15923))
+- Add success and failure metrics for async shard fetch ([#15976](https://github.com/opensearch-project/OpenSearch/pull/15976))
+- [S3 Repository] Change default retry mechanism of s3 clients to Standard Mode ([#15978](https://github.com/opensearch-project/OpenSearch/pull/15978))
+- Add new metric REMOTE_STORE to NodeStats API response ([#15611](https://github.com/opensearch-project/OpenSearch/pull/15611))
+- New `phone` & `phone-search` analyzer + tokenizer ([#15915](https://github.com/opensearch-project/OpenSearch/pull/15915))
+- Add _list/indices API as paginated alternate to _cat/indices ([#14718](https://github.com/opensearch-project/OpenSearch/pull/14718))
 
 ### Dependencies
 - Bump `org.apache.logging.log4j:log4j-core` from 2.23.1 to 2.24.0 ([#15858](https://github.com/opensearch-project/OpenSearch/pull/15858))
@@ -28,9 +33,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.nimbusds:nimbus-jose-jwt` from 9.40 to 9.41.1 ([#16038](https://github.com/opensearch-project/OpenSearch/pull/16038))
 - Bump `actions/github-script` from 5 to 7 ([#16039](https://github.com/opensearch-project/OpenSearch/pull/16039))
 - Bump `dnsjava:dnsjava` from 3.6.1 to 3.6.2 ([#16041](https://github.com/opensearch-project/OpenSearch/pull/16041))
+- Bump `com.maxmind.geoip2:geoip2` from 4.2.0 to 4.2.1 ([#16042](https://github.com/opensearch-project/OpenSearch/pull/16042))
+- Bump `com.maxmind.db:maxmind-db` from 3.1.0 to 3.1.1 ([#16137](https://github.com/opensearch-project/OpenSearch/pull/16137))
+- Bump Apache lucene from 9.11.1 to 9.12.0 ([#15333](https://github.com/opensearch-project/OpenSearch/pull/15333))
+- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133))
+- Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182))
+- Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217))
+- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135))
 
 ### Changed
 - Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049))
+- Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430))
+- Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024))
+- Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154))
 
 
 ### Deprecated
@@ -39,10 +54,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Fixed
 - Fix wildcard query containing escaped character ([#15737](https://github.com/opensearch-project/OpenSearch/pull/15737))
+- Fix case-insensitive query on wildcard field ([#15882](https://github.com/opensearch-project/OpenSearch/pull/15882))
 - Add validation for the search backpressure cancellation settings ([#15501](https://github.com/opensearch-project/OpenSearch/pull/15501))
 - Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988))
 - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985))
 - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931))
+- Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521))
+- Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158))
+- Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194))
 
 ### Security
 

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 2.18.0
-lucene            = 9.11.1
+lucene            = 9.12.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.4+7
@@ -29,7 +29,7 @@ hdrhistogram      = 2.2.2
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.112.Final
+netty             = 4.1.114.Final
 joda              = 2.12.7
 
 # project reactor

client/rest-high-level/src/test/java/org/opensearch/client/OpenSearchRestHighLevelClientTestCase.java

@@ -88,6 +88,7 @@
 public abstract class OpenSearchRestHighLevelClientTestCase extends OpenSearchRestTestCase {
 
     protected static final String CONFLICT_PIPELINE_ID = "conflict_pipeline";
+    protected static final double DOUBLE_DELTA = 0.000001;
 
     private static RestHighLevelClient restHighLevelClient;
     private static boolean async = Booleans.parseBoolean(System.getProperty("tests.rest.async", "false"));

client/rest-high-level/src/test/java/org/opensearch/client/RankEvalIT.java

@@ -158,7 +158,7 @@ public void testMetrics() throws IOException {
 
             RankEvalRequest rankEvalRequest = new RankEvalRequest(spec, new String[] { "index", "index2" });
             RankEvalResponse response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
-            assertEquals(expectedScores[i], response.getMetricScore(), Double.MIN_VALUE);
+            assertEquals(expectedScores[i], response.getMetricScore(), DOUBLE_DELTA);
             i++;
         }
     }

gradle.properties

@@ -31,9 +31,7 @@ systemProp.org.gradle.dependency.duplicate.project.detection=false
 # Enforce the build to fail on deprecated gradle api usage
 systemProp.org.gradle.warning.mode=fail
 
-# forcing to use TLS1.2 to avoid failure in vault
-# see https://github.com/hashicorp/vault/issues/8750#issuecomment-631236121
-systemProp.jdk.tls.client.protocols=TLSv1.2
+systemProp.jdk.tls.client.protocols=TLSv1.2,TLSv1.3
 
 # jvm args for faster test execution by default
 systemProp.tests.jvm.argline=-XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=64m

gradle/missing-javadoc.gradle

@@ -139,6 +139,7 @@ configure([
   project(":plugins:analysis-icu"),
   project(":plugins:analysis-kuromoji"),
   project(":plugins:analysis-nori"),
+  project(":plugins:analysis-phonenumber"),
   project(":plugins:analysis-phonetic"),
   project(":plugins:analysis-smartcn"),
   project(":plugins:analysis-stempel"),

gradle/wrapper/gradle-wrapper.properties

@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=fdfca5dbc2834f0ece5020465737538e5ba679deeff5ab6c09621d67f8bb1a15
+distributionSha256Sum=2ab88d6de2c23e6adae7363ae6e29cbdd2a709e992929b48b6530fd0c7133bd6

libs/core/licenses/lucene-core-9.12.0.jar.sha1

@@ -0,0 +1 @@
+fdb055d569bb20bfce9618fe2b01c29bab7f290c

libs/core/src/main/java/org/opensearch/Version.java

@@ -140,7 +140,8 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_16_1 = new Version(2160199, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1);
     public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1);
-    public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1);
+    public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1);
+    public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_12_0);
     public static final Version CURRENT = V_2_18_0;
 
     public static Version fromId(int id) {

libs/core/src/main/java/org/opensearch/core/common/Strings.java

@@ -815,4 +815,17 @@ public static String toLowercaseAscii(String in) {
         }
         return out.toString();
     }
+
+    /**
+     * Check whether every single character in the string is a digit.
+     *
+     * <p>An empty string returns {@code false}.</p>
+     *
+     * @param s the string, must not be null.
+     * @return {@code true} if the string only contains digits, {@code false} otherwise.
+     */
+    public static boolean isDigits(final String s) {
+        return !s.isEmpty() && s.chars().allMatch(Character::isDigit);
+    }
+
 }

libs/core/src/test/java/org/opensearch/core/common/StringsTests.java

@@ -114,4 +114,15 @@ public void testToStringToXContentWithOrWithoutParams() {
             containsString("\"color_from_param\":\"blue\"")
         );
     }
+
+    public void testIsDigits() {
+        assertTrue(Strings.isDigits("1"));
+        assertTrue(Strings.isDigits("123"));
+        assertFalse(Strings.isDigits(""));
+        assertFalse(Strings.isDigits("abc"));
+        assertFalse(Strings.isDigits("123a"));
+        assertFalse(Strings.isDigits("0x123"));
+        assertFalse(Strings.isDigits("123.4"));
+        assertFalse(Strings.isDigits("123f"));
+    }
 }

modules/cache-common/src/internalClusterTest/java/org/opensearch/cache/common/tier/TieredSpilloverCacheBaseIT.java

@@ -0,0 +1,57 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cache.common.tier;
+
+import org.opensearch.common.cache.CacheType;
+import org.opensearch.common.cache.settings.CacheSettings;
+import org.opensearch.common.cache.store.OpenSearchOnHeapCache;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+public class TieredSpilloverCacheBaseIT extends OpenSearchIntegTestCase {
+
+    public Settings defaultSettings(String onHeapCacheSizeInBytesOrPercentage, int numberOfSegments) {
+        return Settings.builder()
+            .put(FeatureFlags.PLUGGABLE_CACHE, "true")
+            .put(
+                CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(),
+                TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME
+            )
+            .put(
+                TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_NAME.getConcreteSettingForNamespace(
+                    CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()
+                ).getKey(),
+                OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME
+            )
+            .put(
+                TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_STORE_NAME.getConcreteSettingForNamespace(
+                    CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()
+                ).getKey(),
+                MockDiskCache.MockDiskCacheFactory.NAME
+            )
+            .put(
+                TieredSpilloverCacheSettings.TIERED_SPILLOVER_SEGMENTS.getConcreteSettingForNamespace(
+                    CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()
+                ).getKey(),
+                numberOfSegments
+            )
+            .put(
+                TieredSpilloverCacheSettings.TIERED_SPILLOVER_ONHEAP_STORE_SIZE.getConcreteSettingForNamespace(
+                    CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()
+                ).getKey(),
+                onHeapCacheSizeInBytesOrPercentage
+            )
+            .build();
+    }
+
+    public int getNumberOfSegments() {
+        return randomFrom(1, 2, 4, 8, 16, 64, 128, 256);
+    }
+}