Merge branch 'main' into aggregation-precomputation-rare-terms

Signed-off-by: Anthony Leong <aj.leong623@gmail.com>

Author: ajleong623

.ci/bwcVersions

@@ -46,6 +46,7 @@ BWC_VERSION:
   - "2.19.1"
   - "2.19.2"
   - "2.19.3"
+  - "2.19.4"
   - "3.0.0"
   - "3.1.0"
   - "3.2.0"

.gitattributes

@@ -10,5 +10,6 @@
 *.bcfks binary
 *.crt binary
 *.p12 binary
+*.ttf binary
 *.txt text=auto
 CHANGELOG.md merge=union

.github/CODEOWNERS

@@ -7,24 +7,24 @@
 # Default ownership for all repo files
 * @opensearch-project/opensearch-core-maintainers
 
-/modules/lang-painless/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/modules/parent-join/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/modules/lang-painless/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami
+/modules/parent-join/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami
 /modules/transport-netty4/ @opensearch-project/opensearch-core-maintainers @peternied
 
 /plugins/identity-shiro/ @opensearch-project/opensearch-core-maintainers @peternied @cwperks
 
-/server/src/internalClusterTest/java/org/opensearch/index/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/server/src/internalClusterTest/java/org/opensearch/search/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/internalClusterTest/java/org/opensearch/index/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami
+/server/src/internalClusterTest/java/org/opensearch/search/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami
 
 /server/src/main/java/org/opensearch/extensions/ @opensearch-project/opensearch-core-maintainers @peternied
 /server/src/main/java/org/opensearch/identity/ @opensearch-project/opensearch-core-maintainers @peternied @cwperks 
-/server/src/main/java/org/opensearch/index/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/server/src/main/java/org/opensearch/search/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/main/java/org/opensearch/index/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami
+/server/src/main/java/org/opensearch/search/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami
 /server/src/main/java/org/opensearch/threadpool/ @opensearch-project/opensearch-core-maintainers @jed326 @peternied
 /server/src/main/java/org/opensearch/transport/ @opensearch-project/opensearch-core-maintainers @peternied
 
-/server/src/test/java/org/opensearch/index/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
-/server/src/test/java/org/opensearch/search/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/test/java/org/opensearch/index/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami
+/server/src/test/java/org/opensearch/search/ @opensearch-project/opensearch-core-maintainers @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @cwperks @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami
 
 /.github/ @opensearch-project/opensearch-core-maintainers @jed326 @peternied
 

.github/workflows/delete_backport_branch.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: write
-    if: github.repository == 'opensearch-project/OpenSearch' && startsWith(github.event.pull_request.head.ref,'backport/')
+    if: github.repository == 'opensearch-project/OpenSearch' && (startsWith(github.event.pull_request.head.ref,'backport/') || startsWith(github.event.pull_request.head.ref,'release-chores/'))
     steps:
     - name: Delete merged branch
       uses: actions/github-script@v7

CHANGELOG.md

@@ -7,17 +7,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 - Expand fetch phase profiling to support inner hits and top hits aggregation phases  ([##18936](https://github.com/opensearch-project/OpenSearch/pull/18936))
 - Add temporal routing processors for time-based document routing ([#18920](https://github.com/opensearch-project/OpenSearch/issues/18920))
+- Implement Query Rewriting Infrastructure ([#19060](https://github.com/opensearch-project/OpenSearch/pull/19060))
 - The dynamic mapping parameter supports false_allow_templates ([#19065](https://github.com/opensearch-project/OpenSearch/pull/19065))
 - Add a toBuilder method in EngineConfig to support easy modification of configs([#19054](https://github.com/opensearch-project/OpenSearch/pull/19054))
 - Add StoreFactory plugin interface for custom Store implementations([#19091](https://github.com/opensearch-project/OpenSearch/pull/19091))
 - Use S3CrtClient for higher throughput while uploading files to S3 ([#18800](https://github.com/opensearch-project/OpenSearch/pull/18800))
 - Add a dynamic setting to change skip_cache_factor and min_frequency for querycache ([#18351](https://github.com/opensearch-project/OpenSearch/issues/18351))
+- Add overload constructor for Translog to accept Channel Factory as a parameter ([#18918](https://github.com/opensearch-project/OpenSearch/pull/18918))
+- Add subdirectory-aware store module with recovery support ([#19132](https://github.com/opensearch-project/OpenSearch/pull/19132))
 - Added precomputation for rare terms aggregation ([##18978](https://github.com/opensearch-project/OpenSearch/pull/18978))
 
 ### Changed
 - Add CompletionStage variants to methods in the Client Interface and default to ActionListener impl ([#18998](https://github.com/opensearch-project/OpenSearch/pull/18998))
 - IllegalArgumentException when scroll ID references a node not found in Cluster ([#19031](https://github.com/opensearch-project/OpenSearch/pull/19031))
 - Adding ScriptedAvg class to painless spi to allowlist usage from plugins ([#19006](https://github.com/opensearch-project/OpenSearch/pull/19006))
+- Replace centos:8 with almalinux:8 since centos docker images are deprecated ([#19154](https://github.com/opensearch-project/OpenSearch/pull/19154))
+- Add CompletionStage variants to IndicesAdminClient as an alternative to ActionListener ([#19161](https://github.com/opensearch-project/OpenSearch/pull/19161))
+- Remove cap on Java version used by forbidden APIs ([#19163](https://github.com/opensearch-project/OpenSearch/pull/19163))
 
 ### Fixed
 - Fix unnecessary refreshes on update preparation failures ([#15261](https://github.com/opensearch-project/OpenSearch/issues/15261))
@@ -27,7 +33,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix flaky tests in CloseIndexIT by addressing cluster state synchronization issues ([#18878](https://github.com/opensearch-project/OpenSearch/issues/18878))
 - [Tiered Caching] Handle  query execution exception ([#19000](https://github.com/opensearch-project/OpenSearch/issues/19000))
 - Grant access to testclusters dir for tests ([#19085](https://github.com/opensearch-project/OpenSearch/issues/19085))
+- Fix assertion error when collapsing search results with concurrent segment search enabled ([#19053](https://github.com/opensearch-project/OpenSearch/pull/19053))
 - Fix skip_unavailable setting changing to default during node drop issue ([#18766](https://github.com/opensearch-project/OpenSearch/pull/18766))
+- Fix pull-based ingestion pause state initialization during replica promotion ([#19212](https://github.com/opensearch-project/OpenSearch/pull/19212))
+- Fix QueryPhaseResultConsumer incomplete callback loops ([#19231](https://github.com/opensearch-project/OpenSearch/pull/19231))
 
 ### Dependencies
 - Bump `com.netflix.nebula.ospackage-base` from 12.0.0 to 12.1.0 ([#19019](https://github.com/opensearch-project/OpenSearch/pull/19019))
@@ -40,9 +49,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `1password/load-secrets-action` from 2 to 3 ([#19100](https://github.com/opensearch-project/OpenSearch/pull/19100))
 - Bump `com.nimbusds:nimbus-jose-jwt` from 10.3 to 10.4.2 ([#19099](https://github.com/opensearch-project/OpenSearch/pull/19099), [#19101](https://github.com/opensearch-project/OpenSearch/pull/19101))
 - Bump netty from 4.1.121.Final to 4.1.124.Final ([#19103](https://github.com/opensearch-project/OpenSearch/pull/19103))
-- Bump google cloud storage from 1.113.1 to 2.55.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547))
+- Bump Google Cloud Storage SDK from 1.113.1 to 2.55.0 ([#18922](https://github.com/opensearch-project/OpenSearch/pull/18922))
 - Bump `com.google.auth:google-auth-library-oauth2-http` from 1.37.1 to 1.38.0 ([#19144](https://github.com/opensearch-project/OpenSearch/pull/19144))
 - Bump `com.squareup.okio:okio` from 3.15.0 to 3.16.0 ([#19146](https://github.com/opensearch-project/OpenSearch/pull/19146))
+- Bump Slf4j from 1.7.36 to 2.0.17 ([#19136](https://github.com/opensearch-project/OpenSearch/pull/19136))
+- Bump `org.apache.tika` from 2.9.2 to 3.2.2 ([#19125](https://github.com/opensearch-project/OpenSearch/pull/19125))
+- Bump `org.apache.commons:commons-compress` from 1.26.1 to 1.28.0 ([#19125](https://github.com/opensearch-project/OpenSearch/pull/19125))
+- Bump `io.projectreactor.netty:reactor_netty` from `1.2.5` to `1.2.9` ([#19222](https://github.com/opensearch-project/OpenSearch/pull/19222))
+- Bump `org.bouncycastle:bouncycastle_jce` from `2.0.0` to `2.1.1` ([#19222](https://github.com/opensearch-project/OpenSearch/pull/19222))
+- Bump `org.bouncycastle:bouncycastle_tls` from `2.0.20` to `2.1.20` ([#19222](https://github.com/opensearch-project/OpenSearch/pull/19222))
+- Bump `org.bouncycastle:bouncycastle_pkix` from `2.0.8` to `2.1.9` ([#19222](https://github.com/opensearch-project/OpenSearch/pull/19222))
+- Bump `org.bouncycastle:bouncycastle_pg` from `2.0.11` to `2.1.11` ([#19222](https://github.com/opensearch-project/OpenSearch/pull/19222))
+- Bump `org.bouncycastle:bouncycastle_util` from `2.0.3` to `2.1.4` ([#19222](https://github.com/opensearch-project/OpenSearch/pull/19222))
+- Bump `com.azure:azure-core` from 1.55.5 to 1.56.0 ([#19206](https://github.com/opensearch-project/OpenSearch/pull/19206))
+- Bump `com.google.cloud:google-cloud-core` from 2.59.0 to 2.60.0 ([#19208](https://github.com/opensearch-project/OpenSearch/pull/19208))
+- Bump `org.jsoup:jsoup` from 1.20.1 to 1.21.2 ([#19207](https://github.com/opensearch-project/OpenSearch/pull/19207))
+- Bump `org.apache.hadoop:hadoop-minicluster` from 3.4.1 to 3.4.2 ([#19203](https://github.com/opensearch-project/OpenSearch/pull/19203))
+- Bump `com.maxmind.geoip2:geoip2` from 4.3.1 to 4.4.0 ([#19205](https://github.com/opensearch-project/OpenSearch/pull/19205))
 
 ### Deprecated
 

MAINTAINERS.md

@@ -11,6 +11,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Andriy Redko      | [reta](https://github.com/reta)                         | Independent |
 | Ankit Jain        | [jainankitk](https://github.com/jainankitk)             | Amazon      |
 | Ashish Singh      | [ashking94](https://github.com/ashking94)               | Amazon      |
+| Atri Sharma       | [atris](https://github.com/atris)                       | Apple       |
 | Bharathwaj G      | [bharath-techie](https://github.com/bharath-techie)     | Amazon      |
 | Bukhtawar Khan    | [Bukhtawar](https://github.com/Bukhtawar)               | Amazon      |
 | Charlotte Henkle  | [CEHENKLE](https://github.com/CEHENKLE)                 | Amazon      |
@@ -36,7 +37,6 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Shweta Thareja    | [shwetathareja](https://github.com/shwetathareja)       | Amazon      |
 | Sorabh Hamirwasia | [sohami](https://github.com/sohami)                     | Amazon      |
 | Yupeng Fu         | [yupeng9](https://github.com/yupeng9)                   | Uber        |
-| Vacha Shah        | [VachaShah](https://github.com/VachaShah)               | Amazon      |
 
 ## Emeritus
 
@@ -52,3 +52,4 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Tianli Feng              | [tlfeng](https://github.com/tlfeng)         | Amazon      |
 | Suraj Singh              | [dreamer-89](https://github.com/dreamer-89) | Amazon      |
 | Daniel "dB." Doubrovkine | [dblock](https://github.com/dblock)         | Independent |
+| Vacha Shah               | [VachaShah](https://github.com/VachaShah)   | Independent |

benchmarks/src/main/java/org/opensearch/benchmark/store/remote/filecache/FileCacheBenchmark.java

@@ -9,8 +9,6 @@
 package org.opensearch.benchmark.store.remote.filecache;
 
 import org.apache.lucene.store.IndexInput;
-import org.opensearch.core.common.breaker.CircuitBreaker;
-import org.opensearch.core.common.breaker.NoopCircuitBreaker;
 import org.opensearch.index.store.remote.filecache.CachedIndexInput;
 import org.opensearch.index.store.remote.filecache.FileCache;
 import org.opensearch.index.store.remote.filecache.FileCacheFactory;
@@ -93,8 +91,7 @@ public static class CacheParameters {
         public void setup() {
             fileCache = FileCacheFactory.createConcurrentLRUFileCache(
                 (long) maximumNumberOfEntries * INDEX_INPUT.length(),
-                concurrencyLevel,
-                new NoopCircuitBreaker(CircuitBreaker.REQUEST)
+                concurrencyLevel
             );
             for (long i = 0; i < maximumNumberOfEntries; i++) {
                 final Path key = Paths.get(Long.toString(i));

buildSrc/build.gradle

@@ -114,7 +114,7 @@ dependencies {
   api 'com.gradleup.shadow:shadow-gradle-plugin:8.3.5'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
-  api 'de.thetaphi:forbiddenapis:3.8'
+  api 'de.thetaphi:forbiddenapis:3.9'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.12'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.6'

buildSrc/src/main/java/org/opensearch/gradle/DockerBase.java

@@ -36,7 +36,7 @@
  * This class models the different Docker base images that are used to build Docker distributions of OpenSearch.
  */
 public enum DockerBase {
-    CENTOS("centos:8");
+    ALMALINUX("almalinux:8");
 
     private final String image;
 

buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenApisPrecommitPlugin.java

@@ -40,7 +40,6 @@
 import org.opensearch.gradle.ExportOpenSearchBuildResourcesTask;
 import org.opensearch.gradle.info.BuildParams;
 import org.opensearch.gradle.util.GradleUtils;
-import org.gradle.api.JavaVersion;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
@@ -53,6 +52,7 @@
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 public class ForbiddenApisPrecommitPlugin extends PrecommitPlugin {
     @Override
@@ -89,10 +89,6 @@ public TaskProvider<? extends Task> createTask(Project project) {
             t.setClasspath(project.files(sourceSet.getRuntimeClasspath()).plus(sourceSet.getCompileClasspath()));
 
             t.setTargetCompatibility(BuildParams.getRuntimeJavaVersion().getMajorVersion());
-            if (BuildParams.getRuntimeJavaVersion().compareTo(JavaVersion.VERSION_14) > 0) {
-                // TODO: forbidden apis does not yet support java 15, rethink using runtime version
-                t.setTargetCompatibility(JavaVersion.VERSION_14.getMajorVersion());
-            }
             t.setBundledSignatures(new HashSet<>(Arrays.asList("jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out")));
             t.setSignaturesFiles(
                 project.files(
@@ -140,6 +136,18 @@ public Void call(Object... names) {
                     return null;
                 }
             });
+            // Use of the deprecated security manager APIs are pervasive so set them to warn
+            // globally for all projects. Replacements for (most of) these APIs are available
+            // so usages can move to the non-deprecated variants to avoid the warnings.
+            t.setSignaturesWithSeverityWarn(
+                Set.of(
+                    "java.security.AccessController",
+                    "java.security.AccessControlContext",
+                    "java.lang.System#getSecurityManager()",
+                    "java.lang.SecurityManager",
+                    "java.security.Policy"
+                )
+            );
         });
         TaskProvider<Task> forbiddenApis = project.getTasks().named("forbiddenApis");
         forbiddenApis.configure(t -> t.setGroup(""));