Merge branch 'main' into must-not-numeric-terms-rewrite

Signed-off-by: Peter Alfonsi <petealft@amazon.com>

Author: Peter Alfonsi

.github/dependabot.yml

@@ -445,14 +445,6 @@ updates:
     labels:
       - "dependabot"
       - "dependencies"
-  - directory: /distribution/tools/upgrade-cli/
-    open-pull-requests-limit: 1
-    package-ecosystem: gradle
-    schedule:
-      interval: weekly
-    labels:
-      - "dependabot"
-      - "dependencies"
   - directory: /doc-tools/
     open-pull-requests-limit: 1
     package-ecosystem: gradle

.github/workflows/publish-maven-snapshots.yml

@@ -25,16 +25,16 @@ jobs:
           distribution: temurin
           java-version: 21
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
+      - name: Load secret
+        uses: 1password/load-secrets-action@v2
         with:
-          role-to-assume: ${{ secrets.PUBLISH_SNAPSHOTS_ROLE }}
-          aws-region: us-east-1
+          # Export loaded secrets as environment variables
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          SONATYPE_USERNAME: op://opensearch-infra-secrets/maven-central-portal-credentials/username
+          SONATYPE_PASSWORD: op://opensearch-infra-secrets/maven-central-portal-credentials/password
 
       - name: Publish snapshots to maven
         run: |
-          export SONATYPE_USERNAME=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-username --query SecretString --output text)
-          export SONATYPE_PASSWORD=$(aws secretsmanager get-secret-value --secret-id maven-snapshots-password --query SecretString --output text)
-          echo "::add-mask::$SONATYPE_USERNAME"
-          echo "::add-mask::$SONATYPE_PASSWORD"
           ./gradlew publishNebulaPublicationToSnapshotsRepository

.github/workflows/trigger-manifest-generation.yml

@@ -0,0 +1,17 @@
+name: Trigger manifest generation workflow
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - buildSrc/version.properties
+
+jobs:
+  trigger-manifest-workflow:
+    if: github.repository == 'opensearch-project/OpenSearch'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger manifest-update workflow
+        run: |
+            echo "Triggering manifest-update workflow at https://build.ci.opensearch.org/job/manifest-update/"
+            curl -f -X POST https://build.ci.opensearch.org/job/manifest-update/build --user ${{ secrets.JENKINS_GITHUB_USER}}:${{ secrets.JENKINS_GITHUB_USER_TOKEN}}

.github/workflows/version.yml

@@ -29,7 +29,7 @@ jobs:
           fi
           CURRENT_VERSION_ARRAY=($(echo "$TAG" | tr . '\n'))
           BASE=$(IFS=. ; echo "${CURRENT_VERSION_ARRAY[*]:0:2}")
-          BASE_X=$(IFS=. ; echo "${CURRENT_VERSION_ARRAY[*]:0:1}.x")
+          MAIN_BRANCH="main"
           CURRENT_VERSION=$(IFS=. ; echo "${CURRENT_VERSION_ARRAY[*]:0:3}")
           CURRENT_VERSION_UNDERSCORE=$(IFS=_ ; echo "V_${CURRENT_VERSION_ARRAY[*]:0:3}")
           CURRENT_VERSION_ARRAY[2]=$((CURRENT_VERSION_ARRAY[2]+1))
@@ -42,7 +42,7 @@ jobs:
           fi
           echo "TAG=$TAG" >> $GITHUB_ENV
           echo "BASE=$BASE" >> $GITHUB_ENV
-          echo "BASE_X=$BASE_X" >> $GITHUB_ENV
+          echo "MAIN_BRANCH=$MAIN_BRANCH" >> $GITHUB_ENV
           echo "CURRENT_VERSION=$CURRENT_VERSION" >> $GITHUB_ENV
           echo "CURRENT_VERSION_UNDERSCORE=$CURRENT_VERSION_UNDERSCORE" >> $GITHUB_ENV
           echo "NEXT_VERSION=$NEXT_VERSION" >> $GITHUB_ENV
@@ -77,33 +77,7 @@ jobs:
 
       - uses: actions/checkout@v4
         with:
-          ref: ${{ env.BASE_X }}
-
-      - name: Add Patch Version on Major.X branch
-        uses: peternied/opensearch-core-version-updater@v1
-        with:
-          previous-version: ${{ env.CURRENT_VERSION }}
-          new-version: ${{ env.NEXT_VERSION }}
-          update-current: false
-
-      - name: Create PR for BASE_X
-        id: base_x_pr
-        uses: peter-evans/create-pull-request@v7
-        with:
-          base: ${{ env.BASE_X }}
-          branch: 'create-pull-request/patch-${{ env.BASE_X }}'
-          commit-message: Add bwc version ${{ env.NEXT_VERSION }}
-          signoff: true
-          delete-branch: true
-          labels: |
-            autocut
-          title: '[AUTO] [${{ env.BASE_X }}] Add bwc version ${{ env.NEXT_VERSION }}.'
-          body: |
-            I've noticed that a new tag ${{ env.TAG }} was pushed, and added a bwc version ${{ env.NEXT_VERSION }}.
-
-      - uses: actions/checkout@v4
-        with:
-          ref: main
+          ref: ${{ env.MAIN_BRANCH }}
 
       - name: Add Patch Version on main branch
         uses: peternied/opensearch-core-version-updater@v1
@@ -112,18 +86,18 @@ jobs:
           new-version: ${{ env.NEXT_VERSION }}
           update-current: false
 
-      - name: Create PR for main
-        id: main_pr
+      - name: Create PR for MAIN_BRANCH
+        id: main_branch_pr
         uses: peter-evans/create-pull-request@v7
         with:
-          base: main
-          branch: 'create-pull-request/patch-main'
+          base: ${{ env.MAIN_BRANCH }}
+          branch: 'create-pull-request/patch-${{ env.MAIN_BRANCH }}'
           commit-message: Add bwc version ${{ env.NEXT_VERSION }}
           signoff: true
           delete-branch: true
           labels: |
             autocut
-          title: '[AUTO] [main] Add bwc version ${{ env.NEXT_VERSION }}.'
+          title: '[AUTO] [${{ env.MAIN_BRANCH }}] Add bwc version ${{ env.NEXT_VERSION }}.'
           body: |
             I've noticed that a new tag ${{ env.TAG }} was pushed, and added a bwc version ${{ env.NEXT_VERSION }}.
 
@@ -139,8 +113,7 @@ jobs:
              ### Exit Criteria
              Review and merged the following pull requests
              - [ ] ${{ steps.base_pr.outputs.pull-request-url }}
-             - [ ] ${{ steps.base_x_pr.outputs.pull-request-url }}
-             - [ ] ${{ steps.main_pr.outputs.pull-request-url }}
+             - [ ] ${{ steps.main_branch_pr.outputs.pull-request-url }}
 
              ### Additional Context
              See project wide guidance on branching and versions [[link]](https://github.com/opensearch-project/.github/blob/main/RELEASING.md).

CHANGELOG.md

@@ -6,20 +6,72 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## [Unreleased 3.x]
 ### Added
 - Add support for Warm Indices Write Block on Flood Watermark breach ([#18375](https://github.com/opensearch-project/OpenSearch/pull/18375))
+- Add support for custom index name resolver from cluster plugin ([#18593](https://github.com/opensearch-project/OpenSearch/pull/18593))
+- Rename WorkloadGroupTestUtil to WorkloadManagementTestUtil ([#18709](https://github.com/opensearch-project/OpenSearch/pull/18709))
+- Disallow resize for Warm Index, add Parameterized ITs for close in remote store ([#18686](https://github.com/opensearch-project/OpenSearch/pull/18686))
 - Ability to run Code Coverage with Gradle and produce the jacoco reports locally ([#18509](https://github.com/opensearch-project/OpenSearch/issues/18509))
 - Extend BooleanQuery must_not rewrite to numeric must, term, and terms queries ([#18498](https://github.com/opensearch-project/OpenSearch/pull/18498))
+- [Workload Management] Update logging and Javadoc, rename QueryGroup to WorkloadGroup ([#18711](https://github.com/opensearch-project/OpenSearch/issues/18711))
+- Add NodeResourceUsageStats to ClusterInfo ([#18480](https://github.com/opensearch-project/OpenSearch/issues/18472))
+- Introduce SecureHttpTransportParameters experimental API (to complement SecureTransportParameters counterpart) ([#18572](https://github.com/opensearch-project/OpenSearch/issues/18572))
+- Create equivalents of JSM's AccessController in the java agent ([#18346](https://github.com/opensearch-project/OpenSearch/issues/18346))
+- [WLM] Add WLM mode validation for workload group CRUD requests ([#18652](https://github.com/opensearch-project/OpenSearch/issues/18652))
+- Introduced a new cluster-level API to fetch remote store metadata (segments and translogs) for each shard of an index. ([#18257](https://github.com/opensearch-project/OpenSearch/pull/18257))
+- Add last index request timestamp columns to the `_cat/indices` API. ([10766](https://github.com/opensearch-project/OpenSearch/issues/10766))
+- Introduce a new pull-based ingestion plugin for file-based indexing (for local testing) ([#18591](https://github.com/opensearch-project/OpenSearch/pull/18591))
+- Add support for search pipeline in search and msearch template ([#18564](https://github.com/opensearch-project/OpenSearch/pull/18564))
+- [Workload Management] Modify logging message in WorkloadGroupService ([#18712](https://github.com/opensearch-project/OpenSearch/pull/18712))
+- Add BooleanQuery rewrite moving constant-scoring must clauses to filter clauses ([#18510](https://github.com/opensearch-project/OpenSearch/issues/18510))
+- Add functionality for plugins to inject QueryCollectorContext during QueryPhase ([#18637](https://github.com/opensearch-project/OpenSearch/pull/18637))
+- Add support for non-timing info in profiler ([#18460](https://github.com/opensearch-project/OpenSearch/issues/18460))
+- Extend Approximation Framework to other numeric types ([#18530](https://github.com/opensearch-project/OpenSearch/issues/18530))
+- Add Semantic Version field type mapper and extensive unit tests([#18454](https://github.com/opensearch-project/OpenSearch/pull/18454))
+- Pass index settings to system ingest processor factories. ([#18708](https://github.com/opensearch-project/OpenSearch/pull/18708))
+- Include named queries from rescore contexts in matched_queries array ([#18697](https://github.com/opensearch-project/OpenSearch/pull/18697))
+- Add the configurable limit on rule cardinality ([#18663](https://github.com/opensearch-project/OpenSearch/pull/18663))
+- [Experimental] Start in "clusterless" mode if a clusterless ClusterPlugin is loaded ([#18479](https://github.com/opensearch-project/OpenSearch/pull/18479))
 
 ### Changed
+- Update Subject interface to use CheckedRunnable ([#18570](https://github.com/opensearch-project/OpenSearch/issues/18570))
+- Update SecureAuxTransportSettingsProvider to distinguish between aux transport types ([#18616](https://github.com/opensearch-project/OpenSearch/pull/18616))
+- Make node duress values cacheable ([#18649](https://github.com/opensearch-project/OpenSearch/pull/18649))
+- Making multi rate limiters in repository dynamic [#18069](https://github.com/opensearch-project/OpenSearch/pull/18069)
 
 ### Dependencies
 - Bump `stefanzweifel/git-auto-commit-action` from 5 to 6 ([#18524](https://github.com/opensearch-project/OpenSearch/pull/18524))
+- Bump Apache Lucene to 10.2.2 ([#18573](https://github.com/opensearch-project/OpenSearch/pull/18573))
+- Bump `org.apache.logging.log4j:log4j-core` from 2.24.3 to 2.25.1 ([#18589](https://github.com/opensearch-project/OpenSearch/pull/18589), [#18744](https://github.com/opensearch-project/OpenSearch/pull/18744))
+- Bump `com.google.code.gson:gson` from 2.13.0 to 2.13.1 ([#18585](https://github.com/opensearch-project/OpenSearch/pull/18585))
+- Bump `com.azure:azure-core-http-netty` from 1.15.11 to 1.15.12 ([#18586](https://github.com/opensearch-project/OpenSearch/pull/18586))
+- Bump `com.squareup.okio:okio` from 3.13.0 to 3.15.0 ([#18645](https://github.com/opensearch-project/OpenSearch/pull/18645), [#18689](https://github.com/opensearch-project/OpenSearch/pull/18689))
+- Bump `com.netflix.nebula.ospackage-base` from 11.11.2 to 12.0.0 ([#18646](https://github.com/opensearch-project/OpenSearch/pull/18646))
+- Bump `com.azure:azure-storage-blob` from 12.30.0 to 12.30.1 ([#18644](https://github.com/opensearch-project/OpenSearch/pull/18644))
+- Bump `com.google.guava:failureaccess` from 1.0.1 to 1.0.2 ([#18672](https://github.com/opensearch-project/OpenSearch/pull/18672))
+- Bump `io.perfmark:perfmark-api` from 0.26.0 to 0.27.0 ([#18672](https://github.com/opensearch-project/OpenSearch/pull/18672))
+- Bump `org.bouncycastle:bctls-fips` from 2.0.19 to 2.0.20 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668))
+- Bump `org.bouncycastle:bcpkix-fips` from 2.0.7 to 2.0.8 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668))
+- Bump `org.bouncycastle:bcpg-fips` from 2.0.10 to 2.0.11 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668))
+- Bump `com.password4j:password4j` from 1.8.2 to 1.8.3 ([#18668](https://github.com/opensearch-project/OpenSearch/pull/18668))
+- Bump `com.azure:azure-core` from 1.55.3 to 1.55.5 ([#18691](https://github.com/opensearch-project/OpenSearch/pull/18691))
+- Bump `com.google.jimfs:jimfs` from 1.3.0 to 1.3.1 ([#18743](https://github.com/opensearch-project/OpenSearch/pull/18743))
+- Bump `com.azure:azure-storage-common` from 12.29.0 to 12.29.1 ([#18742](https://github.com/opensearch-project/OpenSearch/pull/18742))
+- Bump `org.apache.commons:commons-lang3` from 3.17.0 to 3.18.0 ([#18745](https://github.com/opensearch-project/OpenSearch/pull/18745))
 
 ### Deprecated
 
 ### Removed
 
 ### Fixed
 - Add task cancellation checks in aggregators ([#18426](https://github.com/opensearch-project/OpenSearch/pull/18426))
+- Fix concurrent timings in profiler ([#18540](https://github.com/opensearch-project/OpenSearch/pull/18540))
+- Fix regex query from query string query to work with field alias ([#18215](https://github.com/opensearch-project/OpenSearch/issues/18215))
+- [Autotagging] Fix delete rule event consumption in InMemoryRuleProcessingService ([#18628](https://github.com/opensearch-project/OpenSearch/pull/18628))
+- Cannot communicate with HTTP/2 when reactor-netty is enabled ([#18599](https://github.com/opensearch-project/OpenSearch/pull/18599))
+- Fix the visit of sub queries for HasParentQuery and HasChildQuery ([#18621](https://github.com/opensearch-project/OpenSearch/pull/18621))
+- Fix the backward compatibility regression with COMPLEMENT for Regexp queries introduced in OpenSearch 3.0 ([#18640](https://github.com/opensearch-project/OpenSearch/pull/18640))
+- Fix Replication lag computation ([#18602](https://github.com/opensearch-project/OpenSearch/pull/18602))
+- Fix max_score is null when sorting on score firstly ([#18715](https://github.com/opensearch-project/OpenSearch/pull/18715))
+- Fixed Staggered merge -  load average replace with AverageTrackers, some Default thresholds modified ([#18666](https://github.com/opensearch-project/OpenSearch/pull/18666))
 
 ### Security
 

DEVELOPER_GUIDE.md

@@ -189,7 +189,9 @@ It's typically easier to wait until the console stops scrolling, and then run `c
 
 ```bash
 curl localhost:9200
-
+```
+The expected reponse should be
+```
 {
   "name" : "runTask-0",
   "cluster_name" : "runTask",

MAINTAINERS.md

@@ -11,6 +11,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Andriy Redko      | [reta](https://github.com/reta)                         | Independent |
 | Ankit Jain        | [jainankitk](https://github.com/jainankitk)             | Amazon      |
 | Ashish Singh      | [ashking94](https://github.com/ashking94)               | Amazon      |
+| Bharathwaj G      | [bharath-techie](https://github.com/bharath-techie)     | Amazon      |
 | Bukhtawar Khan    | [Bukhtawar](https://github.com/Bukhtawar)               | Amazon      |
 | Charlotte Henkle  | [CEHENKLE](https://github.com/CEHENKLE)                 | Amazon      |
 | Craig Perkins     | [cwperks](https://github.com/cwperks)                   | Amazon      |
@@ -27,6 +28,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Owais Kazi        | [owaiskazi19](https://github.com/owaiskazi19)           | Amazon      |
 | Pan Guixin        | [bugmakerrrrrr](https://github.com/bugmakerrrrrr)       | ByteDance   |
 | Peter Nied        | [peternied](https://github.com/peternied)               | Amazon      |
+| Prudhvi Godithi   | [prudhvigodithi](https://github.com/prudhvigodithi)     | Amazon      |
 | Rishabh Maurya    | [rishabhmaurya](https://github.com/rishabhmaurya)       | Amazon      |
 | Rishikesh Pasham  | [Rishikesh1159](https://github.com/Rishikesh1159)       | Amazon      |
 | Sachin Kale       | [sachinpkale](https://github.com/sachinpkale)           | Amazon      |

build.gradle

@@ -118,7 +118,7 @@ subprojects {
         }
         maven {
           name = 'Snapshots'
-          url = 'https://aws.oss.sonatype.org/content/repositories/snapshots'
+          url = 'https://central.sonatype.com/repository/maven-snapshots/'
           credentials {
             username = "$System.env.SONATYPE_USERNAME"
             password = "$System.env.SONATYPE_PASSWORD"

buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle

@@ -17,7 +17,7 @@ repositories {
 }
 
 dependencies {
-  implementation "org.apache.logging.log4j:log4j-core:2.24.3"
+  implementation "org.apache.logging.log4j:log4j-core:2.25.1"
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->

client/rest-high-level/src/test/java/org/opensearch/client/RequestConvertersTests.java

@@ -1367,6 +1367,7 @@ public void testSearchTemplate() throws Exception {
         scriptParams.put("field", "name");
         scriptParams.put("value", "soren");
         searchTemplateRequest.setScriptParams(scriptParams);
+        searchTemplateRequest.setSearchPipeline("pipeline");
 
         // Verify that the resulting REST request looks as expected.
         Request request = RequestConverters.searchTemplate(searchTemplateRequest);
@@ -1391,6 +1392,7 @@ public void testRenderSearchTemplate() throws Exception {
         searchTemplateRequest.setScript("template1");
         searchTemplateRequest.setScriptType(ScriptType.STORED);
         searchTemplateRequest.setProfile(randomBoolean());
+        searchTemplateRequest.setSearchPipeline("pipeline");
 
         Map<String, Object> scriptParams = new HashMap<>();
         scriptParams.put("field", "name");
@@ -1431,6 +1433,7 @@ public void testMultiSearchTemplate() throws Exception {
             searchTemplateRequest.setScript("{\"query\": { \"match\" : { \"{{field}}\" : \"{{value}}\" }}}");
             searchTemplateRequest.setScriptType(ScriptType.INLINE);
             searchTemplateRequest.setProfile(randomBoolean());
+            searchTemplateRequest.setSearchPipeline("pipeline");
 
             Map<String, Object> scriptParams = new HashMap<>();
             scriptParams.put("field", "name");