Merge branch 'opensearch-project:main' into filecache_disk_utilization_handling

Author: Harsh-87

.github/benchmark-configs.json

@@ -35,30 +35,13 @@
     "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
   "id_3": {
-    "description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-3.0.0",
-    "supported_major_versions": ["3"],
-    "cluster-benchmark-configs": {
-      "SINGLE_NODE_CLUSTER": "true",
-      "MIN_DISTRIBUTION": "true",
-      "TEST_WORKLOAD": "nyc_taxis",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
-    },
-    "cluster_configuration": {
-      "size": "Single-Node",
-      "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
-  },
-  "id_4": {
     "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.x",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.2.1\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -68,7 +51,7 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
-  "id_5": {
+  "id_4": {
     "description": "Indexing and search configuration for pmc workload",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
@@ -84,7 +67,7 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
-  "id_6": {
+  "id_5": {
     "description": "Indexing only configuration for stack-overflow workload",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
@@ -100,15 +83,15 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
-  "id_7": {
+  "id_6": {
     "description": "Search only test-procedure for big5 with concurrent segment search setting enabled",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "ADDITIONAL_CONFIG": "search.concurrent_segment_search.enabled:true",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -118,15 +101,15 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
-  "id_8": {
+  "id_7": {
     "description": "Search only test-procedure for big5 with concurrent segment search mode as all",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:all",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -136,15 +119,15 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
-  "id_9": {
+  "id_8": {
     "description": "Search only test-procedure for big5 with concurrent segment search mode as auto",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "ADDITIONAL_CONFIG": "search.concurrent_segment_search.mode:auto",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -154,15 +137,15 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
-  "id_10": {
+  "id_9": {
     "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.0.0. Enables range query approximation.",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "big5",
       "ADDITIONAL_CONFIG": "opensearch.experimental.feature.approximate_point_range_query.enabled:true",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"big5_1_shard_single_client\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -172,7 +155,7 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
-  "id_11": {
+  "id_10": {
     "description": "Benchmarking config for NESTED workload, benchmarks nested queries with inner-hits",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
@@ -188,31 +171,14 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
-  "id_12": {
-    "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.0.0",
-    "supported_major_versions": ["3"],
-    "cluster-benchmark-configs": {
-      "SINGLE_NODE_CLUSTER": "true",
-      "MIN_DISTRIBUTION": "true",
-      "TEST_WORKLOAD": "http_logs",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"workload-snapshots-3x\",\"snapshot_name\":\"http_logs_1_shard\"}",
-      "CAPTURE_NODE_STAT": "true",
-      "TEST_PROCEDURE": "restore-from-snapshot"
-    },
-    "cluster_configuration": {
-      "size": "Single-Node",
-      "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    },
-    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
-  },
-  "id_13": {
+  "id_11": {
     "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.x",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "http_logs",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.2.1\",\"snapshot_name\":\"http_logs_1_shard\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"http_logs_1_shard\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },
@@ -222,14 +188,14 @@
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
-  "id_14": {
+  "id_12": {
     "description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-3.x",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
       "MIN_DISTRIBUTION": "true",
       "TEST_WORKLOAD": "nyc_taxis",
-      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.2.1\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
+      "WORKLOAD_PARAMS": "{\"snapshot_repo_name\":\"benchmark-workloads-repo-3x\",\"snapshot_bucket_name\":\"benchmark-workload-snapshots\",\"snapshot_region\":\"us-east-1\",\"snapshot_base_path\":\"10.3.0\",\"snapshot_name\":\"nyc_taxis_1_shard\"}",
       "CAPTURE_NODE_STAT": "true",
       "TEST_PROCEDURE": "restore-from-snapshot"
     },

.github/workflows/assemble.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           java-version: ${{ matrix.java }}
           distribution: temurin

.github/workflows/benchmark-pull-request.yml

@@ -141,15 +141,15 @@ jobs:
           ref: ${{ env.prHeadRefSha }}
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Setup Java
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           java-version: 21
           distribution: 'temurin'
       - name: Build and Assemble OpenSearch from PR
         run: |
           ./gradlew :distribution:archives:linux-tar:assemble -Dbuild.snapshot=false
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@v5
         with:
           role-to-assume: ${{ secrets.UPLOAD_ARCHIVE_ARTIFACT_ROLE }}
           role-session-name: publish-to-s3

.github/workflows/changelog_verifier.yml

@@ -16,3 +16,8 @@ jobs:
       - uses: dangoslen/changelog-enforcer@v3
         with:
           skipLabels: "autocut, skip-changelog"
+          changeIsMissingMessage: |
+            ❌ ERROR: No update to CHANGELOG.md found!
+            This project requires a changelog entry for every user-facing change.
+            Please add an entry to the changelog or ask a maintainer to add the skip-changelog label.
+            See https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#changelog for more details.

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '42 20 * * 6'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v5
+    - name: Set up JDK 21
+      uses: actions/setup-java@v4
+      with:
+        java-version: 21
+        distribution: temurin
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3

.github/workflows/dependabot_pr.yml

@@ -25,7 +25,7 @@ jobs:
 
       # See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
       - name: Set up JDK 21
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           java-version: 21
           distribution: temurin

.github/workflows/detect-breaking-change.yml

@@ -7,15 +7,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v5
-    - uses: actions/setup-java@v4
+    - uses: actions/setup-java@v5
       with:
         distribution: temurin # Temurin is a distribution of adoptium
         java-version: 21
     - uses: gradle/gradle-build-action@v3
       with:
         cache-disabled: true
         arguments: japicmp
-        gradle-version: 8.14.3
+        gradle-version: 9.1.0
         build-root-directory: server
     - if: failure()
       run: cat server/build/reports/java-compatibility/report.txt

.github/workflows/lucene-snapshots.yml

@@ -35,7 +35,7 @@ jobs:
           echo "REVISION=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
       - name: Setup JDK ${{ env.JAVA_VERSION }}
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: 'temurin'
@@ -47,7 +47,7 @@ jobs:
         run: ./gradlew publishJarsPublicationToMavenLocal -Pversion.suffix=snapshot-${{ env.REVISION }}
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@v5
         with:
           role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_SECRET_ROLE }}
           aws-region: us-east-1
@@ -60,7 +60,7 @@ jobs:
           echo "LUCENE_SNAPSHOTS_BUCKET=$lucene_snapshots_bucket" >> $GITHUB_OUTPUT
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@v5
         with:
           role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_S3_ROLE }}
           aws-region: us-east-1
@@ -70,7 +70,7 @@ jobs:
           aws s3 cp ~/.m2/repository/org/apache/lucene/ s3://${{ steps.get_s3_bucket.outputs.LUCENE_SNAPSHOTS_BUCKET }}/snapshots/lucene/org/apache/lucene/  --recursive --no-progress
 
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
+        uses: aws-actions/configure-aws-credentials@v5
         with:
           role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_ROLE }}
           aws-region: us-west-2

.github/workflows/precommit.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           java-version: ${{ matrix.java }}
           distribution: temurin

.github/workflows/publish-maven-snapshots.yml

@@ -20,7 +20,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - name: Set up JDK 21
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           distribution: temurin
           java-version: 21