Merge remote-tracking branch 'upstream/main' into fix_yaml_test

Author: gaobinlong

.github/workflows/codeql-analysis.yml

@@ -43,7 +43,7 @@ jobs:
         distribution: temurin
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v4
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      uses: github/codeql-action/autobuild@v4
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v4

.github/workflows/wrapper.yml

@@ -8,4 +8,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
-      - uses: gradle/actions/wrapper-validation@v4
+      - uses: gradle/actions/wrapper-validation@v5

CHANGELOG.md

@@ -11,12 +11,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Use Lucene `pack` method for `half_float` and `usigned_long` when using `ApproximatePointRangeQuery`.
 - Add a mapper for context aware segments grouping criteria ([#19233](https://github.com/opensearch-project/OpenSearch/pull/19233))
 - Return full error for GRPC error response ([#19568](https://github.com/opensearch-project/OpenSearch/pull/19568))
+- Add support for repository with Server side encryption enabled and client side encryption as well based on a flag. ([#19630)](https://github.com/opensearch-project/OpenSearch/pull/19630))
 - Add pluggable gRPC interceptors with explicit ordering([#19005](https://github.com/opensearch-project/OpenSearch/pull/19005))
 - Add BindableServices extension point to transport-grpc-spi ([#19304](https://github.com/opensearch-project/OpenSearch/pull/19304))
 - Add metrics for the merged segment warmer feature ([#18929](https://github.com/opensearch-project/OpenSearch/pull/18929))
 - Add pointer based lag metric in pull-based ingestion ([#19635](https://github.com/opensearch-project/OpenSearch/pull/19635))
 - Introduced internal API for retrieving metadata about requested indices from transport actions  ([#18523](https://github.com/opensearch-project/OpenSearch/pull/18523))
 - Add cluster defaults for merge autoThrottle, maxMergeThreads, and maxMergeCount; Add segment size filter to the merged segment warmer ([#19629](https://github.com/opensearch-project/OpenSearch/pull/19629))
+- Add build-tooling to run in FIPS environment ([#18921](https://github.com/opensearch-project/OpenSearch/pull/18921))
+- Add SMILE/CBOR/YAML document format support to Bulk GRPC endpoint ([#19744](https://github.com/opensearch-project/OpenSearch/pull/19744))
 
 ### Changed
 - Faster `terms` query creation for `keyword` field with index and docValues enabled ([#19350](https://github.com/opensearch-project/OpenSearch/pull/19350))
@@ -29,7 +32,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Refactor the ThreadPoolStats.Stats class to use the Builder pattern instead of constructors ([#19317](https://github.com/opensearch-project/OpenSearch/pull/19317))
 - Refactor the IndexingStats.Stats class to use the Builder pattern instead of constructors ([#19306](https://github.com/opensearch-project/OpenSearch/pull/19306))
 - Remove FeatureFlag.MERGED_SEGMENT_WARMER_EXPERIMENTAL_FLAG. ([#19715](https://github.com/opensearch-project/OpenSearch/pull/19715))
--
+- Change the default value of doc_values in WildcardFieldMapper to true. ([#19796](https://github.com/opensearch-project/OpenSearch/pull/19796))
+- Make Engine#loadHistoryUUID() protected and Origin#isFromTranslog() public ([#19753](https://github.com/opensearch-project/OpenSearch/pull/19752))
+
 ### Fixed
 - Fix Allocation and Rebalance Constraints of WeightFunction are incorrectly reset ([#19012](https://github.com/opensearch-project/OpenSearch/pull/19012))
 - Fix flaky test FieldDataLoadingIT.testIndicesFieldDataCacheSizeSetting ([#19571](https://github.com/opensearch-project/OpenSearch/pull/19571))
@@ -40,6 +45,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Java Agent] Allow JRT protocol URLs in protection domain extraction ([#19683](https://github.com/opensearch-project/OpenSearch/pull/19683))
 - Fix potential concurrent modification exception when updating allocation filters ([#19701])(https://github.com/opensearch-project/OpenSearch/pull/19701))
 - Fix file-based ingestion consumer to handle start point beyond max line number([#19757])(https://github.com/opensearch-project/OpenSearch/pull/19757))
+- Fix IndexOutOfBoundsException when running include/exclude on non-existent prefix in terms aggregations ([#19637](https://github.com/opensearch-project/OpenSearch/pull/19637))
+- Fixed assertion unsafe use of ClusterService.state() in ResourceUsageCollectorService ([#19775])(https://github.com/opensearch-project/OpenSearch/pull/19775))
+- Add S3Repository.LEGACY_MD5_CHECKSUM_CALCULATION to list of repository-s3 settings ([#19788](https://github.com/opensearch-project/OpenSearch/pull/19788))
 
 ### Dependencies
 - Update to Gradle 9.1 ([#19575](https://github.com/opensearch-project/OpenSearch/pull/19575))
@@ -54,6 +62,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.apache.avro:avro` from 1.12.0 to 1.12.1 ([#19692](https://github.com/opensearch-project/OpenSearch/pull/19692))
 - Bump `com.github.spotbugs:spotbugs-annotations` from 4.9.6 to 4.9.8 ([#19691](https://github.com/opensearch-project/OpenSearch/pull/19691))
 - Bump `stefanzweifel/git-auto-commit-action` from 6 to 7 ([#19689](https://github.com/opensearch-project/OpenSearch/pull/19689))
+- Bump ch.qos.logback modules from 1.5.18 to 1.5.20 in HDFS test fixture ([#19763](https://github.com/opensearch-project/OpenSearch/pull/19763))
+- Bump `github/codeql-action` from 3 to 4 ([#19785](https://github.com/opensearch-project/OpenSearch/pull/19785))
+- Bump `gradle/actions` from 4 to 5 ([#19781](https://github.com/opensearch-project/OpenSearch/pull/19781))
+- Bump org.bouncycastle:bc-fips from 2.1.1 to 2.1.2 ([#19818](https://github.com/opensearch-project/OpenSearch/pull/19818))
+- Bump `com.google.http-client:google-http-client-gson` from 1.47.1 to 2.0.0 ([#19253](https://github.com/opensearch-project/OpenSearch/pull/19253))
 
 ### Deprecated
 - Deprecated existing constructors in ThreadPoolStats.Stats in favor of the new Builder ([#19317](https://github.com/opensearch-project/OpenSearch/pull/19317))

build.gradle

@@ -71,6 +71,11 @@ apply from: 'gradle/run.gradle'
 apply from: 'gradle/missing-javadoc.gradle'
 apply from: 'gradle/code-coverage.gradle'
 
+// Apply FIPS configuration to all projects
+allprojects {
+  apply from: "$rootDir/gradle/fips.gradle"
+}
+
 // common maven publishing configuration
 allprojects {
   group = 'org.opensearch'
@@ -421,8 +426,12 @@ gradle.projectsEvaluated {
           dependsOn(project(':libs:agent-sm:agent').prepareAgent)
           jvmArgs += ["-javaagent:" + project(':libs:agent-sm:agent').jar.archiveFile.get()]
         }
-        if (BuildParams.inFipsJvm) {
-          task.jvmArgs += ["-Dorg.bouncycastle.fips.approved_only=true"]
+        if (BuildParams.isInFipsJvm()) {
+          def fipsSecurityFile = project.rootProject.file('distribution/src/config/fips_java.security')
+          task.jvmArgs += [
+              "-Dorg.bouncycastle.fips.approved_only=true",
+              "-Djava.security.properties=${fipsSecurityFile}"
+          ]
         }
       }
     }
@@ -693,14 +702,6 @@ allprojects {
   plugins.withId('lifecycle-base') {
     checkPart1.configure { dependsOn 'check' }
   }
-
-  plugins.withId('opensearch.testclusters') {
-    testClusters.configureEach {
-      if (BuildParams.inFipsJvm) {
-        keystorePassword 'notarealpasswordphrase'
-      }
-    }
-  }
 }
 
 subprojects {

buildSrc/build.gradle

@@ -133,6 +133,14 @@ dependencies {
   integTestImplementation('org.spockframework:spock-core:2.4-M6-groovy-4.0') {
     exclude module: "groovy"
   }
+  // Integration with JUnit 4 features for Spock 2+ (which internally uses JUnit Platform -
+  // part of JUnit 5) - https://spockframework.org/spock/docs/2.4-M6/modules.html
+  integTestImplementation('org.spockframework:spock-junit4:2.4-M6-groovy-4.0') {
+    exclude module: "groovy"
+  }
+  integTestImplementation(platform("org.junit:junit-bom:${props.getProperty('junit5')}"))
+  integTestRuntimeOnly("org.junit.platform:junit-platform-launcher")
+  integTestRuntimeOnly("org.junit.vintage:junit-vintage-engine")
 }
 
 configurations.all {
@@ -185,7 +193,7 @@ if (project != rootProject) {
   disableTasks('forbiddenApisMain', 'forbiddenApisTest', 'forbiddenApisIntegTest', 'forbiddenApisTestFixtures')
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
-  if (org.opensearch.gradle.info.BuildParams.inFipsJvm) {
+  if (org.opensearch.gradle.info.BuildParams.isInFipsJvm()) {
     // We don't support running gradle with a JVM that is in FIPS 140 mode, so we don't test it.
     // WaitForHttpResourceTests tests would fail as they use JKS/PKCS12 keystores
     test.enabled = false
@@ -253,9 +261,10 @@ if (project != rootProject) {
   }
 
   tasks.register("integTest", Test) {
+    useJUnitPlatform()
     inputs.dir(file("src/testKit")).withPropertyName("testkit dir").withPathSensitivity(PathSensitivity.RELATIVE)
     systemProperty 'test.version_under_test', version
-    onlyIf { org.opensearch.gradle.info.BuildParams.inFipsJvm == false }
+    onlyIf { org.opensearch.gradle.info.BuildParams.isInFipsJvm() == false }
     maxParallelForks = System.getProperty('tests.jvms', org.opensearch.gradle.info.BuildParams.defaultParallel.toString()) as Integer
     testClassesDirs = sourceSets.integTest.output.classesDirs
     classpath = sourceSets.integTest.runtimeClasspath

buildSrc/src/integTest/groovy/org/opensearch/gradle/DistributionDownloadPluginFuncTest.groovy

@@ -30,6 +30,7 @@
 package org.opensearch.gradle
 
 
+import org.opensearch.gradle.Architecture
 import org.opensearch.gradle.fixtures.AbstractGradleFuncTest
 import org.opensearch.gradle.transform.SymbolicLinkPreservingUntarTransform
 import org.gradle.testkit.runner.TaskOutcome
@@ -56,23 +57,25 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
         where:
         version                              | platform                                   | distType
         VersionProperties.getOpenSearch()    | OpenSearchDistribution.Platform.LINUX      | "current"
-        "8.1.0-SNAPSHOT"                     | OpenSearchDistribution.Platform.LINUX   | "bwc"
-        "7.0.0"                              | OpenSearchDistribution.Platform.WINDOWS | "released"
+        "2.19.0-SNAPSHOT"                    | OpenSearchDistribution.Platform.LINUX   | "bwc"
+        "3.3.0"                              | OpenSearchDistribution.Platform.WINDOWS | "released"
     }
 
 
     def "transformed versions are kept across builds"() {
         given:
         def version = VersionProperties.getOpenSearch()
         def platform = OpenSearchDistribution.Platform.LINUX
+        def arch = Architecture.current().name().toLowerCase()
 
         buildFile << applyPluginAndSetupDistro(version, platform)
         buildFile << """
             apply plugin:'base'
         """
 
         when:
-        def runner = gradleRunner('clean', 'setupDistro', '-i')
+        def customGradleUserHome = testProjectDir.newFolder().absolutePath;
+        def runner = gradleRunner('clean', 'setupDistro', '-i', '-g', customGradleUserHome)
         def result = withMockedDistributionDownload(version, platform, runner) {
             // initial run
             build()
@@ -82,15 +85,18 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
         then:
         result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
-        assertOutputContains(result.output, "Skipping ${SymbolicLinkPreservingUntarTransform.class.simpleName}")
+        result.output.count("Unpacking opensearch-${version}-linux-${arch}.tar.gz " +
+            "using SymbolicLinkPreservingUntarTransform.") == 0
     }
 
     def "transforms are reused across projects"() {
         given:
         def version = VersionProperties.getOpenSearch()
         def platform = OpenSearchDistribution.Platform.LINUX
+        def arch = Architecture.current().name().toLowerCase()
 
         3.times {
+            testProjectDir.newFolder("sub-$it")
             settingsFile << """
                 include ':sub-$it'
             """
@@ -119,7 +125,7 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
         then:
         result.tasks.size() == 3
-        result.output.count("Unpacking opensearch-${version}-linux-x64.tar.gz " +
+        result.output.count("Unpacking opensearch-${version}-linux-${arch}.tar.gz " +
                 "using SymbolicLinkPreservingUntarTransform.") == 1
     }
 

buildSrc/src/integTest/groovy/org/opensearch/gradle/JdkDownloadPluginFuncTest.groovy

@@ -108,6 +108,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform)
         def mockedContent = filebytes(jdkVendor, platform)
         3.times {
+            testProjectDir.newFolder("sub-$it")
             settingsFile << """
                 include ':sub-$it'
             """
@@ -193,7 +194,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         }
 
         then:
-        assertOutputContains(result.output, "Skipping $transformType")
+        result.output.count("Unpacking ${platform}-12.0.2-x64.tar.gz using ${transformType}.") == 0
 
         where:
         platform  | transformType
@@ -246,6 +247,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
                 p.repositories.all { repo ->
                     if(repo.name == "jdk_repo_${jdkVendor}_${jdkVersion}") {
                       repo.setUrl('${server.baseUrl()}')
+                      repo.setAllowInsecureProtocol(true)
                     }
                 }
            }"""

buildSrc/src/integTest/groovy/org/opensearch/gradle/fixtures/DistributionDownloadFixture.groovy

@@ -30,6 +30,7 @@
 package org.opensearch.gradle.fixtures
 
 
+import org.opensearch.gradle.Architecture
 import org.opensearch.gradle.OpenSearchDistribution
 import org.opensearch.gradle.Version
 import org.opensearch.gradle.VersionProperties
@@ -66,11 +67,12 @@ class DistributionDownloadFixture {
     private static String urlPath(String version, OpenSearchDistribution.Platform platform) {
         String fileType = ((platform == OpenSearchDistribution.Platform.LINUX ||
                 platform == OpenSearchDistribution.Platform.DARWIN)) ? "tar.gz" : "zip"
+        String arch = Architecture.current().name().toLowerCase()
         if (Version.fromString(version).onOrAfter(Version.fromString("1.0.0"))) {
             if (version.contains("SNAPSHOT")) {
-                return "/snapshots/core/opensearch/${version}/opensearch-min-${version}-${platform}-x64-latest.$fileType"
+                return "/snapshots/core/opensearch/${version}/opensearch-min-${version}-${platform}-${arch}-latest.$fileType"
             }
-            return "/releases/core/opensearch/${version}/opensearch-min-${version}-${platform}-x64.$fileType"
+            return "/releases/core/opensearch/${version}/opensearch-min-${version}-${platform}-${arch}.$fileType"
         } else {
             return "/downloads/elasticsearch/elasticsearch-oss-${version}-${platform}-x86_64.$fileType"
         }

buildSrc/src/integTest/groovy/org/opensearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy

@@ -98,6 +98,9 @@ class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncT
         given:
         file('someFile.txt') << "some content"
 
+        testProjectDir.newFolder("consumer")
+        testProjectDir.newFolder("producer-tar")
+
         settingsFile << """
             include ':consumer'
             include ':producer-tar'

buildSrc/src/integTest/groovy/org/opensearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -29,6 +29,7 @@
 
 package org.opensearch.gradle.internal
 
+import org.opensearch.gradle.Architecture
 import org.opensearch.gradle.VersionProperties
 import org.opensearch.gradle.fixtures.AbstractGradleFuncTest
 import org.gradle.testkit.runner.GradleRunner
@@ -59,7 +60,8 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
     def "resolves current version from local build"() {
         given:
         internalBuild()
-        localDistroSetup()
+        def archive = archiveTask()
+        localDistroSetup(archive)
         def distroVersion = VersionProperties.getOpenSearch()
         buildFile << """
             apply plugin: 'opensearch.internal-distribution-download'
@@ -82,7 +84,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()
 
         then:
-        result.task(":distribution:archives:linux-tar:buildExpanded").outcome == TaskOutcome.SUCCESS
+        result.task(":distribution:archives:${archive}:buildExpanded").outcome == TaskOutcome.SUCCESS
         result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
         assertExtractedDistroIsCreated("build/distro", 'current-marker.txt')
     }
@@ -122,6 +124,8 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         internalBuild()
         bwcMinorProjectSetup()
         buildFile << """
+            import org.opensearch.gradle.JavaPackageType
+
             apply plugin: 'opensearch.internal-distribution-download'
 
             opensearch_distributions {
@@ -130,7 +134,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
                   type = "archive"
                   platform = "linux"
                   architecture = Architecture.current();
-                  bundledJdk = false
+                  bundledJdk = JavaPackageType.NONE
               }
             }
             tasks.register("createExtractedTestDistro") {
@@ -148,42 +152,47 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         settingsFile << """
         include ':distribution:bwc:minor'
         """
+        def archive = archiveTask()
         def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "minor")
         new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=minor"
         new File(bwcSubProjectFolder, 'build.gradle') << """
             apply plugin:'base'
 
             // packed distro
-            configurations.create("linux-tar")
+            configurations.create("${archive}")
             tasks.register("buildBwcTask", Tar) {
                 from('bwc-marker.txt')
                 archiveExtension = "tar.gz"
                 compression = Compression.GZIP
             }
             artifacts {
-                it.add("linux-tar", buildBwcTask)
+                it.add("${archive}", buildBwcTask)
             }
 
             // expanded distro
-            configurations.create("expanded-linux-tar")
+            configurations.create("expanded-${archive}")
             def expandedTask = tasks.register("buildBwcExpandedTask", Copy) {
                 from('bwc-marker.txt')
                 into('build/install/opensearch-distro')
             }
             artifacts {
-                it.add("expanded-linux-tar", file('build/install')) {
+                it.add("expanded-${archive}", file('build/install')) {
                     builtBy expandedTask
                     type = 'directory'
                 }
             }
         """
     }
 
-    private void localDistroSetup() {
+    private String archiveTask() {
+        return Architecture.current() == Architecture.X64 ? "linux-tar" : "linux-${Architecture.current().name().toLowerCase()}-tar"; 
+    }
+
+    private void localDistroSetup(def archive) {
         settingsFile << """
-        include ":distribution:archives:linux-tar"
+        include ":distribution:archives:${archive}"
         """
-        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "linux-tar")
+        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "${archive}")
         new File(bwcSubProjectFolder, 'current-marker.txt') << "current"
         new File(bwcSubProjectFolder, 'build.gradle') << """
             import org.gradle.api.internal.artifacts.ArtifactAttributes;