Merge branch 'main' into indicesLookup

Signed-off-by: Sandesh Kumar <sandeshkr419@gmail.com>

.ci/bwcVersions

@@ -19,3 +19,4 @@ BWC_VERSION:
   - "2.7.0"
   - "2.7.1"
   - "2.8.0"
+  - "2.9.0"

CHANGELOG.md

@@ -41,6 +41,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.3.0
 - Bump `com.diffplug.spotless` from 6.17.0 to 6.18.0
 - Bump `io.opencensus:opencensus-api` from 0.18.0 to 0.31.1 ([#7291](https://github.com/opensearch-project/OpenSearch/pull/7291))
+- Add `org.reactivestreams` 1.0.4 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
+- Add  `com.github.luben:zstd-jni` version 1.5.5-3 ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996))
+- OpenJDK Update (April 2023 Patch releases) ([#7344](https://github.com/opensearch-project/OpenSearch/pull/7344)
+- Bump `com.amazonaws` 1.12.270 to `software.amazon.awssdk` 2.20.55 ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -50,6 +54,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Change http code for DecommissioningFailedException from 500 to 400 ([#5283](https://github.com/opensearch-project/OpenSearch/pull/5283))
 - Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
 - Reduce memory copy in zstd compression ([#7681](https://github.com/opensearch-project/OpenSearch/pull/7681))
+- Add ZSTD compression for snapshotting ([#2996](https://github.com/opensearch-project/OpenSearch/pull/2996))
+- Change `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` to `aws.ec2MetadataServiceEndpoint` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
+- Change `com.amazonaws.sdk.stsEndpointOverride` to `aws.stsEndpointOverride` ([7372](https://github.com/opensearch-project/OpenSearch/pull/7372/))
 
 ### Deprecated
 
@@ -82,9 +89,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 
 ### Dependencies
+- Bump `jackson` from 2.15.1 to 2.15.2 ([#7897](https://github.com/opensearch-project/OpenSearch/pull/7897))
 
 ### Changed
 - Replace jboss-annotations-api_1.2_spec with jakarta.annotation-api ([#7836](https://github.com/opensearch-project/OpenSearch/pull/7836))
+- Add min, max, average and thread info to resource stats in tasks API ([#7673](https://github.com/opensearch-project/OpenSearch/pull/7673))
 - Optimized Metadata build() to skip redundant computations as part of ClusterState build ([#7853](https://github.com/opensearch-project/OpenSearch/pull/7853))
 
 ### Deprecated

buildSrc/src/main/java/org/opensearch/gradle/JdkDownloadPlugin.java

@@ -129,13 +129,9 @@ private void setupRepository(Project project, Jdk jdk) {
                 // To distinguish between those, the GA releases have only major version component (fe 17+32),
                 // the updates always have minor/patch components (fe 17.0.1+12), checking for the presence of
                 // version separator '.' should be enough.
-                artifactPattern = "jdk-"
-                    + jdk.getBaseVersion()
-                    + "+"
-                    + jdk.getBuild()
-                    + "/OpenJDK"
-                    + jdk.getMajor()
-                    + (jdk.getBaseVersion().contains(".") ? "U" : "")
+                artifactPattern = "jdk-" + jdk.getBaseVersion() + "+" + jdk.getBuild() + "/OpenJDK" + jdk.getMajor()
+                // JDK-20 does use 'U' suffix all the time, no matter it is update or GA release
+                    + (jdk.getBaseVersion().contains(".") || jdk.getBaseVersion().matches("^2\\d+$") ? "U" : "")
                     + "-jdk_[classifier]_[module]_hotspot_"
                     + jdk.getBaseVersion()
                     + "_"

buildSrc/src/main/java/org/opensearch/gradle/VersionProperties.java

@@ -31,6 +31,8 @@
 
 package org.opensearch.gradle;
 
+import org.apache.commons.lang3.StringUtils;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.HashMap;
@@ -54,22 +56,26 @@ public static String getLucene() {
         return lucene;
     }
 
-    public static String getBundledJdk(final String platform) {
+    public static String getBundledJdk(final String platform, final String arch) {
         switch (platform) {
             case "darwin": // fall trough
             case "mac":
                 return bundledJdkDarwin;
             case "freebsd":
                 return bundledJdkFreeBSD;
             case "linux":
-                return bundledJdkLinux;
+                return getBundledJdkLinux(arch);
             case "windows":
                 return bundledJdkWindows;
             default:
                 throw new IllegalArgumentException("unknown platform [" + platform + "]");
         }
     }
 
+    public static String getBundledJdk(final String platform) {
+        return getBundledJdk(platform, null);
+    }
+
     public static String getBundledJdkVendor() {
         return bundledJdkVendor;
     }
@@ -84,6 +90,10 @@ public static Map<String, String> getVersions() {
     private static final String bundledJdkFreeBSD;
     private static final String bundledJdkLinux;
     private static final String bundledJdkWindows;
+    private static final String bundledJdkLinux_arm64;
+    private static final String bundledJdkLinux_x64;
+    private static final String bundledJdkLinux_s390x;
+    private static final String bundledJdkLinux_ppc64le;
     private static final String bundledJdkVendor;
     private static final Map<String, String> versions = new HashMap<String, String>();
 
@@ -98,6 +108,12 @@ public static Map<String, String> getVersions() {
         bundledJdkLinux = props.getProperty("bundled_jdk_linux", bundledJdk);
         bundledJdkWindows = props.getProperty("bundled_jdk_windows", bundledJdk);
 
+        // Bundled JDKs per architecture (linux platform)
+        bundledJdkLinux_arm64 = props.getProperty("bundled_jdk_linux_arm64", bundledJdkLinux);
+        bundledJdkLinux_x64 = props.getProperty("bundled_jdk_linux_x64", bundledJdkLinux);
+        bundledJdkLinux_s390x = props.getProperty("bundled_jdk_linux_s390x", bundledJdkLinux);
+        bundledJdkLinux_ppc64le = props.getProperty("bundled_jdk_linux_ppc64le", bundledJdkLinux);
+
         for (String property : props.stringPropertyNames()) {
             versions.put(property, props.getProperty(property));
         }
@@ -119,4 +135,24 @@ private static Properties getVersionProperties() {
     public static boolean isOpenSearchSnapshot() {
         return opensearch.endsWith("-SNAPSHOT");
     }
+
+    private static String getBundledJdkLinux(String arch) {
+        if (StringUtils.isBlank(arch)) {
+            return bundledJdkLinux;
+        }
+
+        switch (arch) {
+            case "aarch64":
+            case "arm64":
+                return bundledJdkLinux_arm64;
+            case "x64":
+                return bundledJdkLinux_x64;
+            case "s390x":
+                return bundledJdkLinux_s390x;
+            case "ppc64le":
+                return bundledJdkLinux_ppc64le;
+            default:
+                throw new IllegalArgumentException("unknown platform [" + arch + "] for 'linux' platform");
+        }
+    }
 }

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -75,9 +75,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "11.0.18+10";
+    private static final String SYSTEM_JDK_VERSION = "11.0.19+7";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.6+10";
+    private static final String GRADLE_JDK_VERSION = "17.0.7+7";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/version.properties

@@ -2,14 +2,15 @@ opensearch        = 3.0.0
 lucene            = 9.7.0-snapshot-4d1ed9e
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 19.0.2+7
-
+bundled_jdk = 20.0.1+9
+# See please https://github.com/adoptium/temurin-build/issues/3371
+bundled_jdk_linux_ppc64le = 20+36
 
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.15.1
-jackson_databind  = 2.15.1
+jackson           = 2.15.2
+jackson_databind  = 2.15.2
 snakeyaml         = 2.0
 icu4j             = 70.1
 supercsv          = 2.4.0
@@ -41,7 +42,8 @@ commonslogging    = 1.2
 commonscodec      = 1.15
 
 # plugin dependencies
-aws               = 1.12.270
+aws               = 2.20.55
+reactivestreams   = 1.0.4
 
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
@@ -57,3 +59,6 @@ bytebuddy         = 1.14.3
 
 # benchmark dependencies
 jmh               = 1.35
+
+# compression
+zstd              = 1.5.5-3

client/rest-high-level/src/test/java/org/opensearch/client/core/tasks/GetTaskResponseTests.java

@@ -43,6 +43,7 @@
 import org.opensearch.tasks.Task;
 import org.opensearch.tasks.TaskId;
 import org.opensearch.tasks.TaskInfo;
+import org.opensearch.tasks.TaskThreadUsage;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.io.IOException;
@@ -138,12 +139,12 @@ private static RawTaskStatus randomRawTaskStatus() {
     }
 
     private static TaskResourceStats randomResourceStats() {
-        return randomBoolean() ? null : new TaskResourceStats(new HashMap<String, TaskResourceUsage>() {
+        return randomBoolean() ? null : new TaskResourceStats(new HashMap<>() {
             {
                 for (int i = 0; i < randomInt(5); i++) {
                     put(randomAlphaOfLength(5), new TaskResourceUsage(randomNonNegativeLong(), randomNonNegativeLong()));
                 }
             }
-        });
+        }, new TaskThreadUsage(randomInt(10), randomInt(10)));
     }
 }

client/sniffer/licenses/jackson-core-2.15.2.jar.sha1

@@ -0,0 +1 @@
+a6fe1836469a69b3ff66037c324d75fc66ef137c

distribution/build.gradle

@@ -283,7 +283,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
       (platform == 'linux' || platform == 'darwin' ? ['x64', 'aarch64', 's390x', 'ppc64le'] : ['x64']).each { architecture ->
         "bundled_${platform}_${architecture}" {
           it.platform = platform
-          it.version = VersionProperties.getBundledJdk(platform)
+          it.version = VersionProperties.getBundledJdk(platform, architecture)
           it.vendor = VersionProperties.bundledJdkVendor
           it.architecture = architecture
         }

distribution/tools/plugin-cli/build.gradle

@@ -82,9 +82,6 @@ thirdPartyAudit.ignoreViolations(
 )
 
 thirdPartyAudit.ignoreMissingClasses(
-  'com.github.luben.zstd.BufferPool',
-  'com.github.luben.zstd.ZstdInputStream',
-  'com.github.luben.zstd.ZstdOutputStream',
   'org.brotli.dec.BrotliInputStream',
   'org.objectweb.asm.AnnotationVisitor',
   'org.objectweb.asm.Attribute',

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.15.2.jar.sha1

@@ -0,0 +1 @@
+4724a65ac8e8d156a24898d50fd5dbd3642870b8

distribution/tools/upgrade-cli/licenses/jackson-databind-2.15.2.jar.sha1

@@ -0,0 +1 @@
+9353b021f10c307c00328f52090de2bdb4b6ff9c

gradle/runtime-jdk-provision.gradle

@@ -32,7 +32,7 @@ if (BuildParams.getIsRuntimeJavaHomeSet()) {
   jdks {
     provisioned_runtime {
       vendor = VersionProperties.bundledJdkVendor
-      version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase())
+      version = VersionProperties.getBundledJdk(OS.current().name().toLowerCase(), Architecture.current().name().toLowerCase())
       platform = OS.current().name().toLowerCase()
       architecture = Architecture.current().name().toLowerCase()
     }

libs/core/licenses/jackson-core-2.15.2.jar.sha1

@@ -0,0 +1 @@
+a6fe1836469a69b3ff66037c324d75fc66ef137c

libs/core/src/main/java/org/opensearch/Version.java

@@ -88,6 +88,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_7_0 = new Version(2070099, org.apache.lucene.util.Version.LUCENE_9_5_0);
     public static final Version V_2_7_1 = new Version(2070199, org.apache.lucene.util.Version.LUCENE_9_5_0);
     public static final Version V_2_8_0 = new Version(2080099, org.apache.lucene.util.Version.LUCENE_9_6_0);
+    public static final Version V_2_9_0 = new Version(2090099, org.apache.lucene.util.Version.LUCENE_9_6_0);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version CURRENT = V_3_0_0;
 

libs/x-content/licenses/jackson-core-2.15.2.jar.sha1

@@ -0,0 +1 @@
+a6fe1836469a69b3ff66037c324d75fc66ef137c

libs/x-content/licenses/jackson-dataformat-cbor-2.15.2.jar.sha1

@@ -0,0 +1 @@
+baafc85c70765594add14bd93f3efd68e1945b76

libs/x-content/licenses/jackson-dataformat-smile-2.15.2.jar.sha1

@@ -0,0 +1 @@
+16d1dd22f7d641459ed056399d4f7df0220f1176

libs/x-content/licenses/jackson-dataformat-yaml-2.15.2.jar.sha1

@@ -0,0 +1 @@
+58194ff9f51915ad6bf6b6f24818232d7566418a

modules/ingest-geoip/licenses/jackson-annotations-2.15.2.jar.sha1

@@ -0,0 +1 @@
+4724a65ac8e8d156a24898d50fd5dbd3642870b8

modules/ingest-geoip/licenses/jackson-databind-2.15.2.jar.sha1

@@ -0,0 +1 @@
+9353b021f10c307c00328f52090de2bdb4b6ff9c

modules/transport-netty4/build.gradle

@@ -185,7 +185,6 @@ thirdPartyAudit {
     'org.slf4j.LoggerFactory',
     'org.slf4j.spi.LocationAwareLogger',
 
-    'com.github.luben.zstd.Zstd',
     'com.google.protobuf.nano.CodedOutputByteBufferNano',
     'com.google.protobuf.nano.MessageNano',
     'com.jcraft.jzlib.Deflater',