Merge branch '2.x' into backport/backport-6004-to-2.x

.github/ISSUE_TEMPLATE/failed_check.md

@@ -0,0 +1,8 @@
+---
+title: '[AUTOCUT] Gradle Check Failure on push to {{ env.branch_name }}'
+labels: '>test-failure, bug'
+---
+
+Gradle check has failed on push of your commit {{ env.pr_from_sha }}.
+Please examine the workflow log {{ env.workflow_url }}.
+Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?

.github/workflows/gradle-check.yml

@@ -8,8 +8,16 @@ on:
   pull_request_target:
     types: [opened, synchronize, reopened]
 
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
 jobs:
   gradle-check:
+    permissions:
+      contents: read # to fetch code (actions/checkout)
+      pull-requests: write # to create or update comment (peter-evans/create-or-update-comment)
+      issues: write # To create an issue if check fails on push.
+
     runs-on: ubuntu-latest
     timeout-minutes: 130
     steps:
@@ -33,6 +41,7 @@ jobs:
             repo_url="https://github.com/opensearch-project/OpenSearch"
             ref_id=$(git rev-parse HEAD)
             branch_name=$(git rev-parse --abbrev-ref HEAD)
+            echo "$branch_name=$branch_name" >> $GITHUB_ENV
             echo "pr_from_sha=$ref_id" >> $GITHUB_ENV
             echo "pr_from_clone_url=$repo_url" >> $GITHUB_ENV
             echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV
@@ -116,3 +125,12 @@ jobs:
               * **CommitID:** ${{ env.pr_from_sha }}
               Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
               Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
+
+      - name: Create Issue On Push Failure
+        if: ${{ github.event_name == 'push' && failure() }}
+        uses: dblock/create-a-github-issue@v3
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          assignees: ${{ github.event.head_commit.author.username }}, ${{ github.triggering_actor }}
+          filename: .github/ISSUE_TEMPLATE/failed_check.md

.github/workflows/links.yml

@@ -2,6 +2,8 @@ name: Link Checker
 on:
   schedule:
     - cron:  '0 0 * * *'
+permissions:
+  contents: read # to fetch code (actions/checkout)
 jobs:
   linkchecker:
     if: github.repository == 'opensearch-project/OpenSearch'

.github/workflows/version.yml

@@ -5,6 +5,7 @@ on:
     tags:
       - '*.*.*'
 
+permissions: {}
 jobs:
   build:
     runs-on: ubuntu-latest

CHANGELOG.md

@@ -5,21 +5,32 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 2.x]
 ### Added
+- Adding index create block when all nodes have breached high disk watermark ([#5852](https://github.com/opensearch-project/OpenSearch/pull/5852))
 - Added cluster manager throttling stats in nodes/stats API ([#5790](https://github.com/opensearch-project/OpenSearch/pull/5790))
 - Added support for feature flags in opensearch.yml ([#4959](https://github.com/opensearch-project/OpenSearch/pull/4959))
 - Add query for initialized extensions ([#5658](https://github.com/opensearch-project/OpenSearch/pull/5658))
 - Add update-index-settings allowlist for searchable snapshot ([#5907](https://github.com/opensearch-project/OpenSearch/pull/5907))
 - Replace latches with CompletableFutures for extensions ([#5646](https://github.com/opensearch-project/OpenSearch/pull/5646))
 - Add support to disallow search request with preference parameter with strict weighted shard routing([#5874](https://github.com/opensearch-project/OpenSearch/pull/5874))
+- Added support to apply index create block ([#4603](https://github.com/opensearch-project/OpenSearch/issues/4603))
+- Adds support for minimum compatible version for extensions ([#6003](https://github.com/opensearch-project/OpenSearch/pull/6003))
 
 ### Dependencies
 - Update nebula-publishing-plugin to 19.2.0 ([#5704](https://github.com/opensearch-project/OpenSearch/pull/5704))
 - Bumps `reactor-netty` from 1.1.1 to 1.1.2 ([#5878](https://github.com/opensearch-project/OpenSearch/pull/5878))
+- OpenJDK Update (January 2023 Patch releases) ([#6075](https://github.com/opensearch-project/OpenSearch/pull/6075))
+- Bumps `Mockito` from 4.7.0 to 5.1.0, `ByteBuddy` from 1.12.18 to 1.12.22 ([#6088](https://github.com/opensearch-project/OpenSearch/pull/6088))
+- Bumps `joda` from 2.10.13 to 2.12.2 ([#6095](https://github.com/opensearch-project/OpenSearch/pull/6095))
+- Upgrade to Lucene 9.5.0 ([#6078](https://github.com/opensearch-project/OpenSearch/pull/6078))
+- Bump antlr4 from 4.9.3 to 4.11.1 ([#6116](https://github.com/opensearch-project/OpenSearch/pull/6116))
+- Bumps `Netty` from 4.1.86.Final to 4.1.87.Final ([#6130](https://github.com/opensearch-project/OpenSearch/pull/6130))
+- Bumps `Jackson` from 2.14.1 to 2.14.2 ([#6129](https://github.com/opensearch-project/OpenSearch/pull/6129))
 
 ### Changed
 - Use ReplicationFailedException instead of OpensearchException in ReplicationTarget ([#4725](https://github.com/opensearch-project/OpenSearch/pull/4725))
 - [Refactor] Use local opensearch.common.SetOnce instead of lucene's utility class ([#5947](https://github.com/opensearch-project/OpenSearch/pull/5947))
 - Cluster health call to throw decommissioned exception for local decommissioned node([#6008](https://github.com/opensearch-project/OpenSearch/pull/6008))
+- [Refactor] core.common to new opensearch-common library ([#5976](https://github.com/opensearch-project/OpenSearch/pull/5976))
 
 ### Deprecated
 
@@ -28,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - [Segment Replication] Fix for peer recovery ([#5344](https://github.com/opensearch-project/OpenSearch/pull/5344))
 - Fix weighted shard routing state across search requests([#6004](https://github.com/opensearch-project/OpenSearch/pull/6004))
+- [Segment Replication] Fix bug where inaccurate sequence numbers are sent during replication ([#6122](https://github.com/opensearch-project/OpenSearch/pull/6122))
 
 ### Security
 

buildSrc/build.gradle

@@ -191,6 +191,7 @@ if (project != rootProject) {
     distribution project(':distribution:archives:linux-tar')
     distribution project(':distribution:archives:windows-zip')
 
+    integTestRuntimeOnly(project(":libs:opensearch-common"))
     integTestRuntimeOnly(project(":libs:opensearch-core"))
   }
 

buildSrc/src/main/java/org/opensearch/gradle/precommit/JarHellPrecommitPlugin.java

@@ -44,11 +44,11 @@ public class JarHellPrecommitPlugin extends PrecommitPlugin {
     @Override
     public TaskProvider<? extends Task> createTask(Project project) {
         Configuration jarHellConfig = project.getConfigurations().create("jarHell");
-        if (BuildParams.isInternal() && project.getPath().equals(":libs:opensearch-core") == false) {
+        if (BuildParams.isInternal() && project.getPath().equals(":libs:opensearch-common") == false) {
             // External plugins will depend on this already via transitive dependencies.
             // Internal projects are not all plugins, so make sure the check is available
             // we are not doing this for this project itself to avoid jar hell with itself
-            project.getDependencies().add("jarHell", project.project(":libs:opensearch-core"));
+            project.getDependencies().add("jarHell", project.project(":libs:opensearch-common"));
         }
 
         TaskProvider<JarHellTask> jarHell = project.getTasks().register("jarHell", JarHellTask.class);

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -75,9 +75,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "11.0.17+8";
+    private static final String SYSTEM_JDK_VERSION = "11.0.18+10";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.5+8";
+    private static final String GRADLE_JDK_VERSION = "17.0.6+10";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/version.properties

@@ -1,32 +1,31 @@
 opensearch        = 2.6.0
-lucene            = 9.4.2
+lucene            = 9.5.0
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 17.0.5+8
-
-
+bundled_jdk = 17.0.6+10
 
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0
-jackson           = 2.14.1
-jackson_databind  = 2.14.1
+jackson           = 2.14.2
+jackson_databind  = 2.14.2
 snakeyaml         = 1.32
 icu4j             = 70.1
 supercsv          = 2.4.0
-# Update to 2.17.2+ is breaking OpenSearchJsonLayout (see https://issues.apache.org/jira/browse/LOG4J2-3562) 
+# Update to 2.17.2+ is breaking OpenSearchJsonLayout (see https://issues.apache.org/jira/browse/LOG4J2-3562)
 log4j             = 2.17.1
 slf4j             = 1.7.36
 asm               = 9.4
 jettison          = 1.5.1
 woodstox          = 6.4.0
 kotlin            = 1.7.10
+antlr4            = 4.11.1
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0
 
-netty             = 4.1.86.Final
-joda              = 2.10.12
+netty             = 4.1.87.Final
+joda              = 2.12.2
 
 # client dependencies
 httpclient        = 4.5.13
@@ -46,10 +45,9 @@ bouncycastle=1.70
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
 hamcrest          = 2.1
-# Update to 4.8.0 is using reflection without SecurityManager checks (fails with java.security.AccessControlException)
-mockito           = 4.7.0
+mockito           = 5.1.0
 objenesis         = 3.2
-bytebuddy         = 1.12.18
+bytebuddy         = 1.12.22
 
 # benchmark dependencies
 jmh               = 1.35

client/rest/build.gradle

@@ -54,6 +54,7 @@ dependencies {
   testImplementation "org.mockito:mockito-core:${versions.mockito}"
   testImplementation "org.objenesis:objenesis:${versions.objenesis}"
   testImplementation "net.bytebuddy:byte-buddy:${versions.bytebuddy}"
+  testImplementation "net.bytebuddy:byte-buddy-agent:${versions.bytebuddy}"
   testImplementation "org.apache.logging.log4j:log4j-api:${versions.log4j}"
   testImplementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
   testImplementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}"

client/sniffer/build.gradle

@@ -50,6 +50,7 @@ dependencies {
   testImplementation "org.mockito:mockito-core:${versions.mockito}"
   testImplementation "org.objenesis:objenesis:${versions.objenesis}"
   testImplementation "net.bytebuddy:byte-buddy:${versions.bytebuddy}"
+  testImplementation "net.bytebuddy:byte-buddy-agent:${versions.bytebuddy}"
 }
 
 tasks.named('forbiddenApisMain').configure {

client/sniffer/licenses/jackson-core-2.14.2.jar.sha1

@@ -0,0 +1 @@
+f804090e6399ce0cf78242db086017512dd71fcc

distribution/tools/upgrade-cli/licenses/jackson-annotations-2.14.2.jar.sha1

@@ -0,0 +1 @@
+a7aae9525864930723e3453ab799521fdfd9d873

distribution/tools/upgrade-cli/licenses/jackson-databind-2.14.2.jar.sha1

@@ -0,0 +1 @@
+01e71fddbc80bb86f71a6345ac1e8ab8a00e7134

gradle/missing-javadoc.gradle

@@ -108,7 +108,6 @@ configure([
   project(":example-plugins:rest-handler"),
   project(":example-plugins:script-expert-scoring"),
   project(":libs:opensearch-cli"),
-  project(":libs:opensearch-core"),
   project(":libs:opensearch-dissect"),
   project(":libs:opensearch-geo"),
   project(":libs:opensearch-grok"),
@@ -176,7 +175,11 @@ configure([
   }
 }
 
-configure(project(":server")) {
+configure([
+  project(":libs:opensearch-common"),
+  project(":libs:opensearch-core"),
+  project(":server")
+]) {
   project.tasks.withType(MissingJavadocTask) {
     // TODO: bump to variable missing level after increasing javadoc coverage
     javadocMissingLevel = "class"

libs/build.gradle

@@ -44,11 +44,12 @@ subprojects {
       dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
         Project depProject = dep.dependencyProject
         if (depProject != null
-          && false == depProject.path.equals(':libs:opensearch-core')
+          && (false == depProject.path.equals(':libs:opensearch-core') &&
+              false == depProject.path.equals(':libs:opensearch-common'))
           && depProject.path.startsWith(':libs')) {
           throw new InvalidUserDataException("projects in :libs "
             + "may not depend on other projects libs except "
-            + ":libs:opensearch-core but "
+            + ":libs:opensearch-core or :libs:opensearch-common but "
             + "${project.path} depends on ${depProject.path}")
         }
       }

libs/cli/build.gradle

@@ -33,6 +33,7 @@ apply plugin: 'opensearch.publish'
 
 dependencies {
   api 'net.sf.jopt-simple:jopt-simple:5.0.4'
+  api project(':libs:opensearch-common')
   api project(':libs:opensearch-core')
 }
 

libs/common/build.gradle

@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+import org.opensearch.gradle.info.BuildParams
+
+apply plugin: 'opensearch.publish'
+
+archivesBaseName = 'opensearch-common'
+
+dependencies {
+  // This dependency is used only by :libs:core for null-checking interop with other tools
+  compileOnly "com.google.code.findbugs:jsr305:3.0.2"
+
+  testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  testImplementation "junit:junit:${versions.junit}"
+  testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
+
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.opensearch', module: 'opensearch-common'
+  }
+}
+
+tasks.named('forbiddenApisMain').configure {
+  // :libs:opensearch-common does not depend on server
+  // TODO: Need to decide how we want to handle for forbidden signatures with the changes to server
+  replaceSignatureFiles 'jdk-signatures'
+}

libs/core/src/main/java/org/opensearch/bootstrap/JarHell.java → libs/common/src/main/java/org/opensearch/bootstrap/JarHell.java

@@ -71,6 +71,8 @@
  *   <li>Checks any {@code X-Compile-OpenSearch-Version} value in
  *       the jar manifest is compatible with the current ES</li>
  * </ul>
+ *
+ * @opensearch.internal
  */
 public class JarHell {
 

libs/core/src/main/java/org/opensearch/bootstrap/JdkJarHellCheck.java → libs/common/src/main/java/org/opensearch/bootstrap/JdkJarHellCheck.java

@@ -44,6 +44,11 @@
 import java.util.HashSet;
 import java.util.Set;
 
+/**
+ * Checks for jdk jar hell
+ *
+ * @opensearch.internal
+ */
 public class JdkJarHellCheck {
 
     private Set<String> detected = new HashSet<>();

libs/common/src/main/java/org/opensearch/bootstrap/package-info.java

@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** Contains JarHell Classes */
+package org.opensearch.bootstrap;

libs/core/src/main/java/org/opensearch/common/Booleans.java → libs/common/src/main/java/org/opensearch/common/Booleans.java

@@ -32,6 +32,11 @@
 
 package org.opensearch.common;
 
+/**
+ * Boolean class utilities
+ *
+ * @opensearch.api
+ */
 public final class Booleans {
     private Booleans() {
         throw new AssertionError("No instances intended");

libs/core/src/main/java/org/opensearch/common/CharArrays.java → libs/common/src/main/java/org/opensearch/common/CharArrays.java

@@ -40,6 +40,8 @@
 
 /**
  * Helper class similar to Arrays to handle conversions for Char arrays
+ *
+ * @opensearch.api
  */
 public final class CharArrays {
 

libs/core/src/main/java/org/opensearch/common/CheckedConsumer.java → libs/common/src/main/java/org/opensearch/common/CheckedConsumer.java

@@ -36,6 +36,8 @@
 
 /**
  * A {@link Consumer}-like interface which allows throwing checked exceptions.
+ *
+ * @opensearch.api
  */
 @FunctionalInterface
 public interface CheckedConsumer<T, E extends Exception> {

libs/core/src/main/java/org/opensearch/common/CheckedFunction.java → libs/common/src/main/java/org/opensearch/common/CheckedFunction.java

@@ -36,6 +36,8 @@
 
 /**
  * A {@link Function}-like interface which allows throwing checked exceptions.
+ *
+ * @opensearch.api
  */
 @FunctionalInterface
 public interface CheckedFunction<T, R, E extends Exception> {

libs/core/src/main/java/org/opensearch/common/CheckedRunnable.java → libs/common/src/main/java/org/opensearch/common/CheckedRunnable.java

@@ -34,6 +34,8 @@
 
 /**
  * A {@link Runnable}-like interface which allows throwing checked exceptions.
+ *
+ * @opensearch.api
  */
 @FunctionalInterface
 public interface CheckedRunnable<E extends Exception> {

libs/core/src/main/java/org/opensearch/common/Glob.java → libs/common/src/main/java/org/opensearch/common/Glob.java

@@ -34,6 +34,8 @@
 
 /**
  * Utility class for glob-like matching
+ *
+ * @opensearch.api
  */
 public class Glob {