Merge branch 'main' into replica-count

.ci/bwcVersions

@@ -41,6 +41,7 @@ BWC_VERSION:
   - "1.3.2"
   - "1.3.3"
   - "1.3.4"
+  - "1.3.5"
   - "2.0.0"
   - "2.0.1"
   - "2.0.2"

CONTRIBUTING.md

@@ -59,6 +59,16 @@ If you would like to contribute to the documentation, please do so in the [docum
 
 As with other types of contributions, the first step is to [**open an issue on GitHub**](https://github.com/opensearch-project/OpenSearch/issues/new/choose). Opening an issue before you make changes makes sure that someone else isn't already working on that particular problem. It also lets us all work together to find the right approach before you spend a bunch of time on a PR. So again, when in doubt, open an issue.
 
+Additionally, here are a few guidelines to help you decide whether a particular feature should be included in OpenSearch.
+
+**Is your feature important to most users of OpenSearch?**
+
+If you believe that a feature is going to fulfill a need for most users of OpenSearch, then it belongs in OpenSearch. However, we don't want every feature built into the core server. If the feature requires additional permissions or brings in extra dependencies it should instead be included as a module in core.
+
+**Is your feature a common dependency across multiple plugins?**
+
+Does this feature contain functionality that cuts across multiple plugins? If so, this most likely belongs in OpenSearch as a core module or plugin.
+
 Once you've opened an issue, check out our [Developer Guide](./DEVELOPER_GUIDE.md) for instructions on how to get started.
 
 ## Developer Certificate of Origin

README.md

@@ -1,12 +1,12 @@
+<img src="https://opensearch.org/assets/img/opensearch-logo-themed.svg" height="64px">
+
+[![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://forum.opensearch.org/c/opensearch/)
+[![Documentation](https://img.shields.io/badge/documentation-reference-blue)](https://opensearch.org/docs/latest/opensearch/index/)
 [![codecov](https://codecov.io/gh/opensearch-project/OpenSearch/branch/main/graph/badge.svg)](https://codecov.io/gh/opensearch-project/OpenSearch)
 [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
 [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)
 [![GHA precommit](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml)
 [![Jenkins gradle check job](https://img.shields.io/jenkins/build?jobUrl=https%3A%2F%2Fbuild.ci.opensearch.org%2Fjob%2Fgradle-check%2F&label=Jenkins%20Gradle%20Check)](https://build.ci.opensearch.org/job/gradle-check/)
-[![Documentation](https://img.shields.io/badge/doc-reference-blue)](https://opensearch.org/docs/latest/opensearch/index/)
-[![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://forum.opensearch.org/c/opensearch/)
-
-<img src="https://opensearch.org/assets/img/opensearch-logo-themed.svg" height="64px">
 
 - [Welcome!](#welcome)
 - [Project Resources](#project-resources)

benchmarks/src/main/java/org/opensearch/benchmark/time/NanoTimeVsCurrentTimeMillisBenchmark.java

@@ -8,7 +8,15 @@
 
 package org.opensearch.benchmark.time;
 
-import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Benchmark;
 
 import java.util.concurrent.TimeUnit;
 

build.gradle

@@ -28,6 +28,9 @@
  * under the License.
  */
 
+import java.nio.charset.StandardCharsets;
+import java.io.ByteArrayOutputStream;
+
 import com.avast.gradle.dockercompose.tasks.ComposePull
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
@@ -37,11 +40,15 @@ import org.opensearch.gradle.Version
 import org.opensearch.gradle.VersionProperties
 import org.opensearch.gradle.info.BuildParams
 import org.opensearch.gradle.plugin.PluginBuildPlugin
+import org.opensearch.gradle.tar.SymbolicLinkPreservingTar
 import org.gradle.plugins.ide.eclipse.model.AccessRule
 import org.gradle.plugins.ide.eclipse.model.EclipseJdt
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
+import org.gradle.api.Project;
+import org.gradle.process.ExecResult;
 import org.gradle.util.DistributionLocator
 import org.gradle.util.GradleVersion
+
 import static org.opensearch.gradle.util.GradleUtils.maybeConfigure
 
 plugins {
@@ -150,6 +157,30 @@ Map<String, String> buildMetadataMap = buildMetadataValue.tokenize(';').collectE
   return [key, value]
 }
 
+    /**
+     * Using 'git' command line (if available), tries to fetch the commit date of the current revision
+     * @return commit date of the current revision or 0 if it is not available
+     */
+long gitRevisionDate = {
+  // Try to get last commit date as Unix timestamp
+  try (ByteArrayOutputStream stdout = new ByteArrayOutputStream()) {
+     ExecResult result = project.exec(spec -> {
+        spec.setIgnoreExitValue(true);
+        spec.setStandardOutput(stdout);
+        spec.commandLine("git", "log", "-1", "--format=%ct");
+     });
+
+    if (result.getExitValue() == 0) {
+      return Long.parseLong(stdout.toString(StandardCharsets.UTF_8).replaceAll("\\s", "")) * 1000; /* seconds to millis */
+    }
+  } catch (IOException | GradleException | NumberFormatException ex) {
+    /* fall back to default Unix epoch timestamp */
+  }
+
+  return 0;
+}()
+
+
 // injecting groovy property variables into all projects
 allprojects {
   project.ext {
@@ -282,11 +313,15 @@ allprojects {
   }
 
   // support for reproducible builds
-  tasks.withType(AbstractArchiveTask).configureEach {
+  tasks.withType(AbstractArchiveTask).configureEach { task ->
     // ignore file timestamps
     // be consistent in archive file order
-    preserveFileTimestamps = false
-    reproducibleFileOrder = true
+    task.preserveFileTimestamps = false
+    task.reproducibleFileOrder = true
+    if (task instanceof SymbolicLinkPreservingTar) {
+      // Replace file timestamps with latest Git revision date (if available)
+      task.lastModifiedTimestamp = gitRevisionDate
+    }
   }
 
   project.afterEvaluate {
@@ -324,7 +359,8 @@ allprojects {
           project.javadoc.dependsOn "${upstreamProject.path}:javadoc"
           String externalLinkName = upstreamProject.archivesBaseName
           String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + externalLinkName.replaceAll('\\.', '/') + '/' + dep.version
-          project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${upstreamProject.buildDir}/docs/javadoc/"
+          String projectRelativePath = project.relativePath(upstreamProject.buildDir)
+          project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${projectRelativePath}/docs/javadoc/"
         }
       }
       boolean hasShadow = project.plugins.hasPlugin(ShadowPlugin)

buildSrc/src/main/java/org/opensearch/gradle/info/BuildParams.java

@@ -192,8 +192,8 @@ public void setRuntimeJavaHome(File runtimeJavaHome) {
             BuildParams.runtimeJavaHome = requireNonNull(runtimeJavaHome);
         }
 
-        public void setIsRutimeJavaHomeSet(boolean isRutimeJavaHomeSet) {
-            BuildParams.isRuntimeJavaHomeSet = isRutimeJavaHomeSet;
+        public void setIsRuntimeJavaHomeSet(boolean isRuntimeJavaHomeSet) {
+            BuildParams.isRuntimeJavaHomeSet = isRuntimeJavaHomeSet;
         }
 
         public void setJavaVersions(List<JavaHome> javaVersions) {

buildSrc/src/main/java/org/opensearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -113,7 +113,7 @@ public void apply(Project project) {
             params.reset();
             params.setRuntimeJavaHome(runtimeJavaHome);
             params.setRuntimeJavaVersion(determineJavaVersion("runtime java.home", runtimeJavaHome, minimumRuntimeVersion));
-            params.setIsRutimeJavaHomeSet(runtimeJavaHomeOpt.isPresent());
+            params.setIsRuntimeJavaHomeSet(runtimeJavaHomeOpt.isPresent());
             params.setRuntimeJavaDetails(getJavaInstallation(runtimeJavaHome).getDisplayName());
             params.setJavaVersions(getAvailableJavaVersions(minimumCompilerVersion));
             params.setMinimumCompilerVersion(minimumCompilerVersion);

buildSrc/src/main/java/org/opensearch/gradle/tar/SymbolicLinkPreservingTar.java

@@ -65,6 +65,11 @@
  * This task is necessary because the built-in task {@link org.gradle.api.tasks.bundling.Tar} does not preserve symbolic links.
  */
 public class SymbolicLinkPreservingTar extends Tar {
+    private long lastModifiedTimestamp = 0;
+
+    public void setLastModifiedTimestamp(long lastModifiedTimestamp) {
+        this.lastModifiedTimestamp = lastModifiedTimestamp;
+    }
 
     @Override
     protected CopyAction createCopyAction() {
@@ -80,23 +85,26 @@ protected CopyAction createCopyAction() {
                 compressor = new SimpleCompressor();
                 break;
         }
-        return new SymbolicLinkPreservingTarCopyAction(getArchiveFile(), compressor, isPreserveFileTimestamps());
+        return new SymbolicLinkPreservingTarCopyAction(getArchiveFile(), compressor, isPreserveFileTimestamps(), lastModifiedTimestamp);
     }
 
     private static class SymbolicLinkPreservingTarCopyAction implements CopyAction {
 
         private final Provider<RegularFile> tarFile;
         private final ArchiveOutputStreamFactory compressor;
         private final boolean isPreserveFileTimestamps;
+        private final long lastModifiedTimestamp;
 
         SymbolicLinkPreservingTarCopyAction(
             final Provider<RegularFile> tarFile,
             final ArchiveOutputStreamFactory compressor,
-            final boolean isPreserveFileTimestamps
+            final boolean isPreserveFileTimestamps,
+            final long lastModifiedTimestamp
         ) {
             this.tarFile = tarFile;
             this.compressor = compressor;
             this.isPreserveFileTimestamps = isPreserveFileTimestamps;
+            this.lastModifiedTimestamp = lastModifiedTimestamp;
         }
 
         @Override
@@ -219,7 +227,7 @@ private void handleProcessingException(final FileCopyDetailsInternal details, fi
         }
 
         private long getModTime(final FileCopyDetails details) {
-            return isPreserveFileTimestamps ? details.getLastModified() : 0;
+            return isPreserveFileTimestamps ? details.getLastModified() : lastModifiedTimestamp;
         }
 
     }

buildSrc/src/main/java/org/opensearch/gradle/testclusters/StandaloneRestIntegTestTask.java

@@ -32,6 +32,7 @@
 package org.opensearch.gradle.testclusters;
 
 import groovy.lang.Closure;
+
 import org.opensearch.gradle.FileSystemOperationsAware;
 import org.opensearch.gradle.test.Fixture;
 import org.opensearch.gradle.util.GradleUtils;
@@ -46,6 +47,8 @@
 import org.gradle.internal.resources.ResourceLock;
 import org.gradle.internal.resources.SharedResource;
 
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -124,12 +127,32 @@ public List<ResourceLock> getSharedResources() {
 
         int nodeCount = clusters.stream().mapToInt(cluster -> cluster.getNodes().size()).sum();
         if (nodeCount > 0) {
-            locks.add(resource.getResourceLock(Math.min(nodeCount, resource.getMaxUsages())));
+            locks.add(getResourceLock(resource, Math.min(nodeCount, resource.getMaxUsages())));
         }
 
         return Collections.unmodifiableList(locks);
     }
 
+    private ResourceLock getResourceLock(SharedResource resource, int nUsages) {
+        try {
+            try {
+                // The getResourceLock(int) is used by Gradle pre-7.5
+                return (ResourceLock) MethodHandles.publicLookup()
+                    .findVirtual(SharedResource.class, "getResourceLock", MethodType.methodType(ResourceLock.class, int.class))
+                    .bindTo(resource)
+                    .invokeExact(nUsages);
+            } catch (NoSuchMethodException | IllegalAccessException ex) {
+                // The getResourceLock() is used by Gradle post-7.4
+                return (ResourceLock) MethodHandles.publicLookup()
+                    .findVirtual(SharedResource.class, "getResourceLock", MethodType.methodType(ResourceLock.class))
+                    .bindTo(resource)
+                    .invokeExact();
+            }
+        } catch (Throwable ex) {
+            throw new IllegalStateException("Unable to find suitable ResourceLock::getResourceLock", ex);
+        }
+    }
+
     @Override
     public Task dependsOn(Object... dependencies) {
         super.dependsOn(dependencies);

buildSrc/src/test/java/org/opensearch/gradle/EmptyDirTaskTests.java

@@ -33,6 +33,9 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Path;
 
 import com.carrotsearch.randomizedtesting.RandomizedTest;
 import org.apache.tools.ant.taskdefs.condition.Os;
@@ -64,7 +67,11 @@ public void testCreateEmptyDir() throws Exception {
     }
 
     public void testCreateEmptyDirNoPermissions() throws Exception {
-        RandomizedTest.assumeFalse("Functionality is Unix specific", Os.isFamily(Os.FAMILY_WINDOWS));
+        // Test depends on Posix file permissions
+        RandomizedTest.assumeFalse("Functionality is Unix-like OS specific", Os.isFamily(Os.FAMILY_WINDOWS));
+        // Java's Files.setPosixFilePermissions is a NOOP inside a Docker container as
+        // files are created by default with UID and GID = 0 (root).
+        RandomizedTest.assumeFalse("Functionality doesn't work in Docker", isRunningInDocker());
 
         Project project = ProjectBuilder.builder().build();
         EmptyDirTask emptyDirTask = project.getTasks().create("emptyDirTask", EmptyDirTask.class);
@@ -92,4 +99,20 @@ private File getNewNonExistingTempFolderFile(Project project) throws IOException
         return newEmptyFolder;
     }
 
+    private static boolean isRunningInDocker() {
+        // Only reliable existing method but may be removed in future
+        if (new File("/.dockerenv").exists()) {
+            return true;
+        }
+        try {
+            // Backup 1: look for 'docker' in one of the paths in /proc/1/cgroup
+            if (Files.lines(Path.of("/proc/1/cgroup")).anyMatch(line -> line.contains("docker"))) {
+                return true;
+            }
+            // Backup 2: look for 'docker' in overlay fs
+            return Files.lines(Path.of("/proc/1/mounts")).anyMatch(line -> line.startsWith("overlay") && line.contains("docker"));
+        } catch (InvalidPathException | IOException e) {
+            return false;
+        }
+    }
 }

buildSrc/version.properties

@@ -22,7 +22,7 @@ asm               = 9.3
 jna               = 5.5.0
 
 netty             = 4.1.79.Final
-joda              = 2.10.12
+joda              = 2.10.13
 
 # client dependencies
 httpclient        = 4.5.13

client/rest-high-level/src/main/java/org/opensearch/client/ClusterRequestConverters.java

@@ -105,7 +105,7 @@ static Request putComponentTemplate(PutComponentTemplateRequest putComponentTemp
             .build();
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
         RequestConverters.Params params = new RequestConverters.Params();
-        params.withClusterManagerTimeout(putComponentTemplateRequest.masterNodeTimeout());
+        params.withClusterManagerTimeout(putComponentTemplateRequest.clusterManagerNodeTimeout());
         if (putComponentTemplateRequest.create()) {
             params.putParam("create", Boolean.TRUE.toString());
         }
@@ -124,7 +124,7 @@ static Request getComponentTemplates(GetComponentTemplatesRequest getComponentTe
         final Request request = new Request(HttpGet.METHOD_NAME, endpoint);
         final RequestConverters.Params params = new RequestConverters.Params();
         params.withLocal(getComponentTemplatesRequest.isLocal());
-        params.withClusterManagerTimeout(getComponentTemplatesRequest.getMasterNodeTimeout());
+        params.withClusterManagerTimeout(getComponentTemplatesRequest.getClusterManagerNodeTimeout());
         request.addParameters(params.asMap());
         return request;
     }
@@ -136,7 +136,7 @@ static Request componentTemplatesExist(ComponentTemplatesExistRequest componentT
         final Request request = new Request(HttpHead.METHOD_NAME, endpoint);
         final RequestConverters.Params params = new RequestConverters.Params();
         params.withLocal(componentTemplatesRequest.isLocal());
-        params.withClusterManagerTimeout(componentTemplatesRequest.getMasterNodeTimeout());
+        params.withClusterManagerTimeout(componentTemplatesRequest.getClusterManagerNodeTimeout());
         request.addParameters(params.asMap());
         return request;
     }
@@ -146,7 +146,7 @@ static Request deleteComponentTemplate(DeleteComponentTemplateRequest deleteComp
         String endpoint = new RequestConverters.EndpointBuilder().addPathPartAsIs("_component_template").addPathPart(name).build();
         Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
         RequestConverters.Params params = new RequestConverters.Params();
-        params.withClusterManagerTimeout(deleteComponentTemplateRequest.masterNodeTimeout());
+        params.withClusterManagerTimeout(deleteComponentTemplateRequest.clusterManagerNodeTimeout());
         request.addParameters(params.asMap());
         return request;
     }