Merge branch 'master' into build-docker-images

* master: (133 commits)
  SNAPSHOT: Increase Timeout to Stabilize Test (elastic#36294)
  Fix get certificates HLRC API (elastic#36198)
  Avoid shutting down the only master (elastic#36272)
  Fix typo in comment
  Fix total hits serialization of the search response (elastic#36290)
  Fix FullClusterRestartIT#testRollupIDSchemeAfterRestart
  Mute FullClusterRestartIT#testRollupIDSchemeAfterRestart as we await a fix.
  [Docs] Add Profile API limitations (elastic#36252)
  Make sure test don't use Math.random for reproducability (elastic#36241)
  Fix compilation
  ingest: support default pipeline through an alias (elastic#36231)
  Zen2: Rename tombstones to exclusions (elastic#36226)
  [Zen2] Hide not recovered state (elastic#36224)
  Test: mute testDataNodeRestartWithBusyMasterDuringSnapshot
  Test: mute testSnapshotAndRestoreWithNested
  Revert "Test: mute failing mtermvector rest test"
  Test: mute failing mtermvector rest test
  add version 6.5.3 (elastic#36268)
  Make hits.total an object in the search response (elastic#35849)
  [DOCS] Fixes broken link in execute watch
  ...

Author: jasontedor

benchmarks/build.gradle

@@ -33,7 +33,7 @@ dependencies {
         exclude group: 'net.sf.jopt-simple', module: 'jopt-simple'
     }
     compile "org.openjdk.jmh:jmh-core:$versions.jmh"
-    compile "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
+    annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
     // Dependencies of JMH
     runtime 'net.sf.jopt-simple:jopt-simple:4.6'
     runtime 'org.apache.commons:commons-math3:3.2'

build.gradle

@@ -24,13 +24,16 @@ import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
+import org.gradle.api.tasks.options.Option
+import  org.gradle.util.GradleVersion
+import  org.gradle.util.DistributionLocator
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 
 import java.util.function.Predicate
 
 plugins {
-    id 'com.gradle.build-scan' version '1.13.2'
+    id 'com.gradle.build-scan' version '2.0.2'
     id 'base'
 }
 if (properties.get("org.elasticsearch.acceptScanTOS", "false") == "true") {
@@ -527,7 +530,7 @@ allprojects {
 class Run extends DefaultTask {
   boolean debug = false
 
-  @org.gradle.api.internal.tasks.options.Option(
+  @Option(
         option = "debug-jvm",
         description = "Enable debugging configuration, to allow attaching a debugger to elasticsearch."
   )

buildSrc/build.gradle

@@ -147,10 +147,8 @@ if (project == rootProject) {
       mavenLocal()
     }
   }
-  test {
-    include "**/*Tests.class"
-    exclude "**/*IT.class"
-  }
+  // only run tests as build-tools
+  test.enabled = false
 }
 
 /*****************************************************************************
@@ -180,9 +178,17 @@ if (project != rootProject) {
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
 
-  // tests can't  be run with randomized test runner
-  // it's fine as we run them as part of :buildSrc
-  test.enabled = false
+  test {
+    include "**/*Tests.class"
+    exclude "**/*IT.class"
+    // The test task is configured to runtimeJava version, but build-tools doesn't support all of them, so test
+    // with compiler instead on the ones that are too old.
+    if (project.runtimeJavaVersion <= JavaVersion.VERSION_1_10) {
+      jvm = "${project.compilerJavaHome}/bin/java"
+    }
+  }
+
+  // This can't be an RandomizedTestingTask because we can't yet reference it
   task integTest(type: Test) {
     // integration test requires the local testing repo for example plugin builds
     dependsOn project.rootProject.allprojects.collect {

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/TestProgressLogger.groovy

@@ -125,7 +125,7 @@ class TestProgressLogger implements AggregatedEventListener {
 
     @Subscribe
     void onTestResult(AggregatedTestResultEvent e) throws IOException {
-        final String statusMessage
+        String statusMessage
         testsCompleted++
         switch (e.status) {
         case ERROR:

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -69,7 +69,7 @@ class BuildPlugin implements Plugin<Project> {
                 + 'elasticsearch.standalone-rest-test, and elasticsearch.build '
                 + 'are mutually exclusive')
         }
-        final String minimumGradleVersion
+        String minimumGradleVersion = null
         InputStream is = getClass().getResourceAsStream("/minimumGradleVersion")
         try { minimumGradleVersion = IOUtils.toString(is, StandardCharsets.UTF_8.toString()) } finally { is.close() }
         if (GradleVersion.current() < GradleVersion.version(minimumGradleVersion.trim())) {
@@ -320,7 +320,7 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     private static String findCompilerJavaHome() {
-        final String compilerJavaHome = System.getenv('JAVA_HOME')
+        String compilerJavaHome = System.getenv('JAVA_HOME')
         final String compilerJavaProperty = System.getProperty('compiler.java')
         if (compilerJavaProperty != null) {
             compilerJavaHome = findJavaHome(compilerJavaProperty)
@@ -857,13 +857,26 @@ class BuildPlugin implements Plugin<Project> {
     }
 
     static void applyCommonTestConfig(Project project) {
-        project.tasks.withType(RandomizedTestingTask) {
+        project.tasks.withType(RandomizedTestingTask) {task ->
             jvm "${project.runtimeJavaHome}/bin/java"
             parallelism System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel)
             ifNoTests System.getProperty('tests.ifNoTests', 'fail')
             onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true
 
+            // Make sure all test tasks are configured properly
+            if (name != "test") {
+                project.tasks.matching { it.name == "test"}.all { testTask ->
+                    task.testClassesDirs = testTask.testClassesDirs
+                    task.classpath = testTask.classpath
+                    task.shouldRunAfter testTask
+                }
+            }
+            // no loose ends: check has to depend on all test tasks
+            project.tasks.matching {it.name == "check"}.all {
+                dependsOn(task)
+            }
+
             // TODO: why are we not passing maxmemory to junit4?
             jvmArg '-Xmx' + System.getProperty('tests.heap.size', '512m')
             jvmArg '-Xms' + System.getProperty('tests.heap.size', '512m')

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -129,7 +129,6 @@ public class PluginBuildPlugin extends BuildPlugin {
         RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
         integTest.mustRunAfter(project.precommit, project.test)
         project.integTestCluster.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
-        project.check.dependsOn(integTest)
     }
 
     /**

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -190,8 +190,7 @@ class PrecommitTasks {
         Task checkstyleTask = project.tasks.create('checkstyle')
         // Apply the checkstyle plugin to create `checkstyleMain` and `checkstyleTest`. It only
         // creates them if there is main or test code to check and it makes `check` depend
-        // on them. But we want `precommit` to depend on `checkstyle` which depends on them so
-        // we have to swap them.
+        // on them. We also want `precommit` to depend on `checkstyle`.
         project.pluginManager.apply('checkstyle')
         project.checkstyle {
             config = project.resources.text.fromFile(checkstyleConf, 'UTF-8')
@@ -202,7 +201,6 @@ class PrecommitTasks {
         }
 
         project.tasks.withType(Checkstyle) { task ->
-            project.tasks[JavaBasePlugin.CHECK_TASK_NAME].dependsOn.remove(task)
             checkstyleTask.dependsOn(task)
             task.dependsOn(copyCheckstyleConf)
             task.inputs.file(checkstyleSuppressions)

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -111,8 +111,8 @@ class ClusterFormationTasks {
         for (int i = 0; i < config.numNodes; i++) {
             // we start N nodes and out of these N nodes there might be M bwc nodes.
             // for each of those nodes we might have a different configuration
-            final Configuration distro
-            final String elasticsearchVersion
+            Configuration distro
+            String elasticsearchVersion
             if (i < config.numBwcNodes) {
                 elasticsearchVersion = config.bwcVersion.toString()
                 if (project.bwcVersions.unreleased.contains(config.bwcVersion)) {
@@ -595,7 +595,7 @@ class ClusterFormationTasks {
     }
 
     static Task configureInstallPluginTask(String name, Project project, Task setup, NodeInfo node, String pluginName, String prefix) {
-        final FileCollection pluginZip;
+        FileCollection pluginZip;
         if (node.nodeVersion != Version.fromString(VersionProperties.elasticsearch)) {
             pluginZip = project.configurations.getByName(pluginBwcConfigurationName(prefix, pluginName))
         } else {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -25,12 +25,12 @@ import org.gradle.api.DefaultTask
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.execution.TaskExecutionAdapter
-import org.gradle.api.internal.tasks.options.Option
 import org.gradle.api.provider.Property
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskState
+import org.gradle.api.tasks.options.Option
 import org.gradle.plugins.ide.idea.IdeaPlugin
 
 import java.nio.charset.StandardCharsets
@@ -53,7 +53,7 @@ public class RestIntegTestTask extends DefaultTask {
 
     /** Flag indicating whether the rest tests in the rest spec should be run. */
     @Input
-    Property<Boolean> includePackaged = project.objects.property(Boolean)
+    Boolean includePackaged = false
 
     public RestIntegTestTask() {
         runner = project.tasks.create("${name}Runner", RandomizedTestingTask.class)
@@ -109,7 +109,7 @@ public class RestIntegTestTask extends DefaultTask {
         }
 
         // copy the rest spec/tests into the test resources
-        Task copyRestSpec = createCopyRestSpecTask(project, includePackaged)
+        Task copyRestSpec = createCopyRestSpecTask()
         runner.dependsOn(copyRestSpec)
 
         // this must run after all projects have been configured, so we know any project
@@ -130,7 +130,7 @@ public class RestIntegTestTask extends DefaultTask {
 
     /** Sets the includePackaged property */
     public void includePackaged(boolean include) {
-        includePackaged.set(include)
+        includePackaged = include
     }
 
     @Option(
@@ -215,7 +215,7 @@ public class RestIntegTestTask extends DefaultTask {
      * @param project The project to add the copy task to
      * @param includePackagedTests true if the packaged tests should be copied, false otherwise
      */
-    static Task createCopyRestSpecTask(Project project, Provider<Boolean> includePackagedTests) {
+    Task createCopyRestSpecTask() {
         project.configurations {
             restSpec
         }
@@ -237,7 +237,7 @@ public class RestIntegTestTask extends DefaultTask {
         project.afterEvaluate {
             copyRestSpec.from({ project.zipTree(project.configurations.restSpec.singleFile) }) {
                 include 'rest-api-spec/api/**'
-                if (includePackagedTests.get()) {
+                if (includePackaged) {
                     include 'rest-api-spec/test/**'
                 }
             }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RunTask.groovy

@@ -2,7 +2,7 @@ package org.elasticsearch.gradle.test
 
 import org.gradle.api.DefaultTask
 import org.gradle.api.Task
-import org.gradle.api.internal.tasks.options.Option
+import org.gradle.api.tasks.options.Option
 import org.gradle.util.ConfigureUtil
 
 public class RunTask extends DefaultTask {