Merge remote-tracking branch 'origin/master' into resize-settings

* origin/master:
  [test] add java packaging test project (elastic#30161)
  Fix macros in changelog (elastic#30269)
  [DOCS] Fixes syskeygen command name
  [ML] Include 3rd party C++ component notices (elastic#30132)
  _cluster/state Skip Test for pre-6.4, not pre-7.0 (elastic#30264)
  Improve docs for disk watermarks (elastic#30249)
  [DOCS] Removes redundant Active Directory realm settings (elastic#30190)
  [DOCS] Removes redundant LDAP realm settings (elastic#30193)
  _cluster/state should always return cluster_uuid (elastic#30143)
  HTML5ify Javadoc for core and test framework (elastic#30234)
  Minor tweaks to reroute documentation (elastic#30246)

Vagrantfile

@@ -337,6 +337,7 @@ export BATS=/project/build/bats
 export BATS_UTILS=/project/build/packaging/bats/utils
 export BATS_TESTS=/project/build/packaging/bats/tests
 export PACKAGING_ARCHIVES=/project/build/packaging/archives
+export PACKAGING_TESTS=/project/build/packaging/tests
 VARS
     cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
 Defaults   env_keep += "ZIP"
@@ -347,6 +348,7 @@ Defaults   env_keep += "BATS"
 Defaults   env_keep += "BATS_UTILS"
 Defaults   env_keep += "BATS_TESTS"
 Defaults   env_keep += "PACKAGING_ARCHIVES"
+Defaults   env_keep += "PACKAGING_TESTS"
 SUDOERS_VARS
     chmod 0440 /etc/sudoers.d/elasticsearch_vars
   SHELL

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -561,8 +561,6 @@ class BuildPlugin implements Plugin<Project> {
              */
             List html4Projects = [
                 ':server',
-                ':libs:elasticsearch-core',
-                ':test:framework',
                 ':x-pack:plugin:core',
             ]
             if (false == html4Projects.contains(project.path)) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy

@@ -41,6 +41,9 @@ class VagrantPropertiesExtension {
     @Input
     Boolean inheritTestUtils
 
+    @Input
+    String testClass
+
     VagrantPropertiesExtension(List<String> availableBoxes) {
         this.boxes = availableBoxes
         this.batsDir = 'src/test/resources/packaging'

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -51,6 +51,7 @@ class VagrantTestPlugin implements Plugin<Project> {
     static List<String> UPGRADE_FROM_ARCHIVES = ['rpm', 'deb']
 
     private static final PACKAGING_CONFIGURATION = 'packaging'
+    private static final PACKAGING_TEST_CONFIGURATION = 'packagingTest'
     private static final BATS = 'bats'
     private static final String BATS_TEST_COMMAND ="cd \$PACKAGING_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
     private static final String PLATFORM_TEST_COMMAND ="rm -rf ~/elasticsearch && rsync -r /elasticsearch/ ~/elasticsearch && cd ~/elasticsearch && ./gradlew test integTest"
@@ -66,6 +67,7 @@ class VagrantTestPlugin implements Plugin<Project> {
 
         // Creates custom configurations for Bats testing files (and associated scripts and archives)
         createPackagingConfiguration(project)
+        project.configurations.create(PACKAGING_TEST_CONFIGURATION)
 
         // Creates all the main Vagrant tasks
         createVagrantTasks(project)
@@ -144,10 +146,12 @@ class VagrantTestPlugin implements Plugin<Project> {
     }
 
     private static void createCleanTask(Project project) {
-        project.tasks.create('clean', Delete.class) {
-            description 'Clean the project build directory'
-            group 'Build'
-            delete project.buildDir
+        if (project.tasks.findByName('clean') == null) {
+            project.tasks.create('clean', Delete.class) {
+                description 'Clean the project build directory'
+                group 'Build'
+                delete project.buildDir
+            }
         }
     }
 
@@ -174,6 +178,18 @@ class VagrantTestPlugin implements Plugin<Project> {
             from project.configurations[PACKAGING_CONFIGURATION]
         }
 
+        File testsDir = new File(packagingDir, 'tests')
+        Copy copyPackagingTests = project.tasks.create('copyPackagingTests', Copy) {
+            into testsDir
+            from project.configurations[PACKAGING_TEST_CONFIGURATION]
+        }
+
+        Task createTestRunnerScript = project.tasks.create('createTestRunnerScript', FileContentsTask) {
+            dependsOn copyPackagingTests
+            file "${testsDir}/run-tests.sh"
+            contents "java -cp \"\$PACKAGING_TESTS/*\" org.junit.runner.JUnitCore ${-> project.extensions.esvagrant.testClass}"
+        }
+
         Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) {
             dependsOn copyPackagingArchives
             file "${archivesDir}/version"
@@ -234,7 +250,8 @@ class VagrantTestPlugin implements Plugin<Project> {
 
         Task vagrantSetUpTask = project.tasks.create('setupPackagingTest')
         vagrantSetUpTask.dependsOn 'vagrantCheckVersion'
-        vagrantSetUpTask.dependsOn copyPackagingArchives, createVersionFile, createUpgradeFromFile, createUpgradeIsOssFile
+        vagrantSetUpTask.dependsOn copyPackagingArchives, copyPackagingTests, createTestRunnerScript
+        vagrantSetUpTask.dependsOn createVersionFile, createUpgradeFromFile, createUpgradeIsOssFile
         vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils
     }
 
@@ -393,20 +410,29 @@ class VagrantTestPlugin implements Plugin<Project> {
                 packagingTest.dependsOn(batsPackagingTest)
             }
 
-            // This task doesn't do anything yet. In the future it will execute a jar containing tests on the vm
-            Task groovyPackagingTest = project.tasks.create("vagrant${boxTask}#groovyPackagingTest")
-            groovyPackagingTest.dependsOn(up)
-            groovyPackagingTest.finalizedBy(halt)
+            Task javaPackagingTest = project.tasks.create("vagrant${boxTask}#javaPackagingTest", VagrantCommandTask) {
+                command 'ssh'
+                boxName box
+                environmentVars vagrantEnvVars
+                dependsOn up, setupPackagingTest
+                finalizedBy halt
+                args '--command', "bash \"\$PACKAGING_TESTS/run-tests.sh\""
+            }
+
+            // todo remove this onlyIf after all packaging tests are consolidated
+            javaPackagingTest.onlyIf {
+                project.extensions.esvagrant.testClass != null
+            }
 
-            TaskExecutionAdapter groovyPackagingReproListener = createReproListener(project, groovyPackagingTest.path)
-            groovyPackagingTest.doFirst {
-                project.gradle.addListener(groovyPackagingReproListener)
+            TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path)
+            javaPackagingTest.doFirst {
+                project.gradle.addListener(javaPackagingReproListener)
             }
-            groovyPackagingTest.doLast {
-                project.gradle.removeListener(groovyPackagingReproListener)
+            javaPackagingTest.doLast {
+                project.gradle.removeListener(javaPackagingReproListener)
             }
             if (project.extensions.esvagrant.boxes.contains(box)) {
-                packagingTest.dependsOn(groovyPackagingTest)
+                packagingTest.dependsOn(javaPackagingTest)
             }
 
             Task platform = project.tasks.create("vagrant${boxTask}#platformTest", VagrantCommandTask) {

distribution/archives/build.gradle

@@ -217,6 +217,24 @@ subprojects {
   }
   check.dependsOn checkNotice
 
+  if (project.name == 'zip' || project.name == 'tar') {
+    task checkMlCppNotice {
+      dependsOn buildDist, checkExtraction
+      onlyIf toolExists
+      doLast {
+        // this is just a small sample from the C++ notices, the idea being that if we've added these lines we've probably added all the required lines
+        final List<String> expectedLines = Arrays.asList("Apache log4cxx", "Boost Software License - Version 1.0 - August 17th, 2003")
+        final Path noticePath = archiveExtractionDir.toPath().resolve("elasticsearch-${VersionProperties.elasticsearch}/modules/x-pack/x-pack-ml/NOTICE.txt")
+        final List<String> actualLines = Files.readAllLines(noticePath)
+        for (final String expectedLine : expectedLines) {
+          if (actualLines.contains(expectedLine) == false) {
+            throw new GradleException("expected [${noticePath}] to contain [${expectedLine}] but it did not")
+          }
+        }
+      }
+    }
+    check.dependsOn checkMlCppNotice
+  }
 }
 
 /*****************************************************************************

docs/CHANGELOG.asciidoc

@@ -1,7 +1,7 @@
 // Use these for links to issue and pulls. Note issues and pulls redirect one to
 // each other on Github, so don't worry too much on using the right prefix.
-// :issue: https://github.com/elastic/elasticsearch/issues/
-// :pull: https://github.com/elastic/elasticsearch/pull/
+:issue: https://github.com/elastic/elasticsearch/issues/
+:pull: https://github.com/elastic/elasticsearch/pull/
 
 = Elasticsearch Release Notes
 

docs/reference/cluster/reroute.asciidoc

@@ -1,13 +1,12 @@
 [[cluster-reroute]]
 == Cluster Reroute
 
-The reroute command allows to explicitly execute a cluster reroute
-allocation command including specific commands. For example, a shard can
-be moved from one node to another explicitly, an allocation can be
-canceled, or an unassigned shard can be explicitly allocated on a
-specific node.
+The reroute command allows for manual changes to the allocation of individual
+shards in the cluster. For example, a shard can be moved from one node to
+another explicitly, an allocation can be cancelled, and an unassigned shard can
+be explicitly allocated to a specific node.
 
-Here is a short example of how a simple reroute API call:
+Here is a short example of a simple reroute API call:
 
 [source,js]
 --------------------------------------------------
@@ -32,59 +31,53 @@ POST /_cluster/reroute
 // CONSOLE
 // TEST[skip:doc tests run with only a single node]
 
-An important aspect to remember is the fact that once when an allocation
-occurs, the cluster will aim at re-balancing its state back to an even
-state. For example, if the allocation includes moving a shard from
-`node1` to `node2`, in an `even` state, then another shard will be moved
-from `node2` to `node1` to even things out.
+It is important to note that that after processing any reroute commands
+Elasticsearch will perform rebalancing as normal (respecting the values of
+settings such as `cluster.routing.rebalance.enable`) in order to remain in a
+balanced state. For example, if the requested allocation includes moving a
+shard from `node1` to `node2` then this may cause a shard to be moved from
+`node2` back to `node1` to even things out.
 
-The cluster can be set to disable allocations, which means that only the
-explicitly allocations will be performed. Obviously, only once all
-commands has been applied, the cluster will aim to be re-balance its
-state.
+The cluster can be set to disable allocations using the
+`cluster.routing.allocation.enable` setting.  If allocations are disabled then
+the only allocations that will be performed are explicit ones given using the
+`reroute` command, and consequent allocations due to rebalancing.
 
-Another option is to run the commands in `dry_run` (as a URI flag, or in
-the request body). This will cause the commands to apply to the current
-cluster state, and return the resulting cluster after the commands (and
-re-balancing) has been applied.
+It is possible to run `reroute` commands in "dry run" mode by using the
+`?dry_run` URI query parameter, or by passing `"dry_run": true` in the request
+body. This will calculate the result of applying the commands to the current
+cluster state, and return the resulting cluster state after the commands (and
+re-balancing) has been applied, but will not actually perform the requested
+changes.
 
-If the `explain` parameter is specified, a detailed explanation of why the
-commands could or could not be executed is returned.
+If the `?explain` URI query parameter is included then a detailed explanation
+of why the commands could or could not be executed is included in the response.
 
 The commands supported are:
 
 `move`::
     Move a started shard from one node to another node. Accepts
     `index` and `shard` for index name and shard number, `from_node` for the
-    node to move the shard `from`, and `to_node` for the node to move the
+    node to move the shard from, and `to_node` for the node to move the
     shard to.
 
 `cancel`::
-    Cancel allocation of a shard (or recovery). Accepts `index`
-    and `shard` for index name and shard number, and `node` for the node to
-    cancel the shard allocation on. It also accepts `allow_primary` flag to
-    explicitly specify that it is allowed to cancel allocation for a primary
-    shard.  This can be used to force resynchronization of existing replicas
-    from the primary shard by cancelling them and allowing them to be
-    reinitialized through the standard reallocation process.
+    Cancel allocation of a shard (or recovery). Accepts `index` and `shard` for
+    index name and shard number, and `node` for the node to cancel the shard
+    allocation on. This can be used to force resynchronization of existing
+    replicas from the primary shard by cancelling them and allowing them to be
+    reinitialized through the standard recovery process. By default only
+    replica shard allocations can be cancelled. If it is necessary to cancel
+    the allocation of a primary shard then the `allow_primary` flag must also
+    be included in the request.
 
 `allocate_replica`::
-    Allocate an unassigned replica shard to a node. Accepts the
-    `index` and `shard` for index name and shard number, and `node` to
-    allocate the shard to. Takes <<modules-cluster,allocation deciders>> into account.
-
-Two more commands are available that allow the allocation of a primary shard
-to a node. These commands should however be used with extreme care, as primary
-shard allocation is usually fully automatically handled by Elasticsearch.
-Reasons why a primary shard cannot be automatically allocated include the following:
-
-- A new index was created but there is no node which satisfies the allocation deciders.
-- An up-to-date shard copy of the data cannot be found on the current data nodes in
-the cluster. To prevent data loss, the system does not automatically promote a stale
-shard copy to primary.
+    Allocate an unassigned replica shard to a node. Accepts `index` and `shard`
+    for index name and shard number, and `node` to allocate the shard to. Takes
+    <<modules-cluster,allocation deciders>> into account.
 
 [float]
-=== Retry failed shards
+=== Retrying failed allocations
 
 The cluster will attempt to allocate a shard a maximum of
 `index.allocation.max_retries` times in a row (defaults to `5`), before giving
@@ -93,36 +86,48 @@ structural problems such as having an analyzer which refers to a stopwords
 file which doesn't exist on all nodes.
 
 Once the problem has been corrected, allocation can be manually retried by
-calling the <<cluster-reroute,`reroute`>> API with `?retry_failed`, which
-will attempt a single retry round for these shards.
+calling the <<cluster-reroute,`reroute`>> API with the `?retry_failed` URI
+query parameter, which will attempt a single retry round for these shards.
 
 [float]
 === Forced allocation on unrecoverable errors
 
+Two more commands are available that allow the allocation of a primary shard to
+a node. These commands should however be used with extreme care, as primary
+shard allocation is usually fully automatically handled by Elasticsearch.
+Reasons why a primary shard cannot be automatically allocated include the
+following:
+
+- A new index was created but there is no node which satisfies the allocation
+  deciders.
+- An up-to-date shard copy of the data cannot be found on the current data
+  nodes in the cluster. To prevent data loss, the system does not automatically
+promote a stale shard copy to primary.
+
 The following two commands are dangerous and may result in data loss. They are
-meant to be used in cases where the original data can not be recovered and the cluster
-administrator accepts the loss. If you have suffered a temporary issue that has been 
-fixed, please see the `retry_failed` flag described above.
+meant to be used in cases where the original data can not be recovered and the
+cluster administrator accepts the loss. If you have suffered a temporary issue
+that can be fixed, please see the `retry_failed` flag described above. To
+emphasise: if these commands are performed and then a node joins the cluster
+that holds a copy of the affected shard then the copy on the newly-joined node
+will be deleted or overwritten.
 
 `allocate_stale_primary`::
     Allocate a primary shard to a node that holds a stale copy. Accepts the
-    `index` and `shard` for index name and shard number, and `node` to
-    allocate the shard to. Using this command may lead to data loss
-    for the provided shard id. If a node which has the good copy of the
-    data rejoins the cluster later on, that data will be overwritten with
-    the data of the stale copy that was forcefully allocated with this
-    command. To ensure that these implications are well-understood,
-    this command requires the special field `accept_data_loss` to be
-    explicitly set to `true` for it to work.
+    `index` and `shard` for index name and shard number, and `node` to allocate
+    the shard to. Using this command may lead to data loss for the provided
+    shard id. If a node which has the good copy of the data rejoins the cluster
+    later on, that data will be deleted or overwritten with the data of the
+    stale copy that was forcefully allocated with this command. To ensure that
+    these implications are well-understood, this command requires the flag
+    `accept_data_loss` to be explicitly set to `true`.
 
 `allocate_empty_primary`::
-    Allocate an empty primary shard to a node. Accepts the
-    `index` and `shard` for index name and shard number, and `node` to
-    allocate the shard to. Using this command leads to a complete loss
-    of all data that was indexed into this shard, if it was previously
-    started. If a node which has a copy of the
-    data rejoins the cluster later on, that data will be deleted!
-    To ensure that these implications are well-understood,
-    this command requires the special field `accept_data_loss` to be
-    explicitly set to `true` for it to work.
+    Allocate an empty primary shard to a node. Accepts the `index` and `shard`
+    for index name and shard number, and `node` to allocate the shard to. Using
+    this command leads to a complete loss of all data that was indexed into
+    this shard, if it was previously started. If a node which has a copy of the
+    data rejoins the cluster later on, that data will be deleted. To ensure
+    that these implications are well-understood, this command requires the flag
+    `accept_data_loss` to be explicitly set to `true`.
 

docs/reference/cluster/state.asciidoc

@@ -15,6 +15,12 @@ of the cluster state (its size when serialized for transmission over
 the network), and the cluster state itself, which can be filtered to
 only retrieve the parts of interest, as described below.
 
+The cluster's `cluster_uuid` is also returned as part of the top-level
+response, in addition to the `metadata` section. added[6.4.0]
+
+NOTE: While the cluster is still forming, it is possible for the `cluster_uuid`
+      to be `_na_` as well as the cluster state's version to be `-1`.
+
 By default, the cluster state request is routed to the master node, to
 ensure that the latest cluster state is returned.   
 For debugging purposes, you can retrieve the cluster state local to a