Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (89 commits)
  Clarify requirements of strict date formats. (#29090)
  Clarify that dates are always rendered as strings. (#29093)
  [Docs] Fix link to Grok patterns (#29088)
  Fix starting on Windows from another drive (#29086)
  Use removeTask instead of finishTask in PersistentTasksClusterService (#29055)
  Added minimal docs for reindex api in java-api docs
  Allow overriding JVM options in Windows service (#29044)
  Clarify how to set compiler and runtime JDKs (#29101)
  Fix Parsing Bug with Update By Query for Stored Scripts (#29039)
  TEST: write ops should execute under shard permit (#28966)
  [DOCS] Add X-Pack upgrade details (#29038)
  Revert "Improve error message for installing plugin (#28298)"
  Docs: HighLevelRestClient#exists (#29073)
  Validate regular expressions in dynamic templates. (#29013)
  [Tests] Fix GetResultTests and DocumentFieldTests failures (#29083)
  Reenable LiveVersionMapTests.testRamBytesUsed on Java 9. (#29063)
  Mute failing GetResultTests and DocumentFieldTests
  [Docs] Fix Java Api index administration usage (#28260)
  Improve error message for installing plugin (#28298)
  Decouple XContentBuilder from BytesReference (#28972)
  ...

TESTING.asciidoc

@@ -270,14 +270,14 @@ The REST tests are run automatically when executing the "./gradlew check" comman
 REST tests use the following command:
 
 ---------------------------------------------------------------------------
-./gradlew :distribution:integ-test-zip:integTest   \
+./gradlew :distribution:archives:integ-test-zip:integTest   \
   -Dtests.class="org.elasticsearch.test.rest.*Yaml*IT"
 ---------------------------------------------------------------------------
 
 A specific test case can be run with
 
 ---------------------------------------------------------------------------
-./gradlew :distribution:integ-test-zip:integTest \
+./gradlew :distribution:archives:integ-test-zip:integTest \
   -Dtests.class="org.elasticsearch.test.rest.*Yaml*IT" \
   -Dtests.method="test {p0=cat.shards/10_basic/Help}"
 ---------------------------------------------------------------------------
@@ -407,7 +407,7 @@ destructive. When working with a single package it's generally faster to run its
 tests in a tighter loop than Gradle provides. In one window:
 
 --------------------------------
-./gradlew :distribution:rpm:assemble
+./gradlew :distribution:packages:rpm:assemble
 --------------------------------
 
 and in another window:

benchmarks/build.gradle

@@ -85,6 +85,10 @@ shadowJar {
     classifier = 'benchmarks'
 }
 
+runShadow {
+    executable = new File(project.runtimeJavaHome, 'bin/java')
+}
+
 // alias the shadowJar and runShadow tasks to abstract from the concrete plugin that we are using and provide a more consistent interface
 task jmhJar(
         dependsOn: shadowJar,

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -140,16 +140,22 @@ class BuildPlugin implements Plugin<Project> {
 
             final GradleVersion minGradle = GradleVersion.version('4.3')
             if (currentGradleVersion < minGradle) {
-                throw new GradleException("${minGradle} or above is required to build elasticsearch")
+                throw new GradleException("${minGradle} or above is required to build Elasticsearch")
             }
 
             // enforce Java version
             if (compilerJavaVersionEnum < minimumCompilerVersion) {
-                throw new GradleException("Java ${minimumCompilerVersion} or above is required to build Elasticsearch")
+                final String message =
+                        "the environment variable JAVA_HOME must be set to a JDK installation directory for Java ${minimumCompilerVersion}" +
+                                " but is [${compilerJavaHome}] corresponding to [${compilerJavaVersionEnum}]"
+                throw new GradleException(message)
             }
 
             if (runtimeJavaVersionEnum < minimumRuntimeVersion) {
-                throw new GradleException("Java ${minimumRuntimeVersion} or above is required to run Elasticsearch")
+                final String message =
+                        "the environment variable RUNTIME_JAVA_HOME must be set to a JDK installation directory for Java ${minimumRuntimeVersion}" +
+                                " but is [${runtimeJavaHome}] corresponding to [${runtimeJavaVersionEnum}]"
+                throw new GradleException(message)
             }
 
             project.rootProject.ext.compilerJavaHome = compilerJavaHome
@@ -558,7 +564,7 @@ class BuildPlugin implements Plugin<Project> {
         return {
             jvm "${project.runtimeJavaHome}/bin/java"
             parallelism System.getProperty('tests.jvms', 'auto')
-            ifNoTests 'fail'
+            ifNoTests System.getProperty('tests.ifNoTests', 'fail')
             onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true
 
@@ -582,8 +588,6 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.task', path
             systemProperty 'tests.security.manager', 'true'
             systemProperty 'jna.nosys', 'true'
-            // default test sysprop values
-            systemProperty 'tests.ifNoTests', 'fail'
             // TODO: remove setting logging level via system property
             systemProperty 'tests.logger.level', 'WARN'
             for (Map.Entry<String, String> property : System.properties.entrySet()) {

buildSrc/src/main/resources/forbidden/es-all-signatures.txt

@@ -48,3 +48,5 @@ java.nio.channels.SocketChannel#connect(java.net.SocketAddress)
 # org.elasticsearch.common.Booleans#parseBoolean(java.lang.String) directly on the string.
 @defaultMessage use org.elasticsearch.common.Booleans#parseBoolean(java.lang.String)
 java.lang.Boolean#getBoolean(java.lang.String)
+
+org.apache.lucene.util.IOUtils @ use @org.elasticsearch.core.internal.io instead

buildSrc/src/main/resources/forbidden/es-server-signatures.txt

@@ -29,8 +29,6 @@ java.util.concurrent.Executors#privilegedThreadFactory()
 java.lang.Character#codePointBefore(char[],int) @ Implicit start offset is error-prone when the char[] is a buffer and the first chars are random chars
 java.lang.Character#codePointAt(char[],int) @ Implicit end offset is error-prone when the char[] is a buffer and the last chars are random chars
 
-java.io.StringReader#<init>(java.lang.String) @ Use FastStringReader instead
-
 @defaultMessage Reference management is tricky, leave it to SearcherManager
 org.apache.lucene.index.IndexReader#decRef()
 org.apache.lucene.index.IndexReader#incRef()
@@ -135,3 +133,17 @@ java.time.OffsetDateTime#withYear(int)
 java.time.zone.ZoneRules#getStandardOffset(java.time.Instant)
 java.time.zone.ZoneRules#getDaylightSavings(java.time.Instant)
 java.time.zone.ZoneRules#isDaylightSavings(java.time.Instant)
+
+@defaultMessage Use logger methods with non-Object parameter
+org.apache.logging.log4j.Logger#trace(java.lang.Object)
+org.apache.logging.log4j.Logger#trace(java.lang.Object, java.lang.Throwable)
+org.apache.logging.log4j.Logger#debug(java.lang.Object)
+org.apache.logging.log4j.Logger#debug(java.lang.Object, java.lang.Throwable)
+org.apache.logging.log4j.Logger#info(java.lang.Object)
+org.apache.logging.log4j.Logger#info(java.lang.Object, java.lang.Throwable)
+org.apache.logging.log4j.Logger#warn(java.lang.Object)
+org.apache.logging.log4j.Logger#warn(java.lang.Object, java.lang.Throwable)
+org.apache.logging.log4j.Logger#error(java.lang.Object)
+org.apache.logging.log4j.Logger#error(java.lang.Object, java.lang.Throwable)
+org.apache.logging.log4j.Logger#fatal(java.lang.Object)
+org.apache.logging.log4j.Logger#fatal(java.lang.Object, java.lang.Throwable)

buildSrc/src/main/resources/plugin-descriptor.properties

@@ -43,6 +43,3 @@ extended.plugins=${extendedPlugins}
 #
 # 'has.native.controller': whether or not the plugin has a native controller
 has.native.controller=${hasNativeController}
-#
-# 'requires.keystore': whether or not the plugin needs the elasticsearch keystore be created
-requires.keystore=${requiresKeystore}

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -330,7 +330,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                 }
                 metadata.endObject();
 
-                BytesRef metadataSource = metadata.bytes().toBytesRef();
+                BytesRef metadataSource = BytesReference.bytes(metadata).toBytesRef();
                 content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
                 content.write(separator);
             }
@@ -345,7 +345,7 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
                     LoggingDeprecationHandler.INSTANCE, indexSource, indexXContentType)) {
                     try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                         builder.copyCurrentStructure(parser);
-                        source = builder.bytes().toBytesRef();
+                        source = BytesReference.bytes(builder).toBytesRef();
                     }
                 }
             } else if (opType == DocWriteRequest.OpType.UPDATE) {

client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java

@@ -617,7 +617,8 @@ public void testBulk() throws IOException {
                 bulkRequest.add(deleteRequest);
 
             } else {
-                BytesReference source = XContentBuilder.builder(xContentType.xContent()).startObject().field("id", i).endObject().bytes();
+                BytesReference source = BytesReference.bytes(XContentBuilder.builder(xContentType.xContent())
+                        .startObject().field("id", i).endObject());
                 if (opType == DocWriteRequest.OpType.INDEX) {
                     IndexRequest indexRequest = new IndexRequest("index", "test", id).source(source, xContentType);
                     if (erroneous) {

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -55,6 +55,7 @@
 import org.elasticsearch.action.search.ShardSearchFailure;
 import org.elasticsearch.cluster.ClusterName;
 import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -272,7 +273,7 @@ private static HttpEntity createBinaryEntity(XContentBuilder xContentBuilder, Co
             builder.startObject();
             builder.field("field", "value");
             builder.endObject();
-            return new ByteArrayEntity(builder.bytes().toBytesRef().bytes, contentType);
+            return new ByteArrayEntity(BytesReference.bytes(builder).toBytesRef().bytes, contentType);
         }
     }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
@@ -478,7 +479,7 @@ public void testSearchScroll() throws Exception {
 
         for (int i = 0; i < 100; i++) {
             XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject();
-            HttpEntity entity = new NStringEntity(builder.string(), ContentType.APPLICATION_JSON);
+            HttpEntity entity = new NStringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON);
             client().performRequest(HttpPut.METHOD_NAME, "test/type1/" + Integer.toString(i), Collections.emptyMap(), entity);
         }
         client().performRequest(HttpPost.METHOD_NAME, "/test/_refresh");

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java

@@ -266,13 +266,13 @@ public void testUpdate() throws Exception {
             assertSame(indexResponse.status(), RestStatus.CREATED);
 
             XContentType xContentType = XContentType.JSON;
-            String script = XContentBuilder.builder(xContentType.xContent())
+            String script = Strings.toString(XContentBuilder.builder(xContentType.xContent())
                     .startObject()
                         .startObject("script")
                             .field("lang", "painless")
                             .field("code", "ctx._source.field += params.count")
                         .endObject()
-                    .endObject().string();
+                    .endObject());
             HttpEntity body = new NStringEntity(script, ContentType.create(xContentType.mediaType()));
             Response response = client().performRequest(HttpPost.METHOD_NAME, "/_scripts/increment-field", emptyMap(), body);
             assertEquals(response.getStatusLine().getStatusCode(), RestStatus.OK.getStatus());
@@ -932,6 +932,49 @@ public void onFailure(Exception e) {
         }
     }
 
+    public void testExists() throws Exception {
+        RestHighLevelClient client = highLevelClient();
+        // tag::exists-request
+        GetRequest getRequest = new GetRequest(
+            "posts", // <1>
+            "doc",   // <2>
+            "1");    // <3>
+        getRequest.fetchSourceContext(new FetchSourceContext(false)); // <4>
+        getRequest.storedFields("_none_");                            // <5>
+        // end::exists-request
+        {
+            // tag::exists-execute
+            boolean exists = client.exists(getRequest);
+            // end::exists-execute
+            assertFalse(exists);
+        }
+        {
+            // tag::exists-execute-listener
+            ActionListener<Boolean> listener = new ActionListener<Boolean>() {
+                @Override
+                public void onResponse(Boolean exists) {
+                    // <1>
+                }
+
+                @Override
+                public void onFailure(Exception e) {
+                    // <2>
+                }
+            };
+            // end::exists-execute-listener
+
+            // Replace the empty listener by a blocking listener in test
+            final CountDownLatch latch = new CountDownLatch(1);
+            listener = new LatchedActionListener<>(listener, latch);
+
+            // tag::exists-execute-async
+            client.existsAsync(getRequest, listener); // <1>
+            // end::exists-execute-async
+
+            assertTrue(latch.await(30L, TimeUnit.SECONDS));
+        }
+    }
+
     public void testBulkProcessor() throws InterruptedException {
         RestHighLevelClient client = highLevelClient();
         {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MigrationDocumentationIT.java

@@ -33,6 +33,7 @@
 import org.apache.http.entity.ContentType;
 import org.apache.http.nio.entity.NStringEntity;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentHelper;
@@ -75,7 +76,7 @@ public void testCreateIndex() throws IOException {
                     .put(SETTING_NUMBER_OF_REPLICAS, 0)
                     .build();
 
-            String payload = XContentFactory.jsonBuilder() // <2>
+            String payload = Strings.toString(XContentFactory.jsonBuilder() // <2>
                     .startObject()
                         .startObject("settings") // <3>
                             .value(indexSettings)
@@ -89,7 +90,7 @@ public void testCreateIndex() throws IOException {
                                 .endObject()
                             .endObject()
                         .endObject()
-                    .endObject().string();
+                    .endObject());
 
             HttpEntity entity = new NStringEntity(payload, ContentType.APPLICATION_JSON); // <5>
 

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MainDocumentationIT.java → client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java

@@ -31,20 +31,10 @@
 import java.io.IOException;
 
 /**
- * This class is used to generate the Java Main API documentation.
- * You need to wrap your code between two tags like:
- * // tag::example[]
- * // end::example[]
- *
- * Where example is your tag name.
- *
- * Then in the documentation, you can extract what is between tag and end tags with
- * ["source","java",subs="attributes,callouts,macros"]
- * --------------------------------------------------
- * include-tagged::{doc-tests}/MainDocumentationIT.java[example]
- * --------------------------------------------------
+ * Documentation for miscellaneous APIs in the high level java client.
+ * Code wrapped in {@code tag} and {@code end} tags is included in the docs.
  */
-public class MainDocumentationIT extends ESRestHighLevelClientTestCase {
+public class MiscellaneousDocumentationIT extends ESRestHighLevelClientTestCase {
 
     public void testMain() throws IOException {
         RestHighLevelClient client = highLevelClient();
@@ -68,6 +58,14 @@ public void testMain() throws IOException {
         }
     }
 
+    public void testPing() throws IOException {
+        RestHighLevelClient client = highLevelClient();
+        //tag::ping-execute
+        boolean response = client.ping();
+        //end::ping-execute
+        assertTrue(response);
+    }
+
     public void testInitializationFromClientBuilder() throws IOException {
         //tag::rest-high-level-client-init
         RestHighLevelClient client = new RestHighLevelClient(

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/QueryDSLDocumentationTests.java

@@ -76,6 +76,7 @@
 import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
 import static org.elasticsearch.index.query.QueryBuilders.typeQuery;
 import static org.elasticsearch.index.query.QueryBuilders.wildcardQuery;
+import static org.elasticsearch.index.query.QueryBuilders.wrapperQuery;
 import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.exponentialDecayFunction;
 import static org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders.randomFunction;
 
@@ -449,4 +450,11 @@ public void testWildcard() {
                 "k?mch*");                                           // <2>
         // end::wildcard
     }
+
+    public void testWrapper() {
+        // tag::wrapper
+        String query = "{\"term\": {\"user\": \"kimchy\"}}"; // <1>
+        wrapperQuery(query);
+        // end::wrapper
+    }
 }

distribution/archives/build.gradle

@@ -21,6 +21,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.apache.tools.ant.filters.FixCrLfFilter
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.EmptyDirTask
+import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.MavenFilteringHack
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 
@@ -125,19 +126,39 @@ subprojects {
   artifacts {
     'default' buildDist
   }
+
+  // sanity checks if a archives can be extracted
+  File extractionDir = new File(buildDir, 'extracted')
+  task testExtraction(type: LoggedExec) {
+    dependsOn buildDist
+    doFirst {
+      project.delete(extractionDir)
+      extractionDir.mkdirs()
+    }
+  }
+  if (project.name.contains('zip')) {
+    testExtraction {
+      onlyIf { new File('/bin/unzip').exists() || new File('/usr/bin/unzip').exists() || new File('/usr/local/bin/unzip').exists() }
+      commandLine 'unzip', "${-> buildDist.outputs.files.singleFile}", '-d', extractionDir
+    }
+  } else { // tar
+    testExtraction {
+      onlyIf { new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }
+      commandLine 'tar', '-xvzf', "${-> buildDist.outputs.files.singleFile}", '-C', extractionDir
+    }
+  }
+  check.dependsOn testExtraction
 }
 
 /*****************************************************************************
  *                            Rest test config                               *
  *****************************************************************************/
-subprojects {
+configure(subprojects.findAll { it.name == 'integ-test-zip' }) {
   apply plugin: 'elasticsearch.standalone-rest-test'
   apply plugin: 'elasticsearch.rest-test'
 
-  if (project.name == 'integ-test-zip') {
-    integTest {
-      includePackaged true
-    }
+  integTest {
+    includePackaged true
   }
 
   integTestCluster {