Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (43 commits)
  ingest: upgraded ingest geoip's geoip2's dependencies.
  [TEST] logging for update by query test #27820
  Use full profile on JDK 10 builds
  Require Gradle 4.3
  Add unreleased v6.1.2 version
  TEST: reduce blob size #testExecuteMultipartUpload
  Check index under the store metadata lock (#27768)
  Upgrade to Lucene 7.2.0. (#27910)
  Fixed test to be up to date with the new database files.
  Use `_refresh` to shrink the version map on inactivity (#27918)
  Make KeyedLock reentrant (#27920)
  Fixes DocStats to not report index size < -1 (#27863)
  Disable TestZenDiscovery in cloud providers integrations test
  ingest: Upgraded the geolite2 databases.
  [Issue-27716]: CONTRIBUTING.md IntelliJ configurations settings are confusing. (#27717)
  [Test] Fix IndicesClientDocumentationIT (#27899)
  Move uid lock into LiveVersionMap (#27905)
  Mute testRetentionPolicyChangeDuringRecovery
  Increase Gradle heap space to 1536m
  Move GlobalCheckpointTracker and remove SequenceNumbersService (#27837)
  ...

CONTRIBUTING.md

@@ -115,11 +115,13 @@ Alternatively, `idea.no.launcher=true` can be set in the
 [`idea.properties`](https://www.jetbrains.com/help/idea/file-idea-properties.html)
 file which can be accessed under Help > Edit Custom Properties (this will require a
 restart of IDEA). For IDEA 2017.3 and above, in addition to the JVM option, you will need to go to
-`Run->Edit Configurations...` and change the value for the `Shorten command line` setting from
+`Run->Edit Configurations->...->Defaults->JUnit` and change the value for the `Shorten command line` setting from
 `user-local default: none` to `classpath file`. You may also need to [remove `ant-javafx.jar` from your
 classpath](https://github.com/elastic/elasticsearch/issues/14348) if that is
 reported as a source of jar hell.
 
+To run an instance of elasticsearch from the source code run `gradle run`
+
 The Elasticsearch codebase makes heavy use of Java `assert`s and the
 test runner requires that assertions be enabled within the JVM. This
 can be accomplished by passing the flag `-ea` to the JVM on startup.

build.gradle

@@ -196,6 +196,7 @@ subprojects {
     "org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
     "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
+    "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
   ]
 
   for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -126,7 +126,7 @@ class BuildPlugin implements Plugin<Project> {
             // enforce Gradle version
             final GradleVersion currentGradleVersion = GradleVersion.current();
 
-            final GradleVersion minGradle = GradleVersion.version('3.3')
+            final GradleVersion minGradle = GradleVersion.version('4.3')
             if (currentGradleVersion < minGradle) {
                 throw new GradleException("${minGradle} or above is required to build elasticsearch")
             }
@@ -408,7 +408,11 @@ class BuildPlugin implements Plugin<Project> {
 
     /** Adds compiler settings to the project */
     static void configureCompile(Project project) {
-        project.ext.compactProfile = 'compact3'
+        if (project.javaVersion < JavaVersion.VERSION_1_10) {
+            project.ext.compactProfile = 'compact3'
+        } else {
+            project.ext.compactProfile = 'full'
+        }
         project.afterEvaluate {
             project.tasks.withType(JavaCompile) {
                 File gradleJavaHome = Jvm.current().javaHome
@@ -444,13 +448,6 @@ class BuildPlugin implements Plugin<Project> {
                     // hack until gradle supports java 9's new "--release" arg
                     assert minimumJava == JavaVersion.VERSION_1_8
                     options.compilerArgs << '--release' << '8'
-                    if (GradleVersion.current().getBaseVersion() < GradleVersion.version("4.1")) {
-                        // this hack is not needed anymore since Gradle 4.1, see https://github.com/gradle/gradle/pull/2474
-                        doFirst {
-                            sourceCompatibility = null
-                            targetCompatibility = null
-                        }
-                    }
                 }
             }
         }

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 6.2.0
-lucene            = 7.2.0-snapshot-7deca62
+lucene            = 7.2.0
 
 # optional dependencies
 spatial4j         = 0.6

client/rest-high-level/build.gradle

@@ -39,6 +39,7 @@ dependencies {
   compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
   compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
+  compile "org.elasticsearch.plugin:rank-eval-client:${version}"
 
   testCompile "org.elasticsearch.client:test:${version}"
   testCompile "org.elasticsearch.test:framework:${version}"
@@ -60,4 +61,4 @@ forbiddenApisMain {
   // specified
   signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
   signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
-}
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -57,6 +57,10 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.cbor.CborXContent;
 import org.elasticsearch.common.xcontent.smile.SmileXContent;
+import org.elasticsearch.index.rankeval.DiscountedCumulativeGain;
+import org.elasticsearch.index.rankeval.EvaluationMetric;
+import org.elasticsearch.index.rankeval.MeanReciprocalRank;
+import org.elasticsearch.index.rankeval.PrecisionAtK;
 import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHits;
@@ -648,7 +652,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(2, namedXContents.size());
+        assertEquals(5, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -658,10 +662,14 @@ public void testProvidedNamedXContents() {
                 categories.put(namedXContent.categoryClass, counter + 1);
             }
         }
-        assertEquals(1, categories.size());
+        assertEquals(2, categories.size());
         assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
         assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
         assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
+        assertEquals(Integer.valueOf(3), categories.get(EvaluationMetric.class));
+        assertTrue(names.contains(PrecisionAtK.NAME));
+        assertTrue(names.contains(DiscountedCumulativeGain.NAME));
+        assertTrue(names.contains(MeanReciprocalRank.NAME));
     }
 
     private static class TrackingActionListener implements ActionListener<Integer> {

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.action.support.ActiveShardCount;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
@@ -86,6 +87,32 @@ public void testDeleteIndex() throws IOException {
             boolean acknowledged = deleteIndexResponse.isAcknowledged(); // <1>
             // end::delete-index-response
             assertTrue(acknowledged);
+        }
+
+        {
+            // tag::delete-index-notfound
+            try {
+                DeleteIndexRequest request = new DeleteIndexRequest("does_not_exist");
+                client.indices().deleteIndex(request);
+            } catch (ElasticsearchException exception) {
+                if (exception.status() == RestStatus.NOT_FOUND) {
+                    // <1>
+                }
+            }
+            // end::delete-index-notfound
+        }
+    }
+
+    public void testDeleteIndexAsync() throws Exception {
+        final RestHighLevelClient client = highLevelClient();
+
+        {
+            CreateIndexResponse createIndexResponse = client.indices().createIndex(new CreateIndexRequest("posts"));
+            assertTrue(createIndexResponse.isAcknowledged());
+        }
+
+        {
+            DeleteIndexRequest request = new DeleteIndexRequest("posts");
 
             // tag::delete-index-execute-async
             client.indices().deleteIndexAsync(request, new ActionListener<DeleteIndexResponse>() {
@@ -100,19 +127,12 @@ public void onFailure(Exception e) {
                 }
             });
             // end::delete-index-execute-async
-        }
 
-        {
-            // tag::delete-index-notfound
-            try {
-                DeleteIndexRequest request = new DeleteIndexRequest("does_not_exist");
-                client.indices().deleteIndex(request);
-            } catch (ElasticsearchException exception) {
-                if (exception.status() == RestStatus.NOT_FOUND) {
-                    // <1>
-                }
-            }
-            // end::delete-index-notfound
+            assertBusy(() -> {
+                // TODO Use Indices Exist API instead once it exists
+                Response response = client.getLowLevelClient().performRequest("HEAD", "posts");
+                assertTrue(RestStatus.NOT_FOUND.getStatus() == response.getStatusLine().getStatusCode());
+            });
         }
     }
 
@@ -174,6 +194,14 @@ public void testCreateIndex() throws IOException {
             // end::create-index-response
             assertTrue(acknowledged);
             assertTrue(shardsAcked);
+        }
+    }
+
+    public void testCreateIndexAsync() throws Exception {
+        final RestHighLevelClient client = highLevelClient();
+
+        {
+            CreateIndexRequest request = new CreateIndexRequest("twitter");
 
             // tag::create-index-execute-async
             client.indices().createIndexAsync(request, new ActionListener<CreateIndexResponse>() {
@@ -188,6 +216,13 @@ public void onFailure(Exception e) {
                 }
             });
             // end::create-index-execute-async
+
+            assertBusy(() -> {
+                // TODO Use Indices Exist API instead once it exists
+                Response response = client.getLowLevelClient().performRequest("HEAD", "twitter");
+                assertTrue(RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode());
+            });
         }
     }
+
 }

client/transport/build.gradle

@@ -32,6 +32,7 @@ dependencies {
   compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
   compile "org.elasticsearch.plugin:percolator-client:${version}"
   compile "org.elasticsearch.plugin:parent-join-client:${version}"
+  compile "org.elasticsearch.plugin:rank-eval-client:${version}"
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
@@ -54,4 +55,4 @@ namingConventions {
   testClass = 'com.carrotsearch.randomizedtesting.RandomizedTest'
   //we don't have integration tests
   skipIntegTestInDisguise = true
-}
+}

core/licenses/lucene-analyzers-common-7.2.0.jar.sha1

@@ -0,0 +1 @@
+4e1b4638fb8b07befc8175880641f821af3e655a

core/licenses/lucene-backward-codecs-7.2.0.jar.sha1

@@ -0,0 +1 @@
+35f5a26abb7fd466749fea7edfedae7897192e95

core/licenses/lucene-core-7.2.0.jar.sha1

@@ -0,0 +1 @@
+f88107aa577ce8edc0a5cee036b485943107a552

core/licenses/lucene-grouping-7.2.0.jar.sha1

@@ -0,0 +1 @@
+1536a1a0fd24d0a8c03cfd45d00a52a88f9f52d1

core/licenses/lucene-highlighter-7.2.0.jar.sha1

@@ -0,0 +1 @@
+afd4093723520b0cdb59852018b545efeefd544a

core/licenses/lucene-join-7.2.0.jar.sha1

@@ -0,0 +1 @@
+16029d54fa9c99b3187b68791b182a1ea4f78e89

core/licenses/lucene-memory-7.2.0.jar.sha1

@@ -0,0 +1 @@
+32f26371224c595f625f061d67fc2edd9c8c836b

core/licenses/lucene-misc-7.2.0.jar.sha1

@@ -0,0 +1 @@
+1067351bfca1fc72ece5cb4a4f219762b097de36

core/licenses/lucene-queries-7.2.0.jar.sha1

@@ -0,0 +1 @@
+e0b41af59bc2baed0315abb04621d62e500d094a

core/licenses/lucene-queryparser-7.2.0.jar.sha1

@@ -0,0 +1 @@
+9a17128e35e5e924cf28c283415d83c7a8935e58

core/licenses/lucene-sandbox-7.2.0.jar.sha1

@@ -0,0 +1 @@
+1fa77169831ec17636357b55bd2c8ca5a97ec7a2

core/licenses/lucene-spatial-7.2.0.jar.sha1

@@ -0,0 +1 @@
+575f7507d526b2692ae461a4df349e90f048ec77

core/licenses/lucene-spatial-extras-7.2.0.jar.sha1

@@ -0,0 +1 @@
+8f6e31d08dc86bb3edeb6ef132f0920941735e15

core/licenses/lucene-spatial3d-7.2.0.jar.sha1

@@ -0,0 +1 @@
+f857630bfafde418e6e3cf748fe8d18f7b771a70

core/licenses/lucene-suggest-7.2.0.jar.sha1

@@ -0,0 +1 @@
+0409ce8d0d7e1203143b5be41aa6dd31d4c1bcf9

core/src/main/java/org/elasticsearch/Version.java

@@ -136,6 +136,8 @@ public class Version implements Comparable<Version> {
     public static final Version V_6_1_0 = new Version(V_6_1_0_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
     public static final int V_6_1_1_ID = 6010199;
     public static final Version V_6_1_1 = new Version(V_6_1_1_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
+    public static final int V_6_1_2_ID = 6010299;
+    public static final Version V_6_1_2 = new Version(V_6_1_2_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
     public static final int V_6_2_0_ID = 6020099;
     public static final Version V_6_2_0 = new Version(V_6_2_0_ID, org.apache.lucene.util.Version.LUCENE_7_2_0);
     public static final Version CURRENT = V_6_2_0;
@@ -153,6 +155,8 @@ public static Version fromId(int id) {
         switch (id) {
             case V_6_2_0_ID:
                 return V_6_2_0;
+            case V_6_1_2_ID:
+                return V_6_1_2;
             case V_6_1_1_ID:
                 return V_6_1_1;
             case V_6_1_0_ID: