Merge branch 'master' into remove-index-thread-pool

* master:
  Remove PipelineExecutionService#executeIndexRequest (elastic#29537)
  Fix overflow error in parsing of long geohashes (elastic#29418)
  Remove unused index.ttl.disable_purge setting (elastic#29527)
  FullClusterRestartIT.testRecovery should wait for all initializing shards
  Build: Fail if any libs depend on non-core libs (elastic#29336)

Author: jasontedor

build.gradle

@@ -231,6 +231,23 @@ subprojects {
     }
   }
 
+  /*
+   * Gradle only resolve project substitutions during dependency resolution but
+   * we sometimes want to do the resolution at other times. This creates a
+   * convenient method we can call to do it.
+   */
+  ext.dependencyToProject = { Dependency dep ->
+    if (dep instanceof ProjectDependency) {
+      return dep.dependencyProject
+    } else {
+      String substitution = projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
+      if (substitution != null) {
+        return findProject(substitution)
+      }
+      return null
+    }
+  }
+
   project.afterEvaluate {
     configurations.all {
       resolutionStrategy.dependencySubstitution { DependencySubstitutions subs ->
@@ -249,11 +266,11 @@ subprojects {
       Closure sortClosure = { a, b -> b.group <=> a.group }
       Closure depJavadocClosure = { dep ->
         if (dep.group != null && dep.group.startsWith('org.elasticsearch')) {
-          String substitution = project.ext.projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
-          if (substitution != null) {
-            project.javadoc.dependsOn substitution + ':javadoc'
+          Project upstreamProject = dependencyToProject(dep)
+          if (upstreamProject != null) {
+            project.javadoc.dependsOn "${upstreamProject.path}:javadoc"
             String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + dep.name.replaceAll('\\.', '/') + '/' + dep.version
-            project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${project.project(substitution).buildDir}/docs/javadoc/"
+            project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${upstreamProject.buildDir}/docs/javadoc/"
           }
         }
       }
@@ -275,17 +292,7 @@ gradle.projectsEvaluated {
     }
     configurations.all {
       dependencies.all { Dependency dep ->
-        Project upstreamProject = null
-        if (dep instanceof ProjectDependency) {
-          upstreamProject = dep.dependencyProject
-        } else {
-          // gradle doesn't apply substitutions until resolve time, so they won't
-          // show up as a ProjectDependency above
-          String substitution = projectSubstitutions.get("${dep.group}:${dep.name}:${dep.version}")
-          if (substitution != null) {
-            upstreamProject = findProject(substitution)
-          }
-        }
+        Project upstreamProject = dependencyToProject(dep)
         if (upstreamProject != null) {
           if (project.path == upstreamProject.path) {
             // TODO: distribution integ tests depend on themselves (!), fix that

docs/reference/mapping/types/geo-point.asciidoc

@@ -92,6 +92,16 @@ format was changed early on to conform to the format used by GeoJSON.
 
 ==================================================
 
+[NOTE]
+A point can be expressed as a http://en.wikipedia.org/wiki/Geohash[geohash].
+Geohashes are https://en.wikipedia.org/wiki/Base32[base32] encoded strings of
+the bits of the latitude and longitude interleaved. Each character in a geohash
+adds additional 5 bits to the precision. So the longer the hash, the more
+precise it is. For the indexing purposed geohashs are translated into
+latitude-longitude pairs. During this process only first 12 characters are
+used, so specifying more than 12 characters in a geohash doesn't increase the
+precision. The 12 characters provide 60 bits, which should reduce a possible
+error to less than 2cm.
 
 [[geo-point-params]]
 ==== Parameters for `geo_point` fields

libs/build.gradle

@@ -0,0 +1,46 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+subprojects {
+    /*
+     * All subprojects are java projects using Elasticsearch's standard build
+     * tools.
+     */
+    apply plugin: 'elasticsearch.build'
+
+    /*
+     * Subprojects may depend on the "core" lib but may not depend on any
+     * other libs. This keeps are dependencies simpler.
+     */
+    project.afterEvaluate {
+        configurations.all { Configuration conf ->
+            dependencies.all { Dependency dep ->
+                Project depProject = dependencyToProject(dep)
+                if (depProject != null
+                        && false == depProject.path.equals(':libs:elasticsearch-core')
+                        && depProject.path.startsWith(':libs')) {
+                    throw new InvalidUserDataException("projects in :libs "
+                            + "may not depend on other projects libs except "
+                            + ":libs:elasticsearch-core but "
+                            + "${project.path} depends on ${depProject.path}")
+                }
+            }
+        }
+    }
+}

libs/elasticsearch-core/build.gradle

@@ -19,7 +19,6 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
  * under the License.
  */
 
-apply plugin: 'elasticsearch.build'
 apply plugin: 'nebula.optional-base'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'

libs/elasticsearch-nio/build.gradle

@@ -19,7 +19,6 @@
 
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 
-apply plugin: 'elasticsearch.build'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
 
@@ -39,7 +38,7 @@ dependencies {
   testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
   testCompile "junit:junit:${versions.junit}"
   testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
-  
+
   if (isEclipse == false || project.path == ":libs:elasticsearch-nio-tests") {
       testCompile("org.elasticsearch.test:framework:${version}") {
         exclude group: 'org.elasticsearch', module: 'elasticsearch-nio'

libs/grok/build.gradle

@@ -19,8 +19,6 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks
  * under the License.
  */
 
-apply plugin: 'elasticsearch.build'
-
 archivesBaseName = 'elasticsearch-grok'
 
 dependencies {

libs/plugin-classloader/build.gradle

@@ -17,8 +17,6 @@
  * under the License.
  */
 
-apply plugin: 'elasticsearch.build'
-
 test.enabled = false
 
 // test depend on ES core...

libs/secure-sm/build.gradle

@@ -19,7 +19,6 @@
 
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 
-apply plugin: 'elasticsearch.build'
 apply plugin: 'nebula.maven-base-publish'
 apply plugin: 'nebula.maven-scm'
 

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -698,6 +698,9 @@ public void testRecovery() throws IOException {
             shouldHaveTranslog = randomBoolean();
 
             indexRandomDocuments(count, true, true, i -> jsonBuilder().startObject().field("field", "value").endObject());
+
+            // make sure all recoveries are done
+            ensureNoInitializingShards();
             // Explicitly flush so we're sure to have a bunch of documents in the Lucene index
             client().performRequest("POST", "/_flush");
             if (shouldHaveTranslog) {

server/src/main/java/org/elasticsearch/common/geo/GeoHashUtils.java

@@ -72,15 +72,19 @@ public static final long longEncode(final double lon, final double lat, final in
     /**
      * Encode from geohash string to the geohash based long format (lon/lat interleaved, 4 least significant bits = level)
      */
-    public static final long longEncode(final String hash) {
-        int level = hash.length()-1;
+    private static long longEncode(final String hash, int length) {
+        int level = length - 1;
         long b;
         long l = 0L;
         for(char c : hash.toCharArray()) {
             b = (long)(BASE_32_STRING.indexOf(c));
             l |= (b<<(level--*5));
+            if (level < 0) {
+                // We cannot handle more than 12 levels
+                break;
+            }
         }
-        return (l<<4)|hash.length();
+        return (l << 4) | length;
     }
 
     /**
@@ -173,6 +177,10 @@ public static final long mortonEncode(final String hash) {
         for(char c : hash.toCharArray()) {
             b = (long)(BASE_32_STRING.indexOf(c));
             l |= (b<<((level--*5) + MORTON_OFFSET));
+            if (level < 0) {
+                // We cannot handle more than 12 levels
+                break;
+            }
         }
         return BitUtil.flipFlop(l);
     }
@@ -200,13 +208,14 @@ private static char encode(int x, int y) {
     public static Rectangle bbox(final String geohash) {
         // bottom left is the coordinate
         GeoPoint bottomLeft = GeoPoint.fromGeohash(geohash);
-        long ghLong = longEncode(geohash);
+        int len = Math.min(12, geohash.length());
+        long ghLong = longEncode(geohash, len);
         // shift away the level
         ghLong >>>= 4;
         // deinterleave and add 1 to lat and lon to get topRight
         long lat = BitUtil.deinterleave(ghLong >>> 1) + 1;
         long lon = BitUtil.deinterleave(ghLong) + 1;
-        GeoPoint topRight = GeoPoint.fromGeohash(BitUtil.interleave((int)lon, (int)lat) << 4 | geohash.length());
+        GeoPoint topRight = GeoPoint.fromGeohash(BitUtil.interleave((int)lon, (int)lat) << 4 | len);
 
         return new Rectangle(bottomLeft.lat(), topRight.lat(), bottomLeft.lon(), topRight.lon());
     }