Merge branch 'master' of github.com:elastic/elasticsearch into feature/fib

Author: Hendrik Muhs

CONTRIBUTING.md

@@ -95,7 +95,7 @@ Contributing to the Elasticsearch codebase
 JDK 10 is required to build Elasticsearch. You must have a JDK 10 installation
 with the environment variable `JAVA_HOME` referencing the path to Java home for
 your JDK 10 installation. By default, tests use the same runtime as `JAVA_HOME`.
-However, since Elasticsearch, supports JDK 8 the build supports compiling with
+However, since Elasticsearch supports JDK 8, the build supports compiling with
 JDK 10 and testing on a JDK 8 runtime; to do this, set `RUNTIME_JAVA_HOME`
 pointing to the Java home of a JDK 8 installation. Note that this mechanism can
 be used to test against other JDKs as well, this is not only limited to JDK 8.
@@ -325,21 +325,19 @@ common configurations in our build and how we use them:
 
 <dl>
 <dt>`compile`</dt><dd>Code that is on the classpath at both compile and
-runtime. If the [`shadow`][shadow-plugin] plugin is applied to the project then
-this code is bundled into the jar produced by the project.</dd>
+runtime.</dd>
 <dt>`runtime`</dt><dd>Code that is not on the classpath at compile time but is
 on the classpath at runtime. We mostly use this configuration to make sure that
 we do not accidentally compile against dependencies of our dependencies also
 known as "transitive" dependencies".</dd>
-<dt>`compileOnly`</dt><dd>Code that is on the classpath at comile time but that
+<dt>`compileOnly`</dt><dd>Code that is on the classpath at compile time but that
 should not be shipped with the project because it is "provided" by the runtime
 somehow. Elasticsearch plugins use this configuration to include dependencies
 that are bundled with Elasticsearch's server.</dd>
-<dt>`shadow`</dt><dd>Only available in projects with the shadow plugin. Code
-that is on the classpath at both compile and runtime but it *not* bundled into
-the jar produced by the project. If you depend on a project with the `shadow`
-plugin then you need to depend on this configuration because it will bring
-along all of the dependencies you need at runtime.</dd>
+<dt>`bundle`</dt><dd>Only available in projects with the shadow plugin,
+dependencies with this configuration are bundled into the jar produced by the
+build. Since IDEs do not understand this configuration we rig them to treat
+dependencies in this configuration as `compile` dependencies.</dd>
 <dt>`testCompile`</dt><dd>Code that is on the classpath for compiling tests
 that are part of this project but not production code. The canonical example
 of this is `junit`.</dd>

benchmarks/build.gradle

@@ -21,10 +21,7 @@ apply plugin: 'elasticsearch.build'
 apply plugin: 'application'
 mainClassName = 'org.openjdk.jmh.Main'
 
-// Not published so no need to assemble
-tasks.remove(assemble)
-build.dependsOn.remove('assemble')
-
+assemble.enabled = false
 archivesBaseName = 'elasticsearch-benchmarks'
 
 test.enabled = false

build.gradle

@@ -16,21 +16,15 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
+import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
-import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
-import org.gradle.util.GradleVersion
-import org.gradle.util.DistributionLocator
-import org.apache.tools.ant.taskdefs.condition.Os
-import org.apache.tools.ant.filters.ReplaceTokens
-
-import java.nio.file.Files
-import java.nio.file.Path
-import java.security.MessageDigest
 
 plugins {
     id 'com.gradle.build-scan' version '1.13.2'
@@ -87,8 +81,15 @@ subprojects {
           }
         }
       }
+      repositories {
+        maven {
+          name = 'localTest'
+          url = "${rootProject.buildDir}/local-test-repo"
+        }
+      }
     }
   }
+
   plugins.withType(BuildPlugin).whenPluginAdded {
     project.licenseFile = project.rootProject.file('licenses/APACHE-LICENSE-2.0.txt')
     project.noticeFile = project.rootProject.file('NOTICE.txt')
@@ -228,6 +229,7 @@ subprojects {
     "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
     "org.elasticsearch.client:test:${version}": ':client:test',
     "org.elasticsearch.client:transport:${version}": ':client:transport',
+    "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip',
@@ -296,7 +298,7 @@ subprojects {
     // org.elasticsearch:elasticsearch must be the last one or all the links for the
     // other packages (e.g org.elasticsearch.client) will point to server rather than
     // their own artifacts.
-    if (project.plugins.hasPlugin(BuildPlugin)) {
+    if (project.plugins.hasPlugin(BuildPlugin) || project.plugins.hasPlugin(PluginBuildPlugin)) {
       String artifactsHost = VersionProperties.elasticsearch.isSnapshot() ? "https://snapshots.elastic.co" : "https://artifacts.elastic.co"
       Closure sortClosure = { a, b -> b.group <=> a.group }
       Closure depJavadocClosure = { shadowed, dep ->
@@ -314,13 +316,6 @@ subprojects {
            */
           project.evaluationDependsOn(upstreamProject.path)
           project.javadoc.source += upstreamProject.javadoc.source
-          /*
-           * Do not add those projects to the javadoc classpath because
-           * we are going to resolve them with their source instead.
-           */
-          project.javadoc.classpath = project.javadoc.classpath.filter { f ->
-            false == upstreamProject.configurations.archives.artifacts.files.files.contains(f)
-          }
           /*
            * Instead we need the upstream project's javadoc classpath so
            * we don't barf on the classes that it references.
@@ -337,16 +332,16 @@ subprojects {
       project.configurations.compile.dependencies
           .findAll()
           .toSorted(sortClosure)
-          .each({ c -> depJavadocClosure(hasShadow, c) })
+          .each({ c -> depJavadocClosure(false, c) })
       project.configurations.compileOnly.dependencies
           .findAll()
           .toSorted(sortClosure)
-          .each({ c -> depJavadocClosure(hasShadow, c) })
+          .each({ c -> depJavadocClosure(false, c) })
       if (hasShadow) {
-        project.configurations.shadow.dependencies
+        project.configurations.bundle.dependencies
             .findAll()
             .toSorted(sortClosure)
-            .each({ c -> depJavadocClosure(false, c) })
+            .each({ c -> depJavadocClosure(true, c) })
       }
     }
   }
@@ -510,30 +505,33 @@ allprojects {
   tasks.cleanEclipse.dependsOn(wipeEclipseSettings)
   // otherwise the eclipse merging is *super confusing*
   tasks.eclipse.dependsOn(cleanEclipse, copyEclipseSettings)
+
+  // work arround https://github.com/gradle/gradle/issues/6582 
+  tasks.eclipseProject.mustRunAfter tasks.cleanEclipseProject
+  tasks.matching { it.name == 'eclipseClasspath' }.all {
+    it.mustRunAfter { tasks.cleanEclipseClasspath }
+  }
+  tasks.matching { it.name == 'eclipseJdt' }.all {
+    it.mustRunAfter { tasks.cleanEclipseJdt }
+  }
+  tasks.copyEclipseSettings.mustRunAfter tasks.wipeEclipseSettings
 }
 
 allprojects {
   /*
    * IntelliJ and Eclipse don't know about the shadow plugin so when we're
-   * in "IntelliJ mode" or "Eclipse mode" add "runtime" dependencies
-   * eveywhere where we see a "shadow" dependency which will cause them to
-   * reference shadowed projects directly rather than rely on the shadowing
-   * to include them. This is the correct thing for it to do because it
-   * doesn't run the jar shadowing at all. This isn't needed for the project
+   * in "IntelliJ mode" or "Eclipse mode" switch "bundle" dependencies into
+   * regular "compile" dependencies. This isn't needed for the project
    * itself because the IDE configuration is done by SourceSets but it is
    * *is* needed for projects that depends on the project doing the shadowing.
    * Without this they won't properly depend on the shadowed project.
    */
   if (isEclipse || isIdea) {
-	configurations.all { Configuration configuration ->
-	  dependencies.all { Dependency dep ->
-		if (dep instanceof ProjectDependency) {
-		  if (dep.getTargetConfiguration() == 'shadow') {
-			configuration.dependencies.add(project.dependencies.project(path: dep.dependencyProject.path, configuration: 'runtime'))
-		  }
-		}
-	  }
-	}
+    project.plugins.withType(ShadowPlugin).whenPluginAdded {
+      project.afterEvaluate {
+        project.configurations.compile.extendsFrom project.configurations.bundle
+      }
+    }
   }
 }
 
@@ -574,76 +572,18 @@ wrapper {
     }
 }
 
-static void assertLinesInFile(final Path path, final List<String> expectedLines) {
-  final List<String> actualLines = Files.readAllLines(path)
-  int line = 0
-  for (final String expectedLine : expectedLines) {
-    final String actualLine = actualLines.get(line)
-    if (expectedLine != actualLine) {
-      throw new GradleException("expected line [${line + 1}] in [${path}] to be [${expectedLine}] but was [${actualLine}]")
-    }
-    line++
-  }
-}
-
-/*
- * Check that all generated JARs have our NOTICE.txt and an appropriate
- * LICENSE.txt in them. We configurate this in gradle but we'd like to
- * be extra paranoid.
- */
-subprojects { project ->
-  project.tasks.withType(Jar).whenTaskAdded { jarTask ->
-    final Task extract = project.task("extract${jarTask.name.capitalize()}", type: LoggedExec) {
-      dependsOn jarTask
-      ext.destination = project.buildDir.toPath().resolve("jar-extracted/${jarTask.name}")
-      commandLine "${->new File(rootProject.compilerJavaHome, 'bin/jar')}",
-          'xf', "${-> jarTask.outputs.files.singleFile}", 'META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'
-      workingDir destination
-      onlyIf {jarTask.enabled}
-      doFirst {
-        project.delete(destination)
-        Files.createDirectories(destination)
-      }
-    }
-
-    final Task checkNotice = project.task("verify${jarTask.name.capitalize()}Notice") {
-      dependsOn extract
-      onlyIf {jarTask.enabled}
-      doLast {
-        final List<String> noticeLines = Files.readAllLines(project.noticeFile.toPath())
-        final Path noticePath = extract.destination.resolve('META-INF/NOTICE.txt')
-        assertLinesInFile(noticePath, noticeLines)
-      }
-    }
-    project.check.dependsOn checkNotice
-
-    final Task checkLicense = project.task("verify${jarTask.name.capitalize()}License") {
-      dependsOn extract
-      onlyIf {jarTask.enabled}
-      doLast {
-        final List<String> licenseLines = Files.readAllLines(project.licenseFile.toPath())
-        final Path licensePath = extract.destination.resolve('META-INF/LICENSE.txt')
-        assertLinesInFile(licensePath, licenseLines)
-      }
-    }
-    project.check.dependsOn checkLicense
-  }
-}
-
 /* Remove assemble/dependenciesInfo on all qa projects because we don't need to publish
  * artifacts for them. */
 gradle.projectsEvaluated {
   subprojects {
     if (project.path.startsWith(':qa')) {
       Task assemble = project.tasks.findByName('assemble')
       if (assemble) {
-        project.tasks.remove(assemble)
-        project.build.dependsOn.remove('assemble')
+        assemble.enabled = false
       }
       Task dependenciesInfo = project.tasks.findByName('dependenciesInfo')
       if (dependenciesInfo) {
-        project.tasks.remove(dependenciesInfo)
-        project.precommit.dependsOn.remove('dependenciesInfo')
+        dependenciesInfo.enabled = false
       }
     }
   }

buildSrc/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 import java.nio.file.Files
+import org.gradle.util.GradleVersion
 
 plugins {
   id 'java-gradle-plugin'
@@ -102,7 +103,6 @@ dependencies {
   compile 'com.netflix.nebula:gradle-info-plugin:3.0.3'
   compile 'org.eclipse.jgit:org.eclipse.jgit:3.2.0.201312181205-r'
   compile 'com.perforce:p4java:2012.3.551082' // THIS IS SUPPOSED TO BE OPTIONAL IN THE FUTURE....
-  compile 'de.thetaphi:forbiddenapis:2.5'
   compile 'org.apache.rat:apache-rat:0.11'
   compile "org.elasticsearch:jna:4.5.1"
   compile 'com.github.jengelman.gradle.plugins:shadow:2.0.4'
@@ -162,11 +162,23 @@ if (project != rootProject) {
   // it's fine as we run them as part of :buildSrc
   test.enabled = false
   task integTest(type: Test) {
+    // integration test requires the local testing repo for example plugin builds
+    dependsOn project.rootProject.allprojects.collect {
+      it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'}
+    }
     exclude "**/*Tests.class"
-    include "**/*IT.class"
     testClassesDirs = sourceSets.test.output.classesDirs
     classpath = sourceSets.test.runtimeClasspath
     inputs.dir(file("src/testKit"))
+    // tell BuildExamplePluginsIT where to find the example plugins
+    systemProperty (
+            'test.build-tools.plugin.examples',
+            files(
+                    project(':example-plugins').subprojects.collect { it.projectDir }
+            ).asPath,
+    )
+    systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo"
+    systemProperty 'test.lucene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1]
   }
   check.dependsOn(integTest)