Merge remote-tracking branch 'elastic/master' into allowed-index-stor…

…e-types

* elastic/master: (89 commits)
  Fix assertion in AbstractSimpleTransportTestCase (elastic#32991)
  [DOC] Splits role mapping APIs into separate pages (elastic#32797)
  HLRC: ML Close Job (elastic#32943)
  Generalize remote license checker (elastic#32971)
  Trim translog when safe commit advanced (elastic#32967)
  Fix an inaccuracy in the dynamic templates documentation. (elastic#32890)
  Logging: Use settings when building daemon threads (elastic#32751)
  All Translog inner closes should happen after tragedy exception is set (elastic#32674)
  HLREST: AwaitsFix ML Test
  Pass DiscoveryNode to initiateChannel (elastic#32958)
  Add mzn and dz to unsupported locales (elastic#32957)
  Use settings from the context in BootstrapChecks (elastic#32908)
  Update docs for node specifications (elastic#30468)
  HLRC: Forbid all Elasticsearch logging infra (elastic#32784)
  Only configure publishing if it's applied externally (elastic#32351)
  Fixes libs:dissect when in eclipse
  Protect ScriptedMetricIT test cases against failures on 0-doc shards (elastic#32959) (elastic#32968)
  [Kerberos] Add documentation for Kerberos realm (elastic#32662)
  Watcher: Properly find next valid date in cron expressions (elastic#32734)
  Fix some small issues in the getting started docs (elastic#30346)
  ...

build.gradle

@@ -87,8 +87,15 @@ subprojects {
           }
         }
       }
+      repositories {
+        maven {
+          name = 'localTest'
+          url = "${rootProject.buildDir}/local-test-repo"
+        }
+      }
     }
   }
+
   plugins.withType(BuildPlugin).whenPluginAdded {
     project.licenseFile = project.rootProject.file('licenses/APACHE-LICENSE-2.0.txt')
     project.noticeFile = project.rootProject.file('NOTICE.txt')
@@ -228,6 +235,7 @@ subprojects {
     "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',
     "org.elasticsearch.client:test:${version}": ':client:test',
     "org.elasticsearch.client:transport:${version}": ':client:transport',
+    "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
     "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip',

buildSrc/build.gradle

@@ -162,11 +162,24 @@ if (project != rootProject) {
   // it's fine as we run them as part of :buildSrc
   test.enabled = false
   task integTest(type: Test) {
+    // integration test requires the local testing repo for example plugin builds
+    dependsOn project.rootProject.allprojects.collect {
+      it.tasks.matching { it.name == 'publishNebulaPublicationToLocalTestRepository'}
+    }
     exclude "**/*Tests.class"
     include "**/*IT.class"
     testClassesDirs = sourceSets.test.output.classesDirs
     classpath = sourceSets.test.runtimeClasspath
     inputs.dir(file("src/testKit"))
+    // tell BuildExamplePluginsIT where to find the example plugins
+    systemProperty (
+            'test.build-tools.plugin.examples',
+            files(
+                    project(':example-plugins').subprojects.collect { it.projectDir }
+            ).asPath,
+    )
+    systemProperty 'test.local-test-repo-path', "${rootProject.buildDir}/local-test-repo"
+    systemProperty 'test.lucene-snapshot-revision', (versions.lucene =~ /\w+-snapshot-([a-z0-9]+)/)[0][1]
   }
   check.dependsOn(integTest)
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -528,11 +528,12 @@ class BuildPlugin implements Plugin<Project> {
         project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom generatePOMTask ->
             // The GenerateMavenPom task is aggressive about setting the destination, instead of fighting it,
             // just make a copy.
+            generatePOMTask.ext.pomFileName = "${project.archivesBaseName}-${project.version}.pom"
             doLast {
                 project.copy {
                     from generatePOMTask.destination
                     into "${project.buildDir}/distributions"
-                    rename { "${project.archivesBaseName}-${project.version}.pom" }
+                    rename { generatePOMTask.ext.pomFileName }
                 }
             }
             // build poms with assemble (if the assemble task exists)
@@ -554,7 +555,7 @@ class BuildPlugin implements Plugin<Project> {
                 project.publishing {
                     publications {
                         nebula(MavenPublication) {
-                            artifact project.tasks.shadowJar
+                            artifacts = [ project.tasks.shadowJar ]
                             artifactId = project.archivesBaseName
                             /*
                             * Configure the pom to include the "shadow" as compile dependencies
@@ -584,7 +585,6 @@ class BuildPlugin implements Plugin<Project> {
                 }
             }
         }
-
     }
 
     /** Adds compiler settings to the project */
@@ -799,6 +799,8 @@ class BuildPlugin implements Plugin<Project> {
             systemProperty 'tests.task', path
             systemProperty 'tests.security.manager', 'true'
             systemProperty 'jna.nosys', 'true'
+            // TODO: remove this deprecation compatibility setting for 7.0
+            systemProperty 'es.aggregations.enable_scripted_metric_agg_param', 'false'
             systemProperty 'compiler.java', project.ext.compilerJavaVersion.getMajorVersion()
             if (project.ext.inFipsJvm) {
                 systemProperty 'runtime.java', project.ext.runtimeJavaVersion.getMajorVersion() + "FIPS"

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -19,27 +19,21 @@
 package org.elasticsearch.gradle.plugin
 
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
-import nebula.plugin.info.scm.ScmInfoPlugin
+import nebula.plugin.publishing.maven.MavenScmPlugin
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.NoticeTask
 import org.elasticsearch.gradle.test.RestIntegTestTask
 import org.elasticsearch.gradle.test.RunTask
-import org.gradle.api.InvalidUserDataException
-import org.gradle.api.JavaVersion
 import org.gradle.api.Project
-import org.gradle.api.Task
-import org.gradle.api.XmlProvider
 import org.gradle.api.publish.maven.MavenPublication
 import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
+import org.gradle.api.publish.maven.tasks.GenerateMavenPom
 import org.gradle.api.tasks.SourceSet
 import org.gradle.api.tasks.bundling.Zip
+import org.gradle.jvm.tasks.Jar
 
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.StandardCopyOption
 import java.util.regex.Matcher
 import java.util.regex.Pattern
-
 /**
  * Encapsulates build configuration for an Elasticsearch plugin.
  */
@@ -57,16 +51,10 @@ public class PluginBuildPlugin extends BuildPlugin {
             String name = project.pluginProperties.extension.name
             project.archivesBaseName = name
 
-            if (project.pluginProperties.extension.hasClientJar) {
-                // for plugins which work with the transport client, we copy the jar
-                // file to a new name, copy the nebula generated pom to the same name,
-                // and generate a different pom for the zip
-                addClientJarPomGeneration(project)
-                addClientJarTask(project)
-            }
-            // while the jar isn't normally published, we still at least build a pom of deps
-            // in case it is published, for instance when other plugins extend this plugin
-            configureJarPom(project)
+            // set teh project description so it will be picked up by publishing
+            project.description = project.pluginProperties.extension.description
+
+            configurePublishing(project)
 
             project.integTestCluster.dependsOn(project.bundlePlugin)
             project.tasks.run.dependsOn(project.bundlePlugin)
@@ -96,6 +84,32 @@ public class PluginBuildPlugin extends BuildPlugin {
         project.tasks.create('run', RunTask) // allow running ES with this plugin in the foreground of a build
     }
 
+    private void configurePublishing(Project project) {
+        // Only configure publishing if applied externally
+        if (project.pluginProperties.extension.hasClientJar) {
+            project.plugins.apply(MavenScmPlugin.class)
+            // Only change Jar tasks, we don't want a -client zip so we can't change archivesBaseName
+            project.tasks.withType(Jar) {
+                baseName = baseName + "-client"
+            }
+            // always configure publishing for client jars
+            project.plugins.apply(MavenScmPlugin.class)
+            project.publishing.publications.nebula(MavenPublication).artifactId(
+                    project.pluginProperties.extension.name + "-client"
+            )
+            project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom generatePOMTask ->
+                generatePOMTask.ext.pomFileName = "${project.archivesBaseName}-client-${project.version}.pom"
+            }
+        } else {
+            project.plugins.withType(MavenPublishPlugin).whenPluginAdded {
+                project.publishing.publications.nebula(MavenPublication).artifactId(
+                        project.pluginProperties.extension.name
+                )
+            }
+
+        }
+    }
+
     private static void configureDependencies(Project project) {
         project.dependencies {
             compileOnly "org.elasticsearch:elasticsearch:${project.versions.elasticsearch}"
@@ -163,33 +177,6 @@ public class PluginBuildPlugin extends BuildPlugin {
     }
 
     /** Adds a task to move jar and associated files to a "-client" name. */
-    protected static void addClientJarTask(Project project) {
-        Task clientJar = project.tasks.create('clientJar')
-        clientJar.dependsOn(project.jar, project.tasks.generatePomFileForClientJarPublication, project.javadocJar, project.sourcesJar)
-        clientJar.doFirst {
-            Path jarFile = project.jar.outputs.files.singleFile.toPath()
-            String clientFileName = jarFile.fileName.toString().replace(project.version, "client-${project.version}")
-            Files.copy(jarFile, jarFile.resolveSibling(clientFileName), StandardCopyOption.REPLACE_EXISTING)
-
-            String clientPomFileName = clientFileName.replace('.jar', '.pom')
-            Files.copy(
-                    project.tasks.generatePomFileForClientJarPublication.outputs.files.singleFile.toPath(),
-                    jarFile.resolveSibling(clientPomFileName),
-                    StandardCopyOption.REPLACE_EXISTING
-            )
-
-            String sourcesFileName = jarFile.fileName.toString().replace('.jar', '-sources.jar')
-            String clientSourcesFileName = clientFileName.replace('.jar', '-sources.jar')
-            Files.copy(jarFile.resolveSibling(sourcesFileName), jarFile.resolveSibling(clientSourcesFileName),
-                    StandardCopyOption.REPLACE_EXISTING)
-
-            String javadocFileName = jarFile.fileName.toString().replace('.jar', '-javadoc.jar')
-            String clientJavadocFileName = clientFileName.replace('.jar', '-javadoc.jar')
-            Files.copy(jarFile.resolveSibling(javadocFileName), jarFile.resolveSibling(clientJavadocFileName),
-                    StandardCopyOption.REPLACE_EXISTING)
-        }
-        project.assemble.dependsOn(clientJar)
-    }
 
     static final Pattern GIT_PATTERN = Pattern.compile(/git@([^:]+):([^\.]+)\.git/)
 
@@ -211,39 +198,11 @@ public class PluginBuildPlugin extends BuildPlugin {
 
     /** Adds nebula publishing task to generate a pom file for the plugin. */
     protected static void addClientJarPomGeneration(Project project) {
-        project.plugins.apply(MavenPublishPlugin.class)
-
-        project.publishing {
-            publications {
-                clientJar(MavenPublication) {
-                    from project.components.java
-                    artifactId = project.pluginProperties.extension.name + '-client'
-                    pom.withXml { XmlProvider xml ->
-                        Node root = xml.asNode()
-                        root.appendNode('name', project.pluginProperties.extension.name)
-                        root.appendNode('description', project.pluginProperties.extension.description)
-                        root.appendNode('url', urlFromOrigin(project.scminfo.origin))
-                        Node scmNode = root.appendNode('scm')
-                        scmNode.appendNode('url', project.scminfo.origin)
-                    }
-                }
-            }
-        }
+        project.plugins.apply(MavenScmPlugin.class)
+        project.description = project.pluginProperties.extension.description
     }
 
     /** Configure the pom for the main jar of this plugin */
-    protected static void configureJarPom(Project project) {
-        project.plugins.apply(ScmInfoPlugin.class)
-        project.plugins.apply(MavenPublishPlugin.class)
-
-        project.publishing {
-            publications {
-                nebula(MavenPublication) {
-                    artifactId project.pluginProperties.extension.name
-                }
-            }
-        }
-    }
 
     protected void addNoticeGeneration(Project project) {
         File licenseFile = project.pluginProperties.extension.licenseFile

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.groovy

@@ -20,6 +20,7 @@ package org.elasticsearch.gradle.plugin
 
 import org.gradle.api.Project
 import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.InputFile
 
 /**
  * A container for plugin properties that will be written to the plugin descriptor, for easy
@@ -55,18 +56,39 @@ class PluginPropertiesExtension {
     boolean requiresKeystore = false
 
     /** A license file that should be included in the built plugin zip. */
-    @Input
-    File licenseFile = null
+    private File licenseFile = null
 
     /**
      * A notice file that should be included in the built plugin zip. This will be
      * extended with notices from the {@code licenses/} directory.
      */
-    @Input
-    File noticeFile = null
+    private File noticeFile = null
+
+    Project project = null
 
     PluginPropertiesExtension(Project project) {
         name = project.name
         version = project.version
+        this.project = project
+    }
+
+    @InputFile
+    File getLicenseFile() {
+        return licenseFile
+    }
+
+    void setLicenseFile(File licenseFile) {
+        project.ext.licenseFile = licenseFile
+        this.licenseFile = licenseFile
+    }
+
+    @InputFile
+    File getNoticeFile() {
+        return noticeFile
+    }
+
+    void setNoticeFile(File noticeFile) {
+        project.ext.noticeFile = noticeFile
+        this.noticeFile = noticeFile
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginPropertiesTask.groovy

@@ -23,7 +23,6 @@ import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Task
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.OutputFile
-
 /**
  * Creates a plugin descriptor.
  */

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -21,6 +21,7 @@ package org.elasticsearch.gradle.precommit
 import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
 import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
+import org.gradle.api.JavaVersion
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.gradle.api.file.FileCollection
@@ -101,6 +102,11 @@ class PrecommitTasks {
                 signaturesURLs = project.forbiddenApis.signaturesURLs +
                     [ getClass().getResource('/forbidden/es-server-signatures.txt') ]
             }
+            // forbidden apis doesn't support Java 11, so stop at 10
+            String targetMajorVersion = (project.compilerJavaVersion.compareTo(JavaVersion.VERSION_1_10) > 0 ?
+                    JavaVersion.VERSION_1_10 :
+                    project.compilerJavaVersion).getMajorVersion()
+            targetCompatibility = Integer.parseInt(targetMajorVersion) >= 9 ?targetMajorVersion : "1.${targetMajorVersion}"
         }
         Task forbiddenApis = project.tasks.findByName('forbiddenApis')
         forbiddenApis.group = "" // clear group, so this does not show up under verification tasks

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy

@@ -177,6 +177,12 @@ class NodeInfo {
             javaVersion = 8
         } else if (nodeVersion.onOrAfter("6.2.0") && nodeVersion.before("6.3.0")) {
             javaVersion = 9
+        } else if (project.inFipsJvm && nodeVersion.onOrAfter("6.3.0") && nodeVersion.before("6.4.0")) {
+            /*
+             * Elasticsearch versions before 6.4.0 cannot be run in a FIPS-140 JVM. If we're running
+             * bwc tests in a FIPS-140 JVM, ensure that the pre v6.4.0 nodes use a Java 10 JVM instead.
+             */
+            javaVersion = 10
         }
 
         args.addAll("-E", "node.portsfile=true")

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -31,6 +31,7 @@ import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskState
+import org.gradle.plugins.ide.idea.IdeaPlugin
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
@@ -243,10 +244,12 @@ public class RestIntegTestTask extends DefaultTask {
                 }
             }
         }
-        project.idea {
-            module {
-                if (scopes.TEST != null) {
-                    scopes.TEST.plus.add(project.configurations.restSpec)
+        if (project.plugins.hasPlugin(IdeaPlugin)) {
+            project.idea {
+                module {
+                    if (scopes.TEST != null) {
+                        scopes.TEST.plus.add(project.configurations.restSpec)
+                    }
                 }
             }
         }