Add JDK11 support and enable in CI

.ci/matrix-build-javas.yml

@@ -7,3 +7,4 @@
 
 ES_BUILD_JAVA:
   - java10
+  - java11

.ci/matrix-runtime-javas.yml

@@ -8,3 +8,4 @@
 ES_RUNTIME_JAVA:
   - java8
   - java10
+  - java11

buildSrc/src/main/resources/forbidden/jdk-signatures.txt

@@ -88,7 +88,6 @@ java.lang.Thread#getAllStackTraces()
 
 @defaultMessage Stopping threads explicitly leads to inconsistent states. Use interrupt() instead.
 java.lang.Thread#stop()
-java.lang.Thread#stop(java.lang.Throwable)
 
 @defaultMessage Please do not terminate the application
 java.lang.System#exit(int)

distribution/bwc/build.gradle

@@ -22,6 +22,8 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 
+import java.nio.charset.Charset
+
 import static org.elasticsearch.gradle.BuildPlugin.getJavaHome
 
 /**
@@ -148,13 +150,20 @@ subprojects {
   task buildBwcVersion(type: Exec) {
     dependsOn checkoutBwcBranch, writeBuildMetadata
     workingDir = checkoutDir
+    // we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds
+    // also send RUNTIME_JAVA_HOME so the build doesn't fails on newer version the branch doesn't know about
     if (["5.6", "6.0", "6.1"].contains(bwcBranch)) {
-      // we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds
       environment('JAVA_HOME', getJavaHome(it, 8))
+      environment('RUNTIME_JAVA_HOME', getJavaHome(it, 8))
     } else if ("6.2".equals(bwcBranch)) {
       environment('JAVA_HOME', getJavaHome(it, 9))
+      environment('RUNTIME_JAVA_HOME', getJavaHome(it, 9))
+    } else if (["6.3", "6.x"].contains(bwcBranch)) {
+      environment('JAVA_HOME', getJavaHome(it, 10))
+      environment('RUNTIME_JAVA_HOME', getJavaHome(it, 10))
     } else {
       environment('JAVA_HOME', project.compilerJavaHome)
+      environment('RUNTIME_JAVA_HOME', project.compilerJavaHome)
     }
     if (Os.isFamily(Os.FAMILY_WINDOWS)) {
       executable 'cmd'
@@ -177,6 +186,8 @@ subprojects {
     } else if (showStacktraceName.equals("ALWAYS_FULL")) {
       args "--full-stacktrace"
     }
+    standardOutput = new IdentingOutputStream(System.out)
+    errorOutput = new IdentingOutputStream(System.err)
     doLast {
       List missing = artifactFiles.grep { file ->
         false == file.exists()
@@ -196,3 +207,25 @@ subprojects {
     }
   }
 }
+
+class IdentingOutputStream extends OutputStream {
+
+  private final OutputStream delegate
+
+  public IdentingOutputStream(OutputStream delegate) {
+      this.delegate = delegate
+  }
+
+  @Override
+  public void write(int b) {
+    write([b] as int[], 0, 1)
+  }
+
+  public void write(int[] bytes, int offset, int length) {
+    delegate.write(
+            new String(bytes, offset, length)
+                    .replace("\n", "\n  [bwc] ")
+                    .getBytes(Charset.forName("UTF-8"))
+    )
+  }
+}

docs/build.gradle

@@ -39,6 +39,13 @@ integTestCluster {
   setting 'reindex.remote.whitelist', '127.0.0.1:*'
 }
 
+integTestRunner {
+    systemProperty 'tests.rest.blacklist', [
+        // remove when https://github.com/elastic/elasticsearch/issues/31305 is fixed
+        'plugins/ingest-attachment/line_164',
+        'plugins/ingest-attachment/line_117'
+    ].join(',')
+}
 // Build the cluster with all plugins
 
 project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->

modules/lang-painless/src/test/java/org/elasticsearch/painless/ArrayTests.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.bootstrap.JavaVersion;
 import org.hamcrest.Matcher;
 
 import java.lang.invoke.MethodHandle;
@@ -41,7 +42,11 @@ protected String valueCtorCall(String valueType, int size) {
 
     @Override
     protected Matcher<String> outOfBoundsExceptionMessageMatcher(int index, int size) {
-        return equalTo(Integer.toString(index));
+        if (JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0) {
+            return equalTo(Integer.toString(index));
+        } else{
+            return equalTo("Index " + Integer.toString(index) + " out of bounds for length " + Integer.toString(size));
+        }
     }
 
     public void testArrayLengthHelper() throws Throwable {

plugins/ingest-attachment/build.gradle

@@ -25,11 +25,20 @@ esplugin {
 versions << [
   'tika': '1.18',
   'pdfbox': '2.0.9',
-  'bouncycastle': '1.55',
+  'bouncycastle': '1.59',
   'poi': '3.17',
   'mime4j': '0.8.1'
 ]
 
+integTestRunner {
+    // disabled untill https://github.com/elastic/elasticsearch/issues/31456
+    // is fixed.
+    systemProperty 'tests.rest.blacklist', [
+      'ingest_attachment/20_attachment_processor/Test indexed chars are configurable',
+      'ingest_attachment/20_attachment_processor/Test indexed chars are configurable per document'
+    ].join(',')
+}
+
 dependencies {
   // mandatory for tika
   compile "org.apache.tika:tika-core:${versions.tika}"

plugins/ingest-attachment/licenses/bcmail-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+db389ade95f48592908a84e7050a691c8834723c

plugins/ingest-attachment/licenses/bcpkix-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+9cef0aab8a4bb849a8476c058ce3ff302aba3fff

plugins/ingest-attachment/licenses/bcprov-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+2507204241ab450456bdb8e8c0a8f986e418bd99

plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/TikaImpl.java

@@ -164,12 +164,11 @@ static PermissionCollection getRestrictedPermissions() {
         perms.add(new RuntimePermission("getClassLoader"));
         // ZipFile needs accessDeclaredMembers on JDK 10; cf. https://bugs.openjdk.java.net/browse/JDK-8187485
         if (JavaVersion.current().compareTo(JavaVersion.parse("10")) >= 0) {
-            /*
-             * See if this permission can be removed in JDK 11, bump the version here to 12 if not. If this permission can be removed, also
-             * remove the grant in the plugin-security.policy.
-             */
-            assert JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0;
-            perms.add(new RuntimePermission("accessDeclaredMembers"));
+            if (JavaVersion.current().compareTo(JavaVersion.parse("11")) < 0) {
+                // TODO remove this and from plugin-security.policy when JDK 11 is the only one we support
+                // this is needed pre 11, but it's fixed in 11 : https://bugs.openjdk.java.net/browse/JDK-8187485
+                perms.add(new RuntimePermission("accessDeclaredMembers"));
+            }
         }
         perms.setReadOnly();
         return perms;

plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/AttachmentProcessorTests.java

@@ -295,6 +295,7 @@ private Map<String, Object> parseDocument(String file, AttachmentProcessor proce
         return attachmentData;
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/31305")
     public void testIndexedChars() throws Exception {
         processor = new AttachmentProcessor(randomAlphaOfLength(10), "source_field",
             "target_field", EnumSet.allOf(AttachmentProcessor.Property.class), 19, false, null);

plugins/repository-hdfs/build.gradle

@@ -214,6 +214,21 @@ RestIntegTestTask integTestSecureHa = project.tasks.create('integTestSecureHa',
   description = "Runs rest tests against an elasticsearch cluster with HDFS configured with HA Namenode and secured by MIT Kerberos."
 }
 
+// TODO remove when: https://github.com/elastic/elasticsearch/issues/31498
+integTestRunner {
+    systemProperty 'tests.rest.blacklist', [
+        'hdfs_repository/30_snapshot/take snapshot',
+        'hdfs_repository/40_restore/Create a snapshot and then restore it',
+        'hdfs_repository/20_repository_verify/HDFS Repository Verify',
+        'hdfs_repository/30_snapshot_get/Get a snapshot',
+        'hdfs_repository/20_repository_create/HDFS Repository Creation',
+        'hdfs_repository/20_repository_delete/HDFS Delete Repository',
+        'hdfs_repository/30_snapshot_readonly/Get a snapshot - readonly',
+    ].join(',')
+}
+// TODO remove when: https://github.com/elastic/elasticsearch/issues/31498
+integTestHa.enabled = false
+
 // Determine HDFS Fixture compatibility for the current build environment.
 boolean fixtureSupported = false
 if (Os.isFamily(Os.FAMILY_WINDOWS)) {

plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java

@@ -43,6 +43,7 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(HdfsPlugin.class);
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/31498")
     public void testSimpleWorkflow() {
         Client client = client();
 

x-pack/plugin/core/build.gradle

@@ -35,8 +35,8 @@ dependencies {
 
     // security deps
     compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
-    compile 'org.bouncycastle:bcprov-jdk15on:1.58'
-    compile 'org.bouncycastle:bcpkix-jdk15on:1.58'
+    compile 'org.bouncycastle:bcprov-jdk15on:1.59'
+    compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
     compile project(path: ':modules:transport-netty4', configuration: 'runtime')
 
     testCompile 'org.elasticsearch:securemock:1.2'

x-pack/plugin/core/licenses/bcpkix-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+9cef0aab8a4bb849a8476c058ce3ff302aba3fff

x-pack/plugin/core/licenses/bcprov-jdk15on-1.59.jar.sha1

@@ -0,0 +1 @@
+2507204241ab450456bdb8e8c0a8f986e418bd99

x-pack/plugin/security/build.gradle

@@ -22,8 +22,8 @@ dependencies {
     testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
 
     compile 'com.unboundid:unboundid-ldapsdk:3.2.0'
-    compile 'org.bouncycastle:bcprov-jdk15on:1.58'
-    compile 'org.bouncycastle:bcpkix-jdk15on:1.58'
+    compile 'org.bouncycastle:bcprov-jdk15on:1.59'
+    compile 'org.bouncycastle:bcpkix-jdk15on:1.59'
 
     // the following are all SAML dependencies - might as well download the whole internet
     compile "org.opensaml:opensaml-core:3.3.0"