merge lastest code from author

.ci/bwcVersions

@@ -9,4 +9,5 @@ BWC_VERSION:
   - "7.3.1"
   - "7.3.2"
   - "7.4.0"
+  - "7.5.0"
   - "8.0.0"

.ci/java-versions.properties

@@ -5,7 +5,7 @@
 # are 'java' or 'openjdk' followed by the major release number.
 
 ES_BUILD_JAVA=openjdk12
-ES_RUNTIME_JAVA=java11
+ES_RUNTIME_JAVA=openjdk11
 GRADLE_TASK=build
 
 

.ci/matrix-runtime-javas.yml

@@ -14,3 +14,4 @@ ES_RUNTIME_JAVA:
   - zulu11
   - zulu12
   - corretto11
+  - adoptopenjdk11

build.gradle

@@ -33,7 +33,7 @@ import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
 
 plugins {
     id 'com.gradle.build-scan' version '2.4'
-    id 'base'
+    id 'lifecycle-base'
     id 'elasticsearch.global-build-info'
 }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -48,7 +48,6 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier
 import org.gradle.api.artifacts.ProjectDependency
 import org.gradle.api.artifacts.ResolvedArtifact
 import org.gradle.api.artifacts.dsl.RepositoryHandler
-import org.gradle.api.artifacts.repositories.ArtifactRepository
 import org.gradle.api.artifacts.repositories.IvyArtifactRepository
 import org.gradle.api.artifacts.repositories.IvyPatternRepositoryLayout
 import org.gradle.api.artifacts.repositories.MavenArtifactRepository
@@ -85,12 +84,10 @@ import org.gradle.util.GradleVersion
 
 import java.nio.charset.StandardCharsets
 import java.nio.file.Files
-import java.time.ZoneOffset
-import java.time.ZonedDateTime
 import java.util.regex.Matcher
 
-import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
 import static org.elasticsearch.gradle.tool.Boilerplate.findByName
+import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
 
 /**
  * Encapsulates build configuration for elasticsearch projects.
@@ -411,11 +408,11 @@ class BuildPlugin implements Plugin<Project> {
         project.getRepositories().all { repository ->
             if (repository instanceof MavenArtifactRepository) {
                 final MavenArtifactRepository maven = (MavenArtifactRepository) repository
-                assertRepositoryURIUsesHttps(maven, project, maven.getUrl())
-                repository.getArtifactUrls().each { uri -> assertRepositoryURIUsesHttps(maven, project, uri) }
+                assertRepositoryURIIsSecure(maven.name, project.path, maven.getUrl())
+                repository.getArtifactUrls().each { uri -> assertRepositoryURIIsSecure(maven.name, project.path, uri) }
             } else if (repository instanceof IvyArtifactRepository) {
                 final IvyArtifactRepository ivy = (IvyArtifactRepository) repository
-                assertRepositoryURIUsesHttps(ivy, project, ivy.getUrl())
+                assertRepositoryURIIsSecure(ivy.name, project.path, ivy.getUrl())
             }
         }
         RepositoryHandler repos = project.repositories
@@ -455,9 +452,15 @@ class BuildPlugin implements Plugin<Project> {
         }
     }
 
-    private static void assertRepositoryURIUsesHttps(final ArtifactRepository repository, final Project project, final URI uri) {
-        if (uri != null && uri.toURL().getProtocol().equals("http")) {
-            throw new GradleException("repository [${repository.name}] on project with path [${project.path}] is using http for artifacts on [${uri.toURL()}]")
+    static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
+        if (uri != null && ["file", "https", "s3"].contains(uri.getScheme()) == false) {
+            final String message = String.format(
+                    Locale.ROOT,
+                    "repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
+                    repositoryName,
+                    projectPath,
+                    uri.toURL())
+            throw new GradleException(message)
         }
     }
 
@@ -884,6 +887,12 @@ class BuildPlugin implements Plugin<Project> {
                 // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
                 test.systemProperty 'es.transport.cname_in_publish_address', 'true'
 
+                // Set netty system properties to the properties we configure in jvm.options
+                test.systemProperty('io.netty.noUnsafe', 'true')
+                test.systemProperty('io.netty.noKeySetOptimization', 'true')
+                test.systemProperty('io.netty.recycler.maxCapacityPerThread', '0')
+                test.systemProperty('io.netty.allocator.numDirectArenas', '0')
+
                 test.testLogging { TestLoggingContainer logging ->
                     logging.showExceptions = true
                     logging.showCauses = true

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

@@ -123,7 +123,7 @@ public class SnippetsTask extends DefaultTask {
                     }
                 }
                 if (snippet.testResponse
-                        && 'js' == snippet.language
+                        && ('js' == snippet.language || 'console-result' == snippet.language)
                         && null == snippet.skip) {
                     String quoted = snippet.contents
                         // quote values starting with $
@@ -162,7 +162,7 @@ public class SnippetsTask extends DefaultTask {
                     }
                     return
                 }
-                matcher = line =~ /\["?source"?,\s*"?(\w+)"?(,.*)?].*/
+                matcher = line =~ /\["?source"?,\s*"?([-\w]+)"?(,.*)?].*/
                 if (matcher.matches()) {
                     lastLanguage = matcher.group(1)
                     lastLanguageLine = lineNumber

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -236,7 +236,7 @@ class RestIntegTestTask extends DefaultTask {
         }
         project.dependencies {
             restSpec ClasspathUtils.isElasticsearchProject() ? project.project(':rest-api-spec') :
-                    "org.elasticsearch.rest-api-spec:${VersionProperties.elasticsearch}"
+                    "org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
         }
         Task copyRestSpec = project.tasks.findByName('copyRestSpec')
         if (copyRestSpec != null) {

buildSrc/src/test/groovy/org/elasticsearch/gradle/BuildPluginTests.java

@@ -22,6 +22,9 @@
 import org.gradle.api.GradleException;
 import org.junit.Test;
 
+import java.net.URI;
+import java.net.URISyntaxException;
+
 
 public class BuildPluginTests extends GradleUnitTestCase {
 
@@ -36,4 +39,25 @@ public void testFailingDockerVersions() {
         BuildPlugin.checkDockerVersionRecent("Docker version 17.04.0, build e68fc7a");
     }
 
+    @Test(expected = GradleException.class)
+    public void testRepositoryURIThatUsesHttpScheme() throws URISyntaxException {
+        final URI uri = new URI("http://s3.amazonaws.com/artifacts.elastic.co/maven");
+        BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
+    }
+
+    public void testRepositoryThatUsesFileScheme() throws URISyntaxException {
+        final URI uri = new URI("file:/tmp/maven");
+        BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
+    }
+
+    public void testRepositoryURIThatUsesHttpsScheme() throws URISyntaxException {
+        final URI uri = new URI("https://s3.amazonaws.com/artifacts.elastic.co/maven");
+        BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
+    }
+
+    public void testRepositoryURIThatUsesS3Scheme() throws URISyntaxException {
+        final URI uri = new URI("s3://artifacts.elastic.co/maven");
+        BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
+    }
+
 }

buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginIT.java

@@ -18,19 +18,29 @@
  */
 package org.elasticsearch.gradle;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
 import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.List;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
 public class BuildPluginIT extends GradleIntegrationTestCase {
 
+    @Rule
+    public TemporaryFolder tmpDir = new TemporaryFolder();
+
     public void testPluginCanBeApplied() {
         BuildResult result = getGradleRunner("elasticsearch.build")
             .withArguments("hello", "-s")
@@ -46,6 +56,51 @@ public void testCheckTask() {
         assertTaskSuccessful(result, ":check");
     }
 
+    public void testInsecureMavenRepository() throws IOException {
+        final String name = "elastic-maven";
+        final String url = "http://s3.amazonaws.com/artifacts.elastic.co/maven";
+        // add an insecure maven repository to the build.gradle
+        final List<String> lines = Arrays.asList(
+            "repositories {",
+            "  maven {",
+            "    name \"elastic-maven\"",
+            "    url \"" +  url + "\"\n",
+            "  }",
+            "}");
+        runInsecureArtifactRepositoryTest(name, url, lines);
+    }
+
+    public void testInsecureIvyRepository() throws IOException {
+        final String name = "elastic-ivy";
+        final String url = "http://s3.amazonaws.com/artifacts.elastic.co/ivy";
+        // add an insecure ivy repository to the build.gradle
+        final List<String> lines = Arrays.asList(
+            "repositories {",
+            "  ivy {",
+            "    name \"elastic-ivy\"",
+            "    url \"" +  url + "\"\n",
+            "  }",
+            "}");
+        runInsecureArtifactRepositoryTest(name, url, lines);
+    }
+
+    private void runInsecureArtifactRepositoryTest(final String name, final String url, final List<String> lines) throws IOException {
+        final File projectDir = getProjectDir("elasticsearch.build");
+        FileUtils.copyDirectory(projectDir, tmpDir.getRoot(), pathname -> pathname.getPath().contains("/build/") == false);
+        final List<String> buildGradleLines =
+            Files.readAllLines(tmpDir.getRoot().toPath().resolve("build.gradle"), StandardCharsets.UTF_8);
+        buildGradleLines.addAll(lines);
+        Files.write(tmpDir.getRoot().toPath().resolve("build.gradle"), buildGradleLines, StandardCharsets.UTF_8);
+        final BuildResult result = GradleRunner.create()
+            .withProjectDir(tmpDir.getRoot())
+            .withArguments("clean", "hello", "-s", "-i", "--warning-mode=all", "--scan")
+            .withPluginClasspath()
+            .buildAndFail();
+        assertOutputContains(
+            result.getOutput(),
+            "repository [" + name + "] on project with path [:] is not using a secure protocol for artifacts on [" + url + "]");
+    }
+
     public void testLicenseAndNotice() throws IOException {
         BuildResult result = getGradleRunner("elasticsearch.build")
             .withArguments("clean", "assemble", "-s", "-Dlocal.repo.path=" + getLocalTestRepoPath())

client/rest-high-level/build.gradle

@@ -85,6 +85,7 @@ processTestResources {
   from({ zipTree(configurations.restSpec.singleFile) }) {
     include 'rest-api-spec/api/**'
   }
+  from(project(':client:rest-high-level').file('src/test/resources'))
 }
 
 dependencyLicenses {
@@ -117,6 +118,7 @@ if (isEclipse) {
 
 File nodeCert = file("./testnode.crt")
 File nodeTrustStore = file("./testnode.jks")
+File pkiTrustCert = file("./src/test/resources/org/elasticsearch/client/security/delegate_pki/testRootCA.crt")
 
 integTest.runner {
   systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
@@ -134,11 +136,18 @@ testClusters.integTest {
   // Truststore settings are not used since TLS is not enabled. Included for testing the get certificates API
   setting 'xpack.security.http.ssl.certificate_authorities', 'testnode.crt'
   setting 'xpack.security.transport.ssl.truststore.path', 'testnode.jks'
+  setting 'xpack.security.authc.realms.file.default_file.order', '0'
+  setting 'xpack.security.authc.realms.native.default_native.order', '1'
+  setting 'xpack.security.authc.realms.pki.pki1.order', '2'
+  setting 'xpack.security.authc.realms.pki.pki1.certificate_authorities', '[ "testRootCA.crt" ]'
+  setting 'xpack.security.authc.realms.pki.pki1.delegation.enabled', 'true'
+
   setting 'indices.lifecycle.poll_interval', '1000ms'
   keystore 'xpack.security.transport.ssl.truststore.secure_password', 'testnode'
   user username:  System.getProperty('tests.rest.cluster.username', 'test_user'),
           password:  System.getProperty('tests.rest.cluster.password', 'test-password')
 
   extraConfigFile nodeCert.name, nodeCert
   extraConfigFile nodeTrustStore.name, nodeTrustStore
+  extraConfigFile pkiTrustCert.name, pkiTrustCert
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityClient.java

@@ -31,6 +31,8 @@
 import org.elasticsearch.client.security.CreateApiKeyResponse;
 import org.elasticsearch.client.security.CreateTokenRequest;
 import org.elasticsearch.client.security.CreateTokenResponse;
+import org.elasticsearch.client.security.DelegatePkiAuthenticationRequest;
+import org.elasticsearch.client.security.DelegatePkiAuthenticationResponse;
 import org.elasticsearch.client.security.DeletePrivilegesRequest;
 import org.elasticsearch.client.security.DeletePrivilegesResponse;
 import org.elasticsearch.client.security.DeleteRoleMappingRequest;
@@ -1005,4 +1007,39 @@ public Cancellable invalidateApiKeyAsync(final InvalidateApiKeyRequest request,
         return restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::invalidateApiKey, options,
                 InvalidateApiKeyResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * Get an Elasticsearch access token from an {@code X509Certificate} chain. The certificate chain is that of the client from a mutually
+     * authenticated TLS session, and it is validated by the PKI realms with {@code delegation.enabled} toggled to {@code true}.<br>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-delegate-pki-authentication.html"> the
+     * docs</a> for more details.
+     * 
+     * @param request the request containing the certificate chain
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response from the delegate-pki-authentication API key call
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public DelegatePkiAuthenticationResponse delegatePkiAuthentication(DelegatePkiAuthenticationRequest request, RequestOptions options)
+            throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request, SecurityRequestConverters::delegatePkiAuthentication, options,
+                DelegatePkiAuthenticationResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously get an Elasticsearch access token from an {@code X509Certificate} chain. The certificate chain is that of the client
+     * from a mutually authenticated TLS session, and it is validated by the PKI realms with {@code delegation.enabled} toggled to
+     * {@code true}.<br>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-delegate-pki-authentication.html"> the
+     * docs</a> for more details.
+     * 
+     * @param request the request containing the certificate chain
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     * @return cancellable that may be used to cancel the request
+     */
+    public Cancellable delegatePkiAuthenticationAsync(DelegatePkiAuthenticationRequest request, RequestOptions options,
+            ActionListener<DelegatePkiAuthenticationResponse> listener) {
+        return restHighLevelClient.performRequestAsyncAndParseEntity(request, SecurityRequestConverters::delegatePkiAuthentication, options,
+                DelegatePkiAuthenticationResponse::fromXContent, listener, emptySet());
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/SecurityRequestConverters.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.client.security.ClearRolesCacheRequest;
 import org.elasticsearch.client.security.CreateApiKeyRequest;
 import org.elasticsearch.client.security.CreateTokenRequest;
+import org.elasticsearch.client.security.DelegatePkiAuthenticationRequest;
 import org.elasticsearch.client.security.DeletePrivilegesRequest;
 import org.elasticsearch.client.security.DeleteRoleMappingRequest;
 import org.elasticsearch.client.security.DeleteRoleRequest;
@@ -221,6 +222,12 @@ static Request createToken(CreateTokenRequest createTokenRequest) throws IOExcep
         return request;
     }
 
+    static Request delegatePkiAuthentication(DelegatePkiAuthenticationRequest delegatePkiAuthenticationRequest) throws IOException {
+        Request request = new Request(HttpPost.METHOD_NAME, "/_security/delegate_pki");
+        request.setEntity(createEntity(delegatePkiAuthenticationRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request invalidateToken(InvalidateTokenRequest invalidateTokenRequest) throws IOException {
         Request request = new Request(HttpDelete.METHOD_NAME, "/_security/oauth2/token");
         request.setEntity(createEntity(invalidateTokenRequest, REQUEST_BODY_CONTENT_TYPE));
@@ -294,7 +301,7 @@ static Request getApiKey(final GetApiKeyRequest getApiKeyRequest) throws IOExcep
         if (Strings.hasText(getApiKeyRequest.getRealmName())) {
             request.addParameter("realm_name", getApiKeyRequest.getRealmName());
         }
-
+        request.addParameter("owner", Boolean.toString(getApiKeyRequest.ownedByAuthenticatedUser()));
         return request;
     }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/MlDataFrameAnalysisNamedXContentProvider.java

@@ -32,6 +32,10 @@ public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
             new NamedXContentRegistry.Entry(
                 DataFrameAnalysis.class,
                 OutlierDetection.NAME,
-                (p, c) -> OutlierDetection.fromXContent(p)));
+                (p, c) -> OutlierDetection.fromXContent(p)),
+            new NamedXContentRegistry.Entry(
+                DataFrameAnalysis.class,
+                Regression.NAME,
+                (p, c) -> Regression.fromXContent(p)));
     }
 }