Merge remote-tracking branch 'elastic/master' into sync-retention-leases

* elastic/master: Optimize warning header de-duplication (elastic#37725) Bubble exceptions up in ClusterApplierService (elastic#37729) SQL: Improve handling of invalid args for PERCENTILE/PERCENTILE_RANK (elastic#37803) Remove unused ThreadBarrier class (elastic#37666) Add built-in user and role for code plugin (elastic#37030) Consolidate testclusters tests into a single project (elastic#37362) Fix docs for MappingUpdatedAction SQL: Introduce SQL DATE data type (elastic#37693) disabling bwc test while backporting elastic#37639 Mute ClusterDisruptionIT testAckedIndexing Set acking timeout to 0 on dynamic mapping update (elastic#31140) Remove index audit output type (elastic#37707) Mute FollowerFailOverIT testReadRequestsReturnsLatestMappingVersion [ML] Increase close job timeout and lower the max number (elastic#37770) Remove Custom Listeners from SnapshotsService (elastic#37629) Use m_m_nodes from Zen1 master for Zen2 bootstrap (elastic#37701) Fix index filtering in follow info api. (elastic#37752) Use project dependency instead of substitutions for distributions (elastic#37730) Update authenticate to allow unknown fields (elastic#37713) Deprecate HLRC EmptyResponse used by security (elastic#37540)
jasontedor · Jan 24, 2019 · 1cb71ec · 1cb71ec
2 parents 0496d83 + 7517e3a
commit 1cb71ec
Show file tree

Hide file tree

Showing 129 changed files with 1,996 additions and 4,993 deletions.
diff --git a/build.gradle b/build.gradle
@@ -159,8 +159,8 @@ task verifyVersions {
  * the enabled state of every bwc task. It should be set back to true
  * after the backport of the backcompat code is complete.
  */
-final boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+final boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "backporting https://github.com/elastic/elasticsearch/pull/37639" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
@@ -221,14 +221,6 @@ allprojects {
     "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
-    "downloads.zip:elasticsearch:${version}": ':distribution:archives:zip',
-    "downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
-    "downloads.tar:elasticsearch:${version}": ':distribution:archives:tar',
-    "downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
-    "downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
-    "downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
-    "downloads.deb:elasticsearch:${version}": ':distribution:packages:deb',
-    "downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
     "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
     "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
     // for transport client
@@ -240,19 +232,6 @@ allprojects {
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
     "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
   ]
-  // substitute unreleased versions with projects that check out and build locally
-  bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
-    Version unreleased = unreleasedVersion.version
-    String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
-    ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject
-    if (unreleased.onOrAfter('6.3.0')) {
-      ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
-    }
-  }
 
   /*
    * Gradle only resolve project substitutions during dependency resolution but

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
@@ -171,6 +172,12 @@ class ClusterFormationTasks {
 
     /** Adds a dependency on the given distribution */
     static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
+        if (distro.equals("integ-test-zip")) {
+            // short circuit integ test so it doesn't complicate the rest of the distribution setup below
+            project.dependencies.add(configuration.name,
+                    "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${elasticsearchVersion}@zip")
+            return
+        }
         // TEMP HACK
         // The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated.
         if (distro.equals('oss-zip')) {
@@ -180,22 +187,31 @@ class ClusterFormationTasks {
             distro = 'default'
         }
         // END TEMP HACK
-        if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) {
+        if (['oss', 'default'].contains(distro) == false) {
             throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
         }
         Version version = Version.fromString(elasticsearchVersion)
-        if (version.before('6.3.0') && distro.startsWith('oss-')) {
-            distro = distro.substring('oss-'.length())
-        }
-        String group = "downloads.zip"
-        if (distro.equals("integ-test-zip")) {
-            group = "org.elasticsearch.distribution.integ-test-zip"
-        }
+        String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo
         String artifactName = 'elasticsearch'
         if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
             artifactName += '-oss'
         }
-        project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip")
+        String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip'
+        Object dependency
+        boolean internalBuild = project.hasProperty('bwcVersions')
+        VersionCollection.UnreleasedVersionInfo unreleasedInfo = null
+        if (project.hasProperty('bwcVersions')) {
+            // NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools?
+            unreleasedInfo = project.bwcVersions.unreleasedInfo(version)
+        }
+        if (unreleasedInfo != null) {
+            dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
+        } else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) {
+            dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}")
+        } else {
+            dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip"
+        }
+        project.dependencies.add(configuration.name, dependency)
     }
 
     /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
@@ -4,6 +4,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.FileContentsTask
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
 import org.gradle.api.*
 import org.gradle.api.artifacts.dsl.RepositoryHandler
 import org.gradle.api.execution.TaskExecutionAdapter
@@ -184,22 +185,38 @@ class VagrantTestPlugin implements Plugin<Project> {
             upgradeFromVersion = Version.fromString(upgradeFromVersionRaw)
         }
 
+        List<Object> dependencies = new ArrayList<>()
         DISTRIBUTIONS.each {
             // Adds a dependency for the current version
-            project.dependencies.add(PACKAGING_CONFIGURATION,
-                    project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
+            dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
         }
 
-        UPGRADE_FROM_ARCHIVES.each {
-            // The version of elasticsearch that we upgrade *from*
-            project.dependencies.add(PACKAGING_CONFIGURATION,
-                    "downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
-            if (upgradeFromVersion.onOrAfter('6.3.0')) {
-                project.dependencies.add(PACKAGING_CONFIGURATION,
-                        "downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+        // The version of elasticsearch that we upgrade *from*
+        VersionCollection.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion)
+        if (unreleasedInfo != null) {
+            // handle snapshots pointing to bwc build
+            UPGRADE_FROM_ARCHIVES.each {
+                dependencies.add(project.dependencies.project(
+                        path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: it))
+                if (upgradeFromVersion.onOrAfter('6.3.0')) {
+                    dependencies.add(project.dependencies.project(
+                            path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: "oss-${it}"))
+                }
+            }
+        } else {
+            UPGRADE_FROM_ARCHIVES.each {
+                // The version of elasticsearch that we upgrade *from*
+                dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+                if (upgradeFromVersion.onOrAfter('6.3.0')) {
+                    dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+                }
             }
         }
 
+        for (Object dependency : dependencies) {
+            project.dependencies.add(PACKAGING_CONFIGURATION, dependency)
+        }
+
         project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
     }
 

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java b/buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java
@@ -20,6 +20,8 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -86,6 +88,7 @@ public class VersionCollection {
 
     private final Version currentVersion;
     private final Map<Integer, List<Version>> groupByMajor;
+    private final Map<Version, UnreleasedVersionInfo> unreleased;
 
     public class UnreleasedVersionInfo {
         public final Version version;
@@ -129,6 +132,16 @@ protected VersionCollection(List<String> versionLines, Version currentVersionPro
         assertCurrentVersionMatchesParsed(currentVersionProperty);
 
         assertNoOlderThanTwoMajors();
+
+        Map<Version, UnreleasedVersionInfo> unreleased = new HashMap<>();
+        for (Version unreleasedVersion : getUnreleased()) {
+            if (unreleasedVersion.equals(currentVersion)) {
+                continue;
+            }
+            unreleased.put(unreleasedVersion,
+                new UnreleasedVersionInfo(unreleasedVersion, getBranchFor(unreleasedVersion), getGradleProjectNameFor(unreleasedVersion)));
+        }
+        this.unreleased = Collections.unmodifiableMap(unreleased);
     }
 
     private void assertNoOlderThanTwoMajors() {
@@ -150,6 +163,13 @@ private void assertCurrentVersionMatchesParsed(Version currentVersionProperty) {
         }
     }
 
+    /**
+      * Returns info about the unreleased version, or {@code null} if the version is released.
+      */
+    public UnreleasedVersionInfo unreleasedInfo(Version version) {
+        return unreleased.get(version);
+    }
+
     public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
         getUnreleased().stream()
             .filter(version -> version.equals(currentVersion) == false)

diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java
@@ -37,34 +37,34 @@ public void testListClusters() {
     }
 
     public void testUseClusterByOne() {
-        BuildResult result = getTestClustersRunner("user1").build();
+        BuildResult result = getTestClustersRunner(":user1").build();
         assertTaskSuccessful(result, ":user1");
         assertStartedAndStoppedOnce(result);
     }
 
     public void testUseClusterByOneWithDryRun() {
-        BuildResult result = getTestClustersRunner("--dry-run", "user1").build();
+        BuildResult result = getTestClustersRunner("--dry-run", ":user1").build();
         assertNull(result.task(":user1"));
         assertNotStarted(result);
     }
 
     public void testUseClusterByTwo() {
-        BuildResult result = getTestClustersRunner("user1", "user2").build();
+        BuildResult result = getTestClustersRunner(":user1", ":user2").build();
         assertTaskSuccessful(result, ":user1", ":user2");
         assertStartedAndStoppedOnce(result);
     }
 
     public void testUseClusterByUpToDateTask() {
         // Run it once, ignoring the result and again to make sure it's considered up to date.
         // Gradle randomly considers tasks without inputs and outputs as as up-to-date or success on the first run
-        getTestClustersRunner("upToDate1", "upToDate2").build();
-        BuildResult result = getTestClustersRunner("upToDate1", "upToDate2").build();
+        getTestClustersRunner(":upToDate1", ":upToDate2").build();
+        BuildResult result = getTestClustersRunner(":upToDate1", ":upToDate2").build();
         assertTaskUpToDate(result, ":upToDate1", ":upToDate2");
         assertNotStarted(result);
     }
 
     public void testUseClusterBySkippedTask() {
-        BuildResult result = getTestClustersRunner("skipped1", "skipped2").build();
+        BuildResult result = getTestClustersRunner(":skipped1", ":skipped2").build();
         assertTaskSkipped(result, ":skipped1", ":skipped2");
         assertNotStarted(result);
     }
@@ -82,17 +82,44 @@ public void testUseClusterBySkippedAndWorkingTask() {
     }
 
     public void testMultiProject() {
-        BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("testclusters_multiproject"))
-            .withArguments("user1", "user2", "-s", "-i", "--parallel", "-Dlocal.repo.path=" + getLocalTestRepoPath())
-            .withPluginClasspath()
-            .build();
-        assertTaskSuccessful(result, ":user1", ":user2");
+        BuildResult result = getTestClustersRunner(
+            "user1", "user2", "-s", "-i", "--parallel", "-Dlocal.repo.path=" + getLocalTestRepoPath()
+        ).build();
+
+        assertTaskSuccessful(
+            result,
+            ":user1", ":user2", ":alpha:user1", ":alpha:user2", ":bravo:user1", ":bravo:user2"
+        );
+        assertStartedAndStoppedOnce(result);
+        assertOutputOnlyOnce(
+            result.getOutput(),
+            "Starting `node{:alpha:myTestCluster}`",
+            "Stopping `node{::myTestCluster}`"
+        );
+        assertOutputOnlyOnce(
+            result.getOutput(),
+            "Starting `node{::myTestCluster}`",
+            "Stopping `node{:bravo:myTestCluster}`"
+        );
+    }
+
+    public void testIncremental() {
+        BuildResult result = getTestClustersRunner("clean", ":user1").build();
+        assertTaskSuccessful(result, ":user1");
+        assertStartedAndStoppedOnce(result);
+
+        result = getTestClustersRunner(":user1").build();
+        assertTaskSuccessful(result, ":user1");
+        assertStartedAndStoppedOnce(result);
+
+        result = getTestClustersRunner("clean", ":user1").build();
+        assertTaskSuccessful(result, ":user1");
+        assertStartedAndStoppedOnce(result);
         assertStartedAndStoppedOnce(result);
     }
 
     public void testUseClusterByFailingOne() {
-        BuildResult result = getTestClustersRunner("itAlwaysFails").buildAndFail();
+        BuildResult result = getTestClustersRunner(":itAlwaysFails").buildAndFail();
         assertTaskFailed(result, ":itAlwaysFails");
         assertStartedAndStoppedOnce(result);
         assertOutputContains(
@@ -103,7 +130,7 @@ public void testUseClusterByFailingOne() {
     }
 
     public void testUseClusterByFailingDependency() {
-        BuildResult result = getTestClustersRunner("dependsOnFailed").buildAndFail();
+        BuildResult result = getTestClustersRunner(":dependsOnFailed").buildAndFail();
         assertTaskFailed(result, ":itAlwaysFails");
         assertNull(result.task(":dependsOnFailed"));
         assertStartedAndStoppedOnce(result);
@@ -115,7 +142,7 @@ public void testUseClusterByFailingDependency() {
     }
 
     public void testConfigurationLocked() {
-        BuildResult result = getTestClustersRunner("illegalConfigAlter").buildAndFail();
+        BuildResult result = getTestClustersRunner(":illegalConfigAlter").buildAndFail();
         assertTaskFailed(result, ":illegalConfigAlter");
         assertOutputContains(
             result.getOutput(),

diff --git a/buildSrc/src/testKit/testclusters/build.gradle b/buildSrc/src/testKit/testclusters/build.gradle
@@ -1,31 +1,45 @@
 plugins {
     id 'elasticsearch.testclusters'
+    id 'base'
 }
 
-testClusters {
-    myTestCluster {
-        distribution = 'ZIP'
-        version = System.getProperty("test.version_under_test")
+allprojects { all ->
+    repositories {
+        maven {
+            url System.getProperty("local.repo.path")
+        }
+        String luceneSnapshotRevision = System.getProperty("test.lucene-snapshot-revision")
+        if (luceneSnapshotRevision != null) {
+            maven {
+                url "http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/" + luceneSnapshotRevision
+            }
+        }
+        jcenter()
     }
-}
 
-repositories {
-    maven {
-        url System.getProperty("local.repo.path")
-    }
-}
+    if (project == rootProject || project.name == "alpha" || project.name == "bravo") {
+        apply plugin: 'elasticsearch.testclusters'
 
-task user1 {
-    useCluster testClusters.myTestCluster
-    doLast {
-        println "$path: Cluster running @ ${testClusters.myTestCluster.httpSocketURI}"
-    }
-}
+        all.testClusters {
+            myTestCluster {
+                distribution = 'ZIP'
+                version = System.getProperty("test.version_under_test")
+                javaHome = file(System.getProperty('java.home'))
+            }
+        }
 
-task user2 {
-    useCluster testClusters.myTestCluster
-    doLast {
-        println "$path: Cluster running @ ${testClusters.myTestCluster.httpSocketURI}"
+        task user1 {
+            useCluster testClusters.myTestCluster
+            doFirst {
+                println "$path: Cluster running @ ${testClusters.myTestCluster.httpSocketURI}"
+            }
+        }
+        task user2 {
+            useCluster testClusters.myTestCluster
+            doFirst {
+                println "$path: Cluster running @ ${testClusters.myTestCluster.httpSocketURI}"
+            }
+        }
     }
 }
 

diff --git a/buildSrc/src/testKit/testclusters/settings.gradle b/buildSrc/src/testKit/testclusters/settings.gradle
@@ -0,0 +1,4 @@
+include 'dummyPlugin'
+include ':alpha'
+include ':bravo'
+include ':charlie'