Use project dependency instead of substitutions for distributions

build.gradle

@@ -221,14 +221,6 @@ allprojects {
     "org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
     "org.elasticsearch.test:framework:${version}": ':test:framework',
     "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
-    "downloads.zip:elasticsearch:${version}": ':distribution:archives:zip',
-    "downloads.zip:elasticsearch-oss:${version}": ':distribution:archives:oss-zip',
-    "downloads.tar:elasticsearch:${version}": ':distribution:archives:tar',
-    "downloads.tar:elasticsearch-oss:${version}": ':distribution:archives:oss-tar',
-    "downloads.rpm:elasticsearch:${version}": ':distribution:packages:rpm',
-    "downloads.rpm:elasticsearch-oss:${version}": ':distribution:packages:oss-rpm',
-    "downloads.deb:elasticsearch:${version}": ':distribution:packages:deb',
-    "downloads.deb:elasticsearch-oss:${version}": ':distribution:packages:oss-deb',
     "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
     "org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
     // for transport client
@@ -240,19 +232,6 @@ allprojects {
     "org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
     "org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
   ]
-  // substitute unreleased versions with projects that check out and build locally
-  bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unreleasedVersion ->
-    Version unreleased = unreleasedVersion.version
-    String snapshotProject = ":distribution:bwc:${unreleasedVersion.gradleProjectName}"
-    ext.projectSubstitutions["downloads.deb:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["downloads.rpm:elasticsearch:${unreleased}"] = snapshotProject
-    ext.projectSubstitutions["downloads.zip:elasticsearch:${unreleased}"] = snapshotProject
-    if (unreleased.onOrAfter('6.3.0')) {
-      ext.projectSubstitutions["downloads.deb:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["downloads.rpm:elasticsearch-oss:${unreleased}"] = snapshotProject
-      ext.projectSubstitutions["downloads.zip:elasticsearch-oss:${unreleased}"] = snapshotProject
-    }
-  }
 
   /*
    * Gradle only resolve project substitutions during dependency resolution but

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
@@ -171,6 +172,12 @@ class ClusterFormationTasks {
 
     /** Adds a dependency on the given distribution */
     static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
+        if (distro.equals("integ-test-zip")) {
+            // short circuit integ test so it doesn't complicate the rest of the distribution setup below
+            project.dependencies.add(configuration.name,
+                    "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${elasticsearchVersion}@zip")
+            return
+        }
         // TEMP HACK
         // The oss docs CI build overrides the distro on the command line. This hack handles backcompat until CI is updated.
         if (distro.equals('oss-zip')) {
@@ -180,22 +187,31 @@ class ClusterFormationTasks {
             distro = 'default'
         }
         // END TEMP HACK
-        if (['integ-test-zip', 'oss', 'default'].contains(distro) == false) {
+        if (['oss', 'default'].contains(distro) == false) {
             throw new GradleException("Unknown distribution: ${distro} in project ${project.path}")
         }
         Version version = Version.fromString(elasticsearchVersion)
-        if (version.before('6.3.0') && distro.startsWith('oss-')) {
-            distro = distro.substring('oss-'.length())
-        }
-        String group = "downloads.zip"
-        if (distro.equals("integ-test-zip")) {
-            group = "org.elasticsearch.distribution.integ-test-zip"
-        }
+        String group = "downloads.zip" // dummy group, does not matter except for integ-test-zip, it is ignored by the fake ivy repo
         String artifactName = 'elasticsearch'
         if (distro.equals('oss') && Version.fromString(elasticsearchVersion).onOrAfter('6.3.0')) {
             artifactName += '-oss'
         }
-        project.dependencies.add(configuration.name, "${group}:${artifactName}:${elasticsearchVersion}@zip")
+        String snapshotProject = distro == 'oss' ? 'oss-zip' : 'zip'
+        Object dependency
+        boolean internalBuild = project.hasProperty('bwcVersions')
+        VersionCollection.UnreleasedVersionInfo unreleasedInfo = null
+        if (project.hasProperty('bwcVersions')) {
+            // NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools?
+            unreleasedInfo = project.bwcVersions.unreleasedInfo(version)
+        }
+        if (unreleasedInfo != null) {
+            dependency = project.dependencies.project(path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: snapshotProject)
+        } else if (internalBuild && elasticsearchVersion.equals(VersionProperties.elasticsearch)) {
+            dependency = project.dependencies.project(path: ":distribution:archives:${snapshotProject}")
+        } else {
+            dependency = "${group}:${artifactName}:${elasticsearchVersion}@zip"
+        }
+        project.dependencies.add(configuration.name, dependency)
     }
 
     /** Adds a dependency on a different version of the given plugin, which will be retrieved using gradle's dependency resolution */

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -4,6 +4,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.FileContentsTask
 import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionCollection
 import org.gradle.api.*
 import org.gradle.api.artifacts.dsl.RepositoryHandler
 import org.gradle.api.execution.TaskExecutionAdapter
@@ -184,22 +185,38 @@ class VagrantTestPlugin implements Plugin<Project> {
             upgradeFromVersion = Version.fromString(upgradeFromVersionRaw)
         }
 
+        List<Object> dependencies = new ArrayList<>()
         DISTRIBUTIONS.each {
             // Adds a dependency for the current version
-            project.dependencies.add(PACKAGING_CONFIGURATION,
-                    project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
+            dependencies.add(project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
         }
 
-        UPGRADE_FROM_ARCHIVES.each {
-            // The version of elasticsearch that we upgrade *from*
-            project.dependencies.add(PACKAGING_CONFIGURATION,
-                    "downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
-            if (upgradeFromVersion.onOrAfter('6.3.0')) {
-                project.dependencies.add(PACKAGING_CONFIGURATION,
-                        "downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+        // The version of elasticsearch that we upgrade *from*
+        VersionCollection.UnreleasedVersionInfo unreleasedInfo = project.bwcVersions.unreleasedInfo(upgradeFromVersion)
+        if (unreleasedInfo != null) {
+            // handle snapshots pointing to bwc build
+            UPGRADE_FROM_ARCHIVES.each {
+                dependencies.add(project.dependencies.project(
+                        path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: it))
+                if (upgradeFromVersion.onOrAfter('6.3.0')) {
+                    dependencies.add(project.dependencies.project(
+                            path: ":distribution:bwc:${unreleasedInfo.gradleProjectName}", configuration: "oss-${it}"))
+                }
+            }
+        } else {
+            UPGRADE_FROM_ARCHIVES.each {
+                // The version of elasticsearch that we upgrade *from*
+                dependencies.add("downloads.${it}:elasticsearch:${upgradeFromVersion}@${it}")
+                if (upgradeFromVersion.onOrAfter('6.3.0')) {
+                    dependencies.add("downloads.${it}:elasticsearch-oss:${upgradeFromVersion}@${it}")
+                }
             }
         }
 
+        for (Object dependency : dependencies) {
+            project.dependencies.add(PACKAGING_CONFIGURATION, dependency)
+        }
+
         project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
     }
 

buildSrc/src/main/java/org/elasticsearch/gradle/VersionCollection.java

@@ -20,6 +20,8 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -86,6 +88,7 @@ public class VersionCollection {
 
     private final Version currentVersion;
     private final Map<Integer, List<Version>> groupByMajor;
+    private final Map<Version, UnreleasedVersionInfo> unreleased;
 
     public class UnreleasedVersionInfo {
         public final Version version;
@@ -129,6 +132,16 @@ protected VersionCollection(List<String> versionLines, Version currentVersionPro
         assertCurrentVersionMatchesParsed(currentVersionProperty);
 
         assertNoOlderThanTwoMajors();
+
+        Map<Version, UnreleasedVersionInfo> unreleased = new HashMap<>();
+        for (Version unreleasedVersion : getUnreleased()) {
+            if (unreleasedVersion.equals(currentVersion)) {
+                continue;
+            }
+            unreleased.put(unreleasedVersion,
+                new UnreleasedVersionInfo(unreleasedVersion, getBranchFor(unreleasedVersion), getGradleProjectNameFor(unreleasedVersion)));
+        }
+        this.unreleased = Collections.unmodifiableMap(unreleased);
     }
 
     private void assertNoOlderThanTwoMajors() {
@@ -150,6 +163,13 @@ private void assertCurrentVersionMatchesParsed(Version currentVersionProperty) {
         }
     }
 
+    /**
+      * Returns info about the unreleased version, or {@code null} if the version is released.
+      */
+    public UnreleasedVersionInfo unreleasedInfo(Version version) {
+        return unreleased.get(version);
+    }
+
     public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
         getUnreleased().stream()
             .filter(version -> version.equals(currentVersion) == false)

distribution/bwc/build.gradle

@@ -120,18 +120,19 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
         }
     }
 
-    List<File> artifactFiles = []
+    Map<String, File> artifactFiles = [:]
     List<String> projectDirs = []
-    for (String project : ['zip', 'deb', 'rpm']) {
+    List<String> projects = ['zip', 'deb', 'rpm']
+    for (String projectName : projects) {
         String baseDir = "distribution"
         if (bwcVersion.onOrAfter('6.3.0')) {
-            baseDir += project == 'zip' ? '/archives' : '/packages'
+            baseDir += projectName == 'zip' ? '/archives' : '/packages'
             // add oss variant first
-            projectDirs.add("${baseDir}/oss-${project}")
-            artifactFiles.add(file("${checkoutDir}/${baseDir}/oss-${project}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${project}"))
+            projectDirs.add("${baseDir}/oss-${projectName}")
+            artifactFiles.put("oss-" + projectName, file("${checkoutDir}/${baseDir}/oss-${projectName}/build/distributions/elasticsearch-oss-${bwcVersion}-SNAPSHOT.${projectName}"))
         }
-        projectDirs.add("${baseDir}/${project}")
-        artifactFiles.add(file("${checkoutDir}/${baseDir}/${project}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${project}"))
+        projectDirs.add("${baseDir}/${projectName}")
+        artifactFiles.put(projectName, file("${checkoutDir}/${baseDir}/${projectName}/build/distributions/elasticsearch-${bwcVersion}-SNAPSHOT.${projectName}"))
     }
 
     task buildBwcVersion(type: Exec) {
@@ -187,7 +188,7 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
         standardOutput = new IndentingOutputStream(System.out, bwcVersion)
         errorOutput = new IndentingOutputStream(System.err, bwcVersion)
         doLast {
-            List missing = artifactFiles.grep { file ->
+            List missing = artifactFiles.values().grep { file ->
                 false == file.exists()
             }
             if (false == missing.empty) {
@@ -197,12 +198,16 @@ bwcVersions.forPreviousUnreleased { VersionCollection.UnreleasedVersionInfo unre
         }
     }
 
-    artifacts {
-      for (File artifactFile : artifactFiles) {
-        String artifactName = artifactFile.name.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
+    for (e in artifactFiles) {
+        String projectName = e.key
+        File artifactFile = e.value
+        String artifactFileName = artifactFile.name
+        String artifactName = artifactFileName.contains('oss') ? 'elasticsearch-oss' : 'elasticsearch'
         String suffix = artifactFile.toString()[-3..-1]
-        'default' file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion
-      }
+        configurations.create(projectName)
+        artifacts {
+            it.add(projectName, [file: artifactFile, name: artifactName, type: suffix, builtBy: buildBwcVersion])
+        }
     }
     // make sure no dependencies were added to assemble; we want it to be a no-op
     assemble.dependsOn = []