Enable Dockerfile from artifacts.elastic.co 6.7 (#38587)

This commit enables the copyDockerfile task to render a Dockerfile that
sources the Elasticsearch binary from artifacts.elastic.co. This is
needed for reproducibility and transparency for the official Docker
images in the Docker library.

distribution/docker/build.gradle

@@ -18,15 +18,34 @@ dependencies {
 }
 
 ext.expansions = { oss ->
+  final String elasticsearch = oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz"
   return [
-    'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz",
-    'jdkUrl' : 'https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz',
-    'jdkVersion' : '11.0.2',
-    'license': oss ? 'Apache-2.0' : 'Elastic License',
-    'version' : VersionProperties.elasticsearch
+    'elasticsearch'       : elasticsearch,
+    'jdkUrl'              : 'https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz',
+    'jdkVersion'          : '11.0.2',
+    'license'             : oss ? 'Apache-2.0' : 'Elastic License',
+    'source_elasticsearch': local() ? "COPY $elasticsearch /opt/" : "RUN cd /opt && curl --retry 8 -s -L -O https://artifacts.elastic.co/downloads/elasticsearch/${elasticsearch} && cd -",
+    'version'             : VersionProperties.elasticsearch
   ]
 }
 
+/*
+ * We need to be able to render a Dockerfile that references the official artifacts on https://artifacts.elastic.co. For this, we use a
+ * substitution in the Dockerfile template where we can either replace source_elasticsearch with a COPY from the Docker build context, or
+ * a RUN curl command to retrieve the artifact from https://artifacts.elastic.co. The system property build.docker.source, which can be
+ * either "local" (default) or "remote" controls which version of the Dockerfile is produced.
+ */
+private static boolean local() {
+  final String buildDockerSource = System.getProperty("build.docker.source")
+  if (buildDockerSource == null || "local".equals(buildDockerSource)) {
+    return true
+  } else if ("remote".equals(buildDockerSource)) {
+    return false
+  } else {
+    throw new IllegalArgumentException("expected build.docker.source to be [local] or [remote] but was [" + buildDockerSource + "]")
+  }
+}
+
 private static String files(final boolean oss) {
   return "build/${ oss ? 'oss-' : ''}docker"
 }
@@ -47,20 +66,22 @@ void addCopyDockerContextTask(final boolean oss) {
       from 'src/docker/config'
     }
 
-    if (oss) {
-      from configurations.ossDockerSource
-    } else {
-      from configurations.dockerSource
-    }
+    if (local()) {
+      if (oss) {
+        from configurations.ossDockerSource
+      } else {
+        from configurations.dockerSource
+      }
 
-    from configurations.dockerPlugins
+      from configurations.dockerPlugins
+    }
   }
 }
 
 void addCopyDockerfileTask(final boolean oss) {
   task(taskName("copy", oss, "Dockerfile"), type: Copy) {
+    dependsOn taskName("copy", oss, "DockerContext")
     inputs.properties(expansions(oss)) // ensure task is run when ext.expansions is changed
-    mustRunAfter(taskName("copy", oss, "DockerContext"))
     into files(oss)
 
     from('src/docker/Dockerfile') {
@@ -69,7 +90,6 @@ void addCopyDockerfileTask(final boolean oss) {
   }
 }
 
-
 preProcessFixture {
   dependsOn taskName("copy", true, "DockerContext")
   dependsOn taskName("copy", true, "Dockerfile")
@@ -87,7 +107,6 @@ check.dependsOn postProcessFixture
 
 void addBuildDockerImage(final boolean oss) {
   final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: LoggedExec) {
-    dependsOn taskName("copy", oss, "DockerContext")
     dependsOn taskName("copy", oss, "Dockerfile")
     List<String> tags
     if (oss) {

distribution/docker/src/docker/Dockerfile

@@ -30,13 +30,13 @@ RUN groupadd -g 1000 elasticsearch && \
 
 WORKDIR /usr/share/elasticsearch
 
-COPY ${elasticsearch} /opt/
+${source_elasticsearch}
+
 RUN tar zxf /opt/${elasticsearch} --strip-components=1
 RUN mkdir -p config data logs
 RUN chmod 0775 config data logs
 COPY config/elasticsearch.yml config/log4j2.properties config/
 
-
 ################################################################################
 # Build stage 1 (the actual elasticsearch image):
 # Copy elasticsearch from stage 0