Support building Iron Bank Docker context (elastic#64336)

This PR adds support for building a Docker context for Iron Bank.
It doesn't actually build the image - we could add that at a later
stage, but this is an attempt to automate at least some of the
process.

Iron Bank is a lot like our UBI build, except it uses a hardened
version of the full UBI image, not the minimal UBI image. They have
particular requirements around how the Docker context should be
arranged. The Docker build cannot fetch its own artefacts, but
instead the context provides a descriptor that locates what is
needed for the build.

I also added a filter so that after performing expansions on the
`Dockerfile`, we squash long runs on newlines together. This makes
the output cleaner, while allowing us to break up the unprocessed
`Dockerfile` for clarity.

buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java

@@ -25,7 +25,9 @@
 public enum DockerBase {
     CENTOS("centos:8"),
     // "latest" here is intentional, since the image name specifies "8"
-    UBI("docker.elastic.co/ubi8/ubi-minimal:latest");
+    UBI("docker.elastic.co/ubi8/ubi-minimal:latest"),
+    // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build
+    IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}");
 
     private final String image;
 

distribution/docker/build.gradle

@@ -6,6 +6,9 @@ import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.docker.DockerBuildTask
 import org.elasticsearch.gradle.info.BuildParams
 import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin
+
+import java.nio.file.Path
+
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.test.fixtures'
 apply plugin: 'elasticsearch.internal-distribution-download'
@@ -46,6 +49,15 @@ ext.expansions = { Architecture architecture, boolean oss, DockerBase base, bool
 
   final String elasticsearch = "elasticsearch-${oss ? 'oss-' : ''}${VersionProperties.elasticsearch}-${classifier}.tar.gz"
 
+  String buildArgs = ''
+  if (base == DockerBase.IRON_BANK) {
+    buildArgs = """
+ARG BASE_REGISTRY=nexus-docker-secure.levelup-nexus.svc.cluster.local:18082
+ARG BASE_IMAGE=redhat/ubi/ubi8
+ARG BASE_TAG=8.2
+"""
+  }
+
   /* Both the following Dockerfile commands put the resulting artifact at
    * the same location, regardless of classifier, so that the commands that
    * follow in the Dockerfile don't have to know about the runtime
@@ -61,58 +73,98 @@ RUN curl --retry 8 -S -L \\
 """
   }
 
+  def (major,minor) = VersionProperties.elasticsearch.split("\\.")
+
   return [
     'base_image'          : base.getImage(),
+    'bin_dir'             : base == DockerBase.IRON_BANK ? 'scripts' : 'bin',
+    'build_args'          : buildArgs,
     'build_date'          : BuildParams.buildDate,
+    'config_dir'          : base == DockerBase.IRON_BANK ? 'scripts' : 'config',
     'git_revision'        : BuildParams.gitRevision,
     'license'             : oss ? 'Apache-2.0' : 'Elastic-License',
     'package_manager'     : base == DockerBase.UBI ? 'microdnf' : 'yum',
     'source_elasticsearch': sourceElasticsearch,
     'docker_base'         : base.name().toLowerCase(),
-    'version'             : VersionProperties.elasticsearch
+    'version'             : VersionProperties.elasticsearch,
+    'major_minor_version' : "${major}.${minor}"
   ]
 }
 
+/**
+ * This filter squashes long runs of newlines so that the output
+ * is a little more aesthetically pleasing.
+ */
+class SquashNewlinesFilter extends FilterReader {
+  SquashNewlinesFilter(Reader input) {
+    super(new StringReader(input.text.replaceAll("\n{2,}", "\n\n")))
+  }
+}
+
 private static String buildPath(Architecture architecture, boolean oss, DockerBase base) {
   return 'build/' +
     (architecture == Architecture.AARCH64 ? 'aarch64-' : '') +
     (oss ? 'oss-' : '') +
     (base == DockerBase.UBI ? 'ubi-' : '') +
+    (base == DockerBase.UBI ? 'ubi-' : (base == DockerBase.IRON_BANK ? 'ironbank-' : '')) +
     'docker'
 }
 
 private static String taskName(String prefix, Architecture architecture, boolean oss, DockerBase base, String suffix) {
   return prefix +
     (architecture == Architecture.AARCH64 ? 'Aarch64' : '') +
     (oss ? 'Oss' : '') +
-    (base == DockerBase.UBI ? 'Ubi' : '') +
+    (base == DockerBase.UBI ? 'Ubi' : (base == DockerBase.IRON_BANK ? 'IronBank' : '')) +
     suffix
 }
 
 project.ext {
   dockerBuildContext = { Architecture architecture, boolean oss, DockerBase base, boolean local ->
     copySpec {
-      into('bin') {
-        from project.projectDir.toPath().resolve("src/docker/bin")
-      }
-
-      into('config') {
-        /*
-         * The OSS and default distributions have different configurations, therefore we want to allow overriding the default configuration
-         * from files in the 'oss' sub-directory. We don't want the 'oss' sub-directory to appear in the final build context, however.
-         */
-        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-        from(project.projectDir.toPath().resolve("src/docker/config")) {
-          exclude 'oss'
+      final Map<String,String> varExpansions = expansions(architecture, oss, base, local)
+      final Path projectDir = project.projectDir.toPath()
+
+      if (base == DockerBase.IRON_BANK) {
+        into('scripts') {
+          from(projectDir.resolve("src/docker/bin")) {
+            // We need to use an entrypoint that doesn't expect to run as `root` for Iron Bank,
+            // so don't copy this one. It's actually the version from v8.0, so this won't be
+            // necessary in future versions.
+            exclude 'docker-entrypoint.sh'
+          }
+          from(projectDir.resolve("src/docker/config")) {
+            exclude '**/oss'
+          }
+          from(projectDir.resolve("src/docker/iron_bank")) {
+            expand(varExpansions)
+            // Exclude this script so that no expansions are performed
+            exclude 'docker-entrypoint.sh'
+          }
+          from(projectDir.resolve("src/docker/iron_bank")) {
+            // ...and now copy the entrypoint without expansions
+            include 'docker-entrypoint.sh'
+          }
         }
-        if (oss) {
-          // Overlay the config file
-          from project.projectDir.toPath().resolve("src/docker/config/oss")
+      } else {
+        into('bin') {
+          from projectDir.resolve("src/docker/bin")
+        }
+
+        into('config') {
+          // The OSS and default distribution can have different configuration, therefore we want to
+          // allow overriding the default configuration by creating config files in oss or default
+          // build-context sub-modules.
+          duplicatesStrategy = DuplicatesStrategy.INCLUDE
+          from projectDir.resolve("src/docker/config")
+          if (oss) {
+            from projectDir.resolve("src/docker/config/oss")
+          }
         }
       }
 
       from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) {
-        expand(expansions(architecture, oss, base, local))
+        expand(varExpansions)
+        filter SquashNewlinesFilter
       }
     }
   }
@@ -324,6 +376,8 @@ subprojects { Project subProject ->
 
     final Architecture architecture = subProject.name.contains('aarch64-') ? Architecture.AARCH64 : Architecture.X64
     final boolean oss = subProject.name.contains('oss-')
+    // We can ignore Iron Bank at the moment as we don't
+    // build those images ourselves.
     final DockerBase base = subProject.name.contains('ubi-') ? DockerBase.UBI : DockerBase.CENTOS
 
     final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''

distribution/docker/ironbank-docker-build-context/build.gradle

@@ -0,0 +1,14 @@
+import org.elasticsearch.gradle.Architecture
+import org.elasticsearch.gradle.DockerBase
+
+apply plugin: 'base'
+
+tasks.register("buildIronBankDockerBuildContext", Tar) {
+  archiveExtension = 'tar.gz'
+  compression = Compression.GZIP
+  archiveClassifier = "docker-build-context"
+  archiveBaseName = "elasticsearch-ironbank"
+  // We always treat Iron Bank builds as local, because that is how they
+  // are built
+  with dockerBuildContext(Architecture.X64, false, DockerBase.IRON_BANK, true)
+}

distribution/docker/src/docker/Dockerfile

@@ -3,6 +3,7 @@
 #
 # Beginning of multi stage Dockerfile
 ################################################################################
+
 <% /*
   This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes
   have to be escaped in order for them to appear in the final Dockerfile. You
@@ -13,17 +14,31 @@
   We use control-flow tags in this file to conditionally render the content. The
   layout/presentation here has been adjusted so that it looks reasonable when rendered,
   at the slight expense of how it looks here.
+
+  Note that this file is also filtered to squash together newlines, so we can
+  add as many newlines here as necessary to improve legibility.
 */ %>
 ################################################################################
 # Build stage 0 `builder`:
 # Extract Elasticsearch artifact
 ################################################################################
 
+${build_args}
+
 FROM ${base_image} AS builder
+
 <% if (docker_base == 'ubi') { %>
 # Install required packages to extract the Elasticsearch distribution
 RUN ${package_manager} install -y tar gzip
 <% } %>
+
+<% if (docker_base == 'iron_bank') { %>
+# `tini` is a tiny but valid init for containers. This is used to cleanly
+# control how ES and any child processes are shut down.
+COPY tini /bin/tini
+RUN chmod 0755 /bin/tini
+<% } else { %>
+
 # `tini` is a tiny but valid init for containers. This is used to cleanly
 # control how ES and any child processes are shut down.
 #
@@ -41,19 +56,23 @@ RUN set -eux ; \\
     curl --retry 8 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\
     sha256sum -c \${tini_bin}.sha256sum ; \\
     rm \${tini_bin}.sha256sum ; \\
-    mv \${tini_bin} /tini ; \\
-    chmod +x /tini
+    mv \${tini_bin} /bin/tini ; \\
+    chmod +x /bin/tini
+
+<% } %>
 
 RUN mkdir /usr/share/elasticsearch
 WORKDIR /usr/share/elasticsearch
 
 ${source_elasticsearch}
 
-RUN tar zxf /opt/elasticsearch.tar.gz --strip-components=1
+RUN tar -zxf /opt/elasticsearch.tar.gz --strip-components=1
+
+# Configure the distribution for Docker
 RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env
-RUN mkdir -p config config/jvm.options.d data logs
+RUN mkdir -p config config/jvm.options.d data logs plugins
 RUN chmod 0775 config config/jvm.options.d data logs plugins
-COPY config/elasticsearch.yml config/log4j2.properties config/
+COPY ${config_dir}/elasticsearch.yml ${config_dir}/log4j2.properties config/
 RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
 
 ################################################################################
@@ -65,7 +84,17 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
 
 FROM ${base_image}
 
-ENV ELASTIC_CONTAINER true
+<% if (docker_base == "iron_bank") { %>
+<%
+/* Reviews of the Iron Bank Dockerfile said that they preferred simpler */
+/* scripting so this version doesn't have the retry loop featured below. */
+%>
+RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\
+    ${package_manager} install --setopt=tsflags=nodocs -y \\
+      nc shadow-utils zip unzip && \\
+    ${package_manager} clean all
+
+<% } else { %>
 
 RUN for iter in {1..10}; do \\
       ${package_manager} update --setopt=tsflags=nodocs -y && \\
@@ -76,14 +105,18 @@ RUN for iter in {1..10}; do \\
     done; \\
     (exit \$exit_code)
 
+<% } %>
+
 RUN groupadd -g 1000 elasticsearch && \\
     adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\
     chmod 0775 /usr/share/elasticsearch && \\
     chown -R 1000:0 /usr/share/elasticsearch
 
+ENV ELASTIC_CONTAINER true
+
 WORKDIR /usr/share/elasticsearch
 COPY --from=builder --chown=1000:0 /usr/share/elasticsearch /usr/share/elasticsearch
-COPY --from=builder --chown=0:0 /tini /tini
+COPY --from=builder --chown=0:0 /bin/tini /bin/tini
 
 # Replace OpenJDK's built-in CA certificate keystore with the one from the OS
 # vendor. The latter is superior in several ways.
@@ -92,7 +125,7 @@ RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk
 
 ENV PATH /usr/share/elasticsearch/bin:\$PATH
 
-COPY bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
 # 1. The JDK's directories' permissions don't allow `java` to be executed under a different
 #    group to the default. Fix this.
@@ -128,7 +161,8 @@ LABEL org.label-schema.build-date="${build_date}" \\
   org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\
   org.opencontainers.image.vendor="Elastic" \\
   org.opencontainers.image.version="${version}"
-<% if (docker_base == 'ubi') { %>
+
+<% if (docker_base == 'ubi' || docker_base == 'iron_bank') { %>
 LABEL name="Elasticsearch" \\
   maintainer="infra@elastic.co" \\
   vendor="Elastic" \\
@@ -145,6 +179,10 @@ ENTRYPOINT ["/tini", "--", "/usr/local/bin/docker-entrypoint.sh"]
 # Dummy overridable parameter parsed by entrypoint
 CMD ["eswrapper"]
 
+<% if (docker_base == 'iron_bank') { %>
+HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9200 || exit 1
+<% } %>
+
 ################################################################################
 # End of multi-stage Dockerfile
 ################################################################################

distribution/docker/src/docker/iron_bank/.gitignore

@@ -0,0 +1,2 @@
+# Ignore any locally downloaded or dropped releases
+*.tar.gz

distribution/docker/src/docker/iron_bank/Jenkinsfile

@@ -0,0 +1,2 @@
+@Library('DCCSCR@master') _
+dccscrPipeline(version: '${version}')

distribution/docker/src/docker/iron_bank/README.md

@@ -0,0 +1,37 @@
+# Elasticsearch
+
+**Elasticsearch** is a distributed, RESTful search and analytics engine capable of
+solving a growing number of use cases. As the heart of the Elastic Stack, it
+centrally stores your data so you can discover the expected and uncover the
+unexpected.
+
+For more information about Elasticsearch, please visit
+https://www.elastic.co/products/elasticsearch.
+
+### Installation instructions
+
+Please follow the documentation on [how to install Elasticsearch with Docker](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html).
+
+### Where to file issues and PRs
+
+- [Issues](https://github.com/elastic/elasticsearch/issues)
+- [PRs](https://github.com/elastic/elasticsearch/pulls)
+
+### Where to get help
+
+- [Elasticsearch Discuss Forums](https://discuss.elastic.co/c/elasticsearch)
+- [Elasticsearch Documentation](https://www.elastic.co/guide/en/elasticsearch/reference/master/index.html)
+
+### Still need help?
+
+You can learn more about the Elastic Community and also understand how to get more help
+visiting [Elastic Community](https://www.elastic.co/community).
+
+
+This software is governed by the [Elastic
+License](https://github.com/elastic/elasticsearch/blob/${major_minor_version}/licenses/ELASTIC-LICENSE.txt),
+and includes the full set of [free
+features](https://www.elastic.co/subscriptions).
+
+View the detailed release notes
+[here](https://www.elastic.co/guide/en/elasticsearch/reference/${major_minor_version}/es-release-notes.html).