diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java index cab250e3417b4..2a60a873079a4 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java @@ -25,7 +25,9 @@ public enum DockerBase { CENTOS("centos:8"), // "latest" here is intentional, since the image name specifies "8" - UBI("docker.elastic.co/ubi8/ubi-minimal:latest"); + UBI("docker.elastic.co/ubi8/ubi-minimal:latest"), + // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build + IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}"); private final String image; diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index def2831c70b0c..b0da0bff74589 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -6,6 +6,9 @@ import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.docker.DockerBuildTask import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin + +import java.nio.file.Path + apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.test.fixtures' apply plugin: 'elasticsearch.internal-distribution-download' @@ -46,6 +49,15 @@ ext.expansions = { Architecture architecture, boolean oss, DockerBase base, bool final String elasticsearch = "elasticsearch-${oss ? 'oss-' : ''}${VersionProperties.elasticsearch}-${classifier}.tar.gz" + String buildArgs = '#' + if (base == DockerBase.IRON_BANK) { + buildArgs = """ +ARG BASE_REGISTRY=nexus-docker-secure.levelup-nexus.svc.cluster.local:18082 +ARG BASE_IMAGE=redhat/ubi/ubi8 +ARG BASE_TAG=8.2 +""" + } + /* Both the following Dockerfile commands put the resulting artifact at * the same location, regardless of classifier, so that the commands that * follow in the Dockerfile don't have to know about the runtime @@ -61,23 +73,40 @@ RUN curl --retry 8 -S -L \\ """.trim() } + def (major,minor) = VersionProperties.elasticsearch.split("\\.") + return [ 'base_image' : base.getImage(), + 'bin_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'bin', + 'build_args' : buildArgs, 'build_date' : BuildParams.buildDate, + 'config_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'config', 'git_revision' : BuildParams.gitRevision, 'license' : oss ? 'Apache-2.0' : 'Elastic-License', 'package_manager' : base == DockerBase.UBI ? 'microdnf' : 'yum', 'source_elasticsearch': sourceElasticsearch, 'docker_base' : base.name().toLowerCase(), - 'version' : VersionProperties.elasticsearch + 'version' : VersionProperties.elasticsearch, + 'major_minor_version' : "${major}.${minor}" ] } +/** + * This filter squashes long runs of newlines so that the output + * is a little more aesthetically pleasing. + */ +class SquashNewlinesFilter extends FilterReader { + SquashNewlinesFilter(Reader input) { + super(new StringReader(input.text.replaceAll("\n{2,}", "\n\n"))) + } +} + private static String buildPath(Architecture architecture, boolean oss, DockerBase base) { return 'build/' + (architecture == Architecture.AARCH64 ? 'aarch64-' : '') + (oss ? 'oss-' : '') + (base == DockerBase.UBI ? 'ubi-' : '') + + (base == DockerBase.UBI ? 'ubi-' : (base == DockerBase.IRON_BANK ? 'ironbank-' : '')) + 'docker' } @@ -85,34 +114,46 @@ private static String taskName(String prefix, Architecture architecture, boolean return prefix + (architecture == Architecture.AARCH64 ? 'Aarch64' : '') + (oss ? 'Oss' : '') + - (base == DockerBase.UBI ? 'Ubi' : '') + + (base == DockerBase.UBI ? 'Ubi' : (base == DockerBase.IRON_BANK ? 'IronBank' : '')) + suffix } project.ext { dockerBuildContext = { Architecture architecture, boolean oss, DockerBase base, boolean local -> copySpec { - into('bin') { - from project.projectDir.toPath().resolve("src/docker/bin") - } - - into('config') { - /* - * The OSS and default distributions have different configurations, therefore we want to allow overriding the default configuration - * from files in the 'oss' sub-directory. We don't want the 'oss' sub-directory to appear in the final build context, however. - */ - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - from(project.projectDir.toPath().resolve("src/docker/config")) { - exclude 'oss' + final Map varExpansions = expansions(architecture, oss, base, local) + final Path projectDir = project.projectDir.toPath() + + if (base == DockerBase.IRON_BANK) { + into('scripts') { + from projectDir.resolve("src/docker/bin") + from(projectDir.resolve("src/docker/config")) { + exclude '**/oss' + } + } + from(projectDir.resolve("src/docker/iron_bank")) { + expand(varExpansions) } - if (oss) { - // Overlay the config file - from project.projectDir.toPath().resolve("src/docker/config/oss") + } else { + into('bin') { + from projectDir.resolve("src/docker/bin") + } + + into('config') { + // The OSS and default distribution can have different configuration, therefore we want to + // allow overriding the default configuration by creating config files in oss or default + // build-context sub-modules. + duplicatesStrategy = DuplicatesStrategy.INCLUDE + from projectDir.resolve("src/docker/config") + if (oss) { + from projectDir.resolve("src/docker/config/oss") + } } } from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) { - expand(expansions(architecture, oss, base, local)) + expand(varExpansions) + filter SquashNewlinesFilter } } } @@ -324,6 +365,8 @@ subprojects { Project subProject -> final Architecture architecture = subProject.name.contains('aarch64-') ? Architecture.AARCH64 : Architecture.X64 final boolean oss = subProject.name.contains('oss-') + // We can ignore Iron Bank at the moment as we don't + // build those images ourselves. final DockerBase base = subProject.name.contains('ubi-') ? DockerBase.UBI : DockerBase.CENTOS final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' diff --git a/distribution/docker/ironbank-docker-build-context/build.gradle b/distribution/docker/ironbank-docker-build-context/build.gradle new file mode 100644 index 0000000000000..00ddbe6ac374f --- /dev/null +++ b/distribution/docker/ironbank-docker-build-context/build.gradle @@ -0,0 +1,14 @@ +import org.elasticsearch.gradle.Architecture +import org.elasticsearch.gradle.DockerBase + +apply plugin: 'base' + +tasks.register("buildIronBankDockerBuildContext", Tar) { + archiveExtension = 'tar.gz' + compression = Compression.GZIP + archiveClassifier = "docker-build-context" + archiveBaseName = "elasticsearch-ironbank" + // We always treat Iron Bank builds as local, because that is how they + // are built + with dockerBuildContext(Architecture.X64, false, DockerBase.IRON_BANK, true) +} diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index ee6e1844c72aa..d36dc339189d8 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -3,6 +3,7 @@ # # Beginning of multi stage Dockerfile ################################################################################ + <% /* This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes have to be escaped in order for them to appear in the final Dockerfile. You @@ -13,13 +14,16 @@ We use control-flow tags in this file to conditionally render the content. The layout/presentation here has been adjusted so that it looks reasonable when rendered, at the slight expense of how it looks here. + + Note that this file is also filtered to squash together newlines, so we can + add as many newlines here as necessary to improve legibility. */ %> + <% if (docker_base == "ubi") { %> ################################################################################ # Build stage 0 `builder`: # Extract Elasticsearch artifact ################################################################################ - FROM ${base_image} AS builder # Install required packages to extract the Elasticsearch distribution @@ -44,7 +48,21 @@ RUN set -eux ; \\ rm \${tini_bin}.sha256sum ; \\ mv \${tini_bin} /bin/tini ; \\ chmod +x /bin/tini + +<% } else if (docker_base == 'iron_bank') { %> +${build_args} + +FROM ${base_image} AS builder + +# `tini` is a tiny but valid init for containers. This is used to cleanly +# control how ES and any child processes are shut down. +COPY tini /bin/tini +RUN chmod 0755 /bin/tini + <% } else { %> + +<% /* CentOS builds are actaully a custom base image with a minimal set of dependencies */ %> + ################################################################################ # Stage 1. Build curl statically. Installing it from RPM on CentOS pulls in too # many dependencies. @@ -194,6 +212,7 @@ COPY --from=curl /work/curl /rootfs/usr/bin/curl # Step 3. Fetch the Elasticsearch distribution and configure it for Docker ################################################################################ FROM ${base_image} AS builder + <% } %> RUN mkdir /usr/share/elasticsearch @@ -202,16 +221,17 @@ WORKDIR /usr/share/elasticsearch # Fetch the appropriate Elasticsearch distribution for this architecture ${source_elasticsearch} -RUN tar zxf /opt/elasticsearch.tar.gz --strip-components=1 +RUN tar -zxf /opt/elasticsearch.tar.gz --strip-components=1 # Configure the distribution for Docker RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env -RUN mkdir -p config config/jvm.options.d data logs +RUN mkdir -p config config/jvm.options.d data logs plugins RUN chmod 0775 config config/jvm.options.d data logs plugins -COPY config/elasticsearch.yml config/log4j2.properties config/ +COPY ${config_dir}/elasticsearch.yml ${config_dir}/log4j2.properties config/ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties -<% if (docker_base == "ubi") { %> +<% if (docker_base == "ubi" || docker_base == "iron_bank") { %> + ################################################################################ # Build stage 1 (the actual Elasticsearch image): # @@ -221,6 +241,8 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties FROM ${base_image} +<% if (docker_base == "ubi") { %> + RUN for iter in {1..10}; do \\ ${package_manager} update --setopt=tsflags=nodocs -y && \\ ${package_manager} install --setopt=tsflags=nodocs -y \\ @@ -231,11 +253,26 @@ RUN for iter in {1..10}; do \\ done; \\ (exit \$exit_code) +%> } else { %> + +<% +/* Reviews of the Iron Bank Dockerfile said that they preferred simpler */ +/* scripting so this version doesn't have the retry loop featured above. */ +%> +RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\ + ${package_manager} install --setopt=tsflags=nodocs -y \\ + nc shadow-utils zip unzip && \\ + ${package_manager} clean all + +<% } %> + RUN groupadd -g 1000 elasticsearch && \\ adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\ chmod 0775 /usr/share/elasticsearch && \\ chown -R 1000:0 /usr/share/elasticsearch + <% } else { %> + ################################################################################ # Stage 4. Build the final image, using the rootfs above as the basis, and # copying in the Elasticsearch distribution @@ -250,13 +287,15 @@ RUN addgroup -g 1000 elasticsearch && \\ addgroup elasticsearch root && \\ chmod 0775 /usr/share/elasticsearch && \\ chgrp 0 /usr/share/elasticsearch + <% } %> ENV ELASTIC_CONTAINER true WORKDIR /usr/share/elasticsearch COPY --from=builder --chown=1000:0 /usr/share/elasticsearch /usr/share/elasticsearch -<% if (docker_base == "ubi") { %> + +<% if (docker_base == "ubi" || docker_base == "iron_bank") { %> COPY --from=builder --chown=0:0 /bin/tini /bin/tini <% } %> @@ -267,7 +306,7 @@ RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk ENV PATH /usr/share/elasticsearch/bin:\$PATH -COPY bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh # 1. The JDK's directories' permissions don't allow `java` to be executed under a different # group to the default. Fix this. @@ -303,7 +342,8 @@ LABEL org.label-schema.build-date="${build_date}" \\ org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\ org.opencontainers.image.vendor="Elastic" \\ org.opencontainers.image.version="${version}" -<% if (docker_base == 'ubi') { %> + +<% if (docker_base == 'ubi' || docker_base == 'iron_bank') { %> LABEL name="Elasticsearch" \\ maintainer="infra@elastic.co" \\ vendor="Elastic" \\ @@ -324,6 +364,10 @@ ENTRYPOINT ["/bin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] # Dummy overridable parameter parsed by entrypoint CMD ["eswrapper"] +<% if (docker_base == 'iron_bank') { %> +HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9200 || exit 1 +<% } %> + ################################################################################ # End of multi-stage Dockerfile ################################################################################ diff --git a/distribution/docker/src/docker/iron_bank/.gitignore b/distribution/docker/src/docker/iron_bank/.gitignore new file mode 100644 index 0000000000000..cda31c3986d13 --- /dev/null +++ b/distribution/docker/src/docker/iron_bank/.gitignore @@ -0,0 +1,2 @@ +# Ignore any locally downloaded or dropped releases +*.tar.gz diff --git a/distribution/docker/src/docker/iron_bank/Jenkinsfile b/distribution/docker/src/docker/iron_bank/Jenkinsfile new file mode 100644 index 0000000000000..7422f1f7a2a9c --- /dev/null +++ b/distribution/docker/src/docker/iron_bank/Jenkinsfile @@ -0,0 +1,2 @@ +@Library('DCCSCR@master') _ +dccscrPipeline(version: '${version}') diff --git a/distribution/docker/src/docker/iron_bank/README.md b/distribution/docker/src/docker/iron_bank/README.md new file mode 100644 index 0000000000000..6767a6faa66f8 --- /dev/null +++ b/distribution/docker/src/docker/iron_bank/README.md @@ -0,0 +1,37 @@ +# Elasticsearch + +**Elasticsearch** is a distributed, RESTful search and analytics engine capable of +solving a growing number of use cases. As the heart of the Elastic Stack, it +centrally stores your data so you can discover the expected and uncover the +unexpected. + +For more information about Elasticsearch, please visit +https://www.elastic.co/products/elasticsearch. + +### Installation instructions + +Please follow the documentation on [how to install Elasticsearch with Docker](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html). + +### Where to file issues and PRs + +- [Issues](https://github.com/elastic/elasticsearch/issues) +- [PRs](https://github.com/elastic/elasticsearch/pulls) + +### Where to get help + +- [Elasticsearch Discuss Forums](https://discuss.elastic.co/c/elasticsearch) +- [Elasticsearch Documentation](https://www.elastic.co/guide/en/elasticsearch/reference/master/index.html) + +### Still need help? + +You can learn more about the Elastic Community and also understand how to get more help +visiting [Elastic Community](https://www.elastic.co/community). + + +This software is governed by the [Elastic +License](https://github.com/elastic/elasticsearch/blob/${major_minor_version}/licenses/ELASTIC-LICENSE.txt), +and includes the full set of [free +features](https://www.elastic.co/subscriptions). + +View the detailed release notes +[here](https://www.elastic.co/guide/en/elasticsearch/reference/${major_minor_version}/es-release-notes.html). diff --git a/distribution/docker/src/docker/iron_bank/download.json b/distribution/docker/src/docker/iron_bank/download.json new file mode 100644 index 0000000000000..63f61a289b0e7 --- /dev/null +++ b/distribution/docker/src/docker/iron_bank/download.json @@ -0,0 +1,20 @@ +{ + "resources": [ + { + "url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-x86_64.tar.gz", + "filename": "elasticsearch-${version}-linux-x86_64.tar.gz", + "validation": { + "type": "sha512", + "value": "" + } + }, + { + "url": "https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64", + "filename": "tini", + "validation": { + "type": "sha256", + "value": "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c" + } + } + ] +} diff --git a/settings.gradle b/settings.gradle index 1404db8846fe4..c8881d73adb70 100644 --- a/settings.gradle +++ b/settings.gradle @@ -38,6 +38,7 @@ List projects = [ 'distribution:docker:docker-aarch64-export', 'distribution:docker:docker-build-context', 'distribution:docker:docker-export', + 'distribution:docker:ironbank-docker-build-context', 'distribution:docker:oss-docker-aarch64-build-context', 'distribution:docker:oss-docker-aarch64-export', 'distribution:docker:oss-docker-build-context',