From e703b1756d1e5487d07b93b2befbb013c37d7fc1 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 29 Jul 2020 16:14:58 +0100 Subject: [PATCH 01/32] Reinstate UBI builds --- .../elasticsearch/gradle/Architecture.java | 6 ++ .../InternalDistributionDownloadPlugin.java | 4 +- distribution/docker/build.gradle | 95 ++++++++++++------- .../docker/docker-build-context/build.gradle | 3 +- .../oss-docker-build-context/build.gradle | 3 +- distribution/docker/src/docker/Dockerfile | 38 +++++--- .../ubi-docker-build-context/build.gradle | 12 +++ .../docker/ubi-docker-x64-export/build.gradle | 2 + settings.gradle | 2 + 9 files changed, 111 insertions(+), 54 deletions(-) create mode 100644 distribution/docker/ubi-docker-build-context/build.gradle create mode 100644 distribution/docker/ubi-docker-x64-export/build.gradle diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java index f230d9af86e11..0cce78e4ec8e6 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java @@ -19,6 +19,8 @@ package org.elasticsearch.gradle; +import java.util.Locale; + public enum Architecture { X64, @@ -37,4 +39,8 @@ public static Architecture current() { } } + @Override + public String toString() { + return super.toString().toLowerCase(Locale.ROOT); + } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java index 9399f1f062ca0..bc1372fb641f3 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -127,7 +127,7 @@ private static String distributionProjectName(ElasticsearchDistribution distribu final String archString = platform == ElasticsearchDistribution.Platform.WINDOWS || architecture == Architecture.X64 ? "" - : "-" + architecture.toString().toLowerCase(); + : "-" + architecture; if (distribution.getFlavor() == ElasticsearchDistribution.Flavor.OSS) { projectName += "oss-"; @@ -139,7 +139,7 @@ private static String distributionProjectName(ElasticsearchDistribution distribu switch (distribution.getType()) { case ARCHIVE: - projectName += platform.toString() + archString + (platform == ElasticsearchDistribution.Platform.WINDOWS + projectName += platform + archString + (platform == ElasticsearchDistribution.Platform.WINDOWS ? "-zip" : "-tar"); break; diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 34f78f3206ebc..f21f47106522f 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -28,14 +28,14 @@ dependencies { ossDockerSource project(path: ":distribution:archives:oss-linux-tar") } -ext.expansions = { architecture, oss, local -> +ext.expansions = { Architecture architecture, boolean oss, boolean ubi, boolean local -> String classifier if (local) { switch (architecture) { - case "aarch64": + case Architecture.AARCH64: classifier = "linux-aarch64" break - case "x64": + case Architecture.X64: classifier = "linux-x86_64" break default: @@ -66,24 +66,46 @@ RUN curl --retry 8 -S -L \\ } return [ + 'base_image' : ubi ? 'registry.access.redhat.com/ubi7/ubi-minimal:7.7' : 'centos:7', 'build_date' : BuildParams.buildDate, 'git_revision' : BuildParams.gitRevision, 'license' : oss ? 'Apache-2.0' : 'Elastic-License', - 'source_elasticsearch': sourceElasticsearch, + 'package_manager' : ubi ? 'microdnf' : 'yum', + 'source_elasticsearch': sourceElasticsearch.trim(), 'version' : VersionProperties.elasticsearch ] } -private static String buildPath(final String architecture, final boolean oss) { - return "build/${"aarch64".equals(architecture) ? 'aarch64-' : ''}${oss ? 'oss-' : ''}docker" +private static String buildPath(final Architecture architecture, final boolean oss, final boolean ubi) { + String prefix = "" + if (architecture == Architecture.AARCH64) { + prefix += "aarch64-" + } + if (oss) { + prefix += "oss-" + } + if (ubi) { + prefix += "ubi-" + } + return "build/${prefix}docker" } -private static String taskName(final String prefix, final String architecture, final boolean oss, final String suffix) { - return "${prefix}${"aarch64".equals(architecture) ? 'Aarch64' : ''}${oss ? 'Oss' : ''}${suffix}" +private static String taskName(final String prefix, final Architecture architecture, final boolean oss, final boolean ubi, final String suffix) { + String infix = "" + if (architecture == Architecture.AARCH64) { + infix += "Aarch64" + } + if (oss) { + infix += "Oss" + } + if (ubi) { + infix += "Ubi" + } + return prefix + infix + suffix } project.ext { - dockerBuildContext = { String architecture, boolean oss, boolean local -> + dockerBuildContext = { Architecture architecture, boolean oss, boolean ubi, boolean local -> copySpec { into('bin') { from project.projectDir.toPath().resolve("src/docker/bin") @@ -102,22 +124,22 @@ project.ext { } from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) { - expand(expansions(architecture, oss, local)) + expand(expansions(architecture, oss, ubi, local)) } } } } -void addCopyDockerContextTask(final String architecture, final boolean oss) { - task(taskName("copy", architecture, oss, "DockerContext"), type: Sync) { - expansions(architecture, oss, true).findAll { it.key != 'build_date' }.each { k, v -> +void addCopyDockerContextTask(final Architecture architecture, final boolean oss, final boolean ubi) { + task(taskName("copy", architecture, oss, ubi, "DockerContext"), type: Sync) { + expansions(architecture, oss, ubi, true).findAll { it.key != 'build_date' }.each { k, v -> inputs.property(k, { v.toString() }) } - into buildPath(architecture, oss) + into buildPath(architecture, oss, ubi) - with dockerBuildContext(architecture, oss, true) + with dockerBuildContext(architecture, oss, ubi, true) - if ("aarch64".equals(architecture)) { + if (architecture == Architecture.AARCH64) { if (oss) { from configurations.aarch64OssDockerSource } else { @@ -200,34 +222,36 @@ task integTest(type: Test) { check.dependsOn integTest -void addBuildDockerImage(final String architecture, final boolean oss) { - final Task buildDockerImageTask = task(taskName("build", architecture, oss, "DockerImage"), type: DockerBuildTask) { - TaskProvider copyContextTask = tasks.named(taskName("copy", architecture, oss, "DockerContext")) +void addBuildDockerImage(final Architecture architecture, final boolean oss, final boolean ubi) { + final Task buildDockerImageTask = task(taskName("build", architecture, oss, ubi, "DockerImage"), type: DockerBuildTask) { + TaskProvider copyContextTask = tasks.named(taskName("copy", architecture, oss, ubi, "DockerContext")) dependsOn(copyContextTask) dockerContext.fileProvider(copyContextTask.map { it.destinationDir }) if (oss) { tags = [ - "docker.elastic.co/elasticsearch/elasticsearch-oss:${VersionProperties.elasticsearch}", - "elasticsearch-oss:test" + "docker.elastic.co/elasticsearch/elasticsearch-oss${ubi ? '-ubi7' : ''}:${VersionProperties.elasticsearch}", + "elasticsearch-oss${ubi ? '-ubi7' : ''}:test" ] } else { tags = [ - "elasticsearch:${VersionProperties.elasticsearch}", - "docker.elastic.co/elasticsearch/elasticsearch:${VersionProperties.elasticsearch}", - "docker.elastic.co/elasticsearch/elasticsearch-full:${VersionProperties.elasticsearch}", - "elasticsearch:test", + "elasticsearch${ubi ? '-ubi7' : ''}:${VersionProperties.elasticsearch}", + "docker.elastic.co/elasticsearch/elasticsearch${ubi ? '-ubi7' : ''}:${VersionProperties.elasticsearch}", + "docker.elastic.co/elasticsearch/elasticsearch${ubi ? '-ubi7' : ''}-full:${VersionProperties.elasticsearch}", + "elasticsearch${ubi ? '-ubi7' : ''}:test", ] } } - buildDockerImageTask.onlyIf { Architecture.current().name().toLowerCase().equals(architecture) } + buildDockerImageTask.onlyIf { Architecture.current() == architecture } assemble.dependsOn(buildDockerImageTask) } -for (final String architecture : ["aarch64", "x64"]) { +for (final Architecture architecture : Architecture.values()) { for (final boolean oss : [false, true]) { - addCopyDockerContextTask(architecture, oss) - addBuildDockerImage(architecture, oss) + for (final boolean ubi : [false, true]) { + addCopyDockerContextTask(architecture, oss, ubi) + addBuildDockerImage(architecture, oss, ubi) + } } } @@ -246,12 +270,13 @@ subprojects { Project subProject -> if (subProject.name.endsWith('-export')) { apply plugin: 'distribution' - final String architecture = subProject.name.contains('aarch64-') ? 'aarch64' : 'x64' + final Architecture architecture = subProject.name.contains('aarch64-') ? Architecture.AARCH64 : Architecture.X64 final boolean oss = subProject.name.contains('oss-') + final boolean ubi = subProject.name.contains('ubi-') - def exportTaskName = taskName("export", architecture, oss, "DockerImage") - def buildTaskName = taskName("build", architecture, oss, "DockerImage") - def tarFile = "${parent.projectDir}/build/elasticsearch${"aarch64".equals(architecture) ? '-aarch64' : ''}${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar" + def exportTaskName = taskName("export", architecture, oss, ubi, "DockerImage") + def buildTaskName = taskName("build", architecture, oss, ubi, "DockerImage") + def tarFile = "${parent.projectDir}/build/elasticsearch${architecture == Architecture.AARCH64 ? '-aarch64' : ''}${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar" final Task exportDockerImageTask = task(exportTaskName, type: LoggedExec) { inputs.file("${parent.projectDir}/build/markers/${buildTaskName}.marker") @@ -260,7 +285,7 @@ subprojects { Project subProject -> args "save", "-o", tarFile, - "elasticsearch${oss ? '-oss' : ''}:test" + "elasticsearch${oss ? '-oss' : ''}${ubi ? '-ubi7' : ''}:test" } exportDockerImageTask.dependsOn(parent.tasks.getByName(buildTaskName)) @@ -269,7 +294,7 @@ subprojects { Project subProject -> artifacts.add('default', file(tarFile)) { type 'tar' - name "elasticsearch${"aarch64".equals(architecture) ? '-aarch64' : ''}${oss ? '-oss' : ''}" + name "elasticsearch${architecture == Architecture.AARCH64 ? '-aarch64' : ''}${oss ? '-oss' : ''}${ubi ? '-ubi7' : ''}" builtBy exportTaskName } diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle index b641e697c4ab6..15604ce94c04f 100644 --- a/distribution/docker/docker-build-context/build.gradle +++ b/distribution/docker/docker-build-context/build.gradle @@ -6,8 +6,7 @@ task buildDockerBuildContext(type: Tar) { archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch" // Non-local builds don't need to specify an architecture. - // Make this explicit via the string value. - with dockerBuildContext("", false, false) + with dockerBuildContext(null, false, false, false) } assemble.dependsOn buildDockerBuildContext diff --git a/distribution/docker/oss-docker-build-context/build.gradle b/distribution/docker/oss-docker-build-context/build.gradle index a3603254d1318..172bfe0c7e781 100644 --- a/distribution/docker/oss-docker-build-context/build.gradle +++ b/distribution/docker/oss-docker-build-context/build.gradle @@ -6,8 +6,7 @@ task buildOssDockerBuildContext(type: Tar) { archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch-oss" // Non-local builds don't need to specify an architecture. - // Make this explicit via the string value. - with dockerBuildContext("", true, false) + with dockerBuildContext(null, true, false, false) } assemble.dependsOn buildOssDockerBuildContext diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 28b611a446acb..f50c40b70b66f 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -13,10 +13,19 @@ ################################################################################ # Build stage 0 `builder`: # Extract elasticsearch artifact -# Set gid=0 and make group perms==owner perms ################################################################################ -FROM centos:7 AS builder +FROM ${base_image} AS builder + +# Ensure some requires tools are installed. They may not be on some minimal base images. +# We run ${package_manager} commands in a loop, in order to prevent transient repository access errors +# from failing the build. We only loop on failure, using \$exit_code to track success or failure. +RUN for iter in {1..10}; do \\ + ${package_manager} install --setopt=tsflags=nodocs -y tar gzip && \\ + exit_code=0 && break || \\ + exit_code=\$? && echo "${package_manager} error: retry \$iter in 10s" && sleep 10 ; \\ + done; \\ + (exit \$exit_code) # `tini` is a tiny but valid init for containers. This is used to cleanly # control how ES and any child processes are shut down. @@ -39,11 +48,7 @@ RUN set -eux ; \\ mv \${tini_bin} /tini ; \\ chmod +x /tini -ENV PATH /usr/share/elasticsearch/bin:\$PATH - -RUN groupadd -g 1000 elasticsearch && \\ - adduser -u 1000 -g 1000 -d /usr/share/elasticsearch elasticsearch - +RUN mkdir /usr/share/elasticsearch WORKDIR /usr/share/elasticsearch ${source_elasticsearch} @@ -61,15 +66,21 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties # Add entrypoint ################################################################################ -FROM centos:7 +FROM ${base_image} ENV ELASTIC_CONTAINER true -COPY --from=builder /tini /tini - -RUN for iter in {1..10}; do yum update --setopt=tsflags=nodocs -y && \\ - yum install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \\ - yum clean all && exit_code=0 && break || exit_code=\$? && echo "yum error: retry \$iter in 10s" && sleep 10; done; \\ +COPY --from=builder --chown=0:0 /tini /tini + +# We run ${package_manager} commands in a loop, in order to prevent transient repository access errors +# from failing the build. We only loop on failure, using \$exit_code to track success or failure. +RUN for iter in {1..10}; do \\ + ${package_manager} update --setopt=tsflags=nodocs -y && \\ + ${package_manager} install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \\ + ${package_manager} clean all && \\ + exit_code=0 && break || \\ + exit_code=\$? && echo "${package_manager} error: retry \$iter in 10s" && sleep 10 ; \\ + done; \\ (exit \$exit_code) RUN groupadd -g 1000 elasticsearch && \\ @@ -123,6 +134,7 @@ ENTRYPOINT ["/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] # Dummy overridable parameter parsed by entrypoint CMD ["eswrapper"] +HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9200 || exit 1 ################################################################################ # End of multi-stage Dockerfile ################################################################################ diff --git a/distribution/docker/ubi-docker-build-context/build.gradle b/distribution/docker/ubi-docker-build-context/build.gradle new file mode 100644 index 0000000000000..09c65f818a778 --- /dev/null +++ b/distribution/docker/ubi-docker-build-context/build.gradle @@ -0,0 +1,12 @@ +apply plugin: 'base' + +task buildUbiDockerBuildContext(type: Tar) { + archiveExtension = 'tar.gz' + compression = Compression.GZIP + archiveClassifier = "docker-build-context" + archiveBaseName = "elasticsearch-ubi7" + // Non-local builds don't need to specify an architecture. + with dockerBuildContext(null, false, true, false) +} + +assemble.dependsOn buildUbiDockerBuildContext diff --git a/distribution/docker/ubi-docker-x64-export/build.gradle b/distribution/docker/ubi-docker-x64-export/build.gradle new file mode 100644 index 0000000000000..537b5a093683e --- /dev/null +++ b/distribution/docker/ubi-docker-x64-export/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// export is done in the parent project. diff --git a/settings.gradle b/settings.gradle index 4a8bc1f9e4268..a638195b75fc6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -42,6 +42,8 @@ List projects = [ 'distribution:docker:docker-export', 'distribution:docker:oss-docker-build-context', 'distribution:docker:oss-docker-export', + 'distribution:docker:ubi-docker-build-context', + 'distribution:docker:ubi-docker-export', 'distribution:packages:aarch64-oss-deb', 'distribution:packages:oss-deb', 'distribution:packages:aarch64-deb', From 20e8e5976e21b9fa67f50bfd27047ecf7f0954a0 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 30 Jul 2020 12:37:21 +0100 Subject: [PATCH 02/32] Only provide a task to build a UBI context --- distribution/docker/build.gradle | 108 ++++++++++-------- distribution/docker/src/docker/Dockerfile | 67 ++++++----- distribution/docker/src/docker/ubi/.gitignore | 2 + .../docker/src/docker/ubi/Jenkinsfile | 2 + distribution/docker/src/docker/ubi/README.md | 37 ++++++ .../ubi-docker-build-context/build.gradle | 5 +- .../docker/ubi-docker-x64-export/build.gradle | 2 - settings.gradle | 1 - 8 files changed, 139 insertions(+), 85 deletions(-) create mode 100644 distribution/docker/src/docker/ubi/.gitignore create mode 100644 distribution/docker/src/docker/ubi/Jenkinsfile create mode 100644 distribution/docker/src/docker/ubi/README.md delete mode 100644 distribution/docker/ubi-docker-x64-export/build.gradle diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index f21f47106522f..4d96b35984228 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -1,3 +1,4 @@ +import java.nio.file.Path import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor import org.elasticsearch.gradle.LoggedExec @@ -30,7 +31,7 @@ dependencies { ext.expansions = { Architecture architecture, boolean oss, boolean ubi, boolean local -> String classifier - if (local) { + if (local || ubi) { switch (architecture) { case Architecture.AARCH64: classifier = "linux-aarch64" @@ -56,7 +57,12 @@ ext.expansions = { Architecture architecture, boolean oss, boolean ubi, boolean * architecture. */ String sourceElasticsearch if (local) { - sourceElasticsearch = "COPY $elasticsearch /opt/elasticsearch.tar.gz" + sourceElasticsearch = "COPY ${elasticsearch} /opt/elasticsearch.tar.gz" + } else if (ubi) { + sourceElasticsearch = """ +ARG TARBALL=${elasticsearch} +COPY \${TARBALL} /opt/elasticsearch.tar.gz +""" } else { sourceElasticsearch = """ RUN curl --retry 8 -S -L \\ @@ -66,17 +72,18 @@ RUN curl --retry 8 -S -L \\ } return [ - 'base_image' : ubi ? 'registry.access.redhat.com/ubi7/ubi-minimal:7.7' : 'centos:7', + 'is_ubi' : ubi, + 'bin_dir' : ubi ? 'scripts' : 'bin', + 'config_dir' : ubi ? 'scripts' : 'config', 'build_date' : BuildParams.buildDate, 'git_revision' : BuildParams.gitRevision, 'license' : oss ? 'Apache-2.0' : 'Elastic-License', - 'package_manager' : ubi ? 'microdnf' : 'yum', 'source_elasticsearch': sourceElasticsearch.trim(), 'version' : VersionProperties.elasticsearch ] } -private static String buildPath(final Architecture architecture, final boolean oss, final boolean ubi) { +private static String buildPath(final Architecture architecture, final boolean oss) { String prefix = "" if (architecture == Architecture.AARCH64) { prefix += "aarch64-" @@ -84,13 +91,10 @@ private static String buildPath(final Architecture architecture, final boolean o if (oss) { prefix += "oss-" } - if (ubi) { - prefix += "ubi-" - } return "build/${prefix}docker" } -private static String taskName(final String prefix, final Architecture architecture, final boolean oss, final boolean ubi, final String suffix) { +private static String taskName(final String prefix, final Architecture architecture, final boolean oss, final String suffix) { String infix = "" if (architecture == Architecture.AARCH64) { infix += "Aarch64" @@ -98,46 +102,58 @@ private static String taskName(final String prefix, final Architecture architect if (oss) { infix += "Oss" } - if (ubi) { - infix += "Ubi" - } return prefix + infix + suffix } project.ext { dockerBuildContext = { Architecture architecture, boolean oss, boolean ubi, boolean local -> copySpec { - into('bin') { - from project.projectDir.toPath().resolve("src/docker/bin") - } + final Map varExpansions = expansions(architecture, oss, ubi, local) + final Path projectDir = project.projectDir.toPath() + + if (ubi) { + into('scripts') { + from projectDir.resolve("src/docker/bin") + from(projectDir.resolve("src/docker/config")) { + exclude '**/oss' + } + } + from(projectDir.resolve("src/docker/ubi")) { + expand(varExpansions) + } + } else { + into('bin') { + from projectDir.resolve("src/docker/bin") + } - into('config') { - /* + into('config') { + /* * Oss and default distribution can have different configuration, therefore we want to allow overriding the default configuration * by creating config files in oss or default build-context sub-modules. */ - duplicatesStrategy = DuplicatesStrategy.INCLUDE - from project.projectDir.toPath().resolve("src/docker/config") - if (oss) { - from project.projectDir.toPath().resolve("src/docker/config/oss") + duplicatesStrategy = DuplicatesStrategy.INCLUDE + from projectDir.resolve("src/docker/config") + if (oss) { + from projectDir.resolve("src/docker/config/oss") + } } } from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) { - expand(expansions(architecture, oss, ubi, local)) + expand(varExpansions) } } } } -void addCopyDockerContextTask(final Architecture architecture, final boolean oss, final boolean ubi) { - task(taskName("copy", architecture, oss, ubi, "DockerContext"), type: Sync) { - expansions(architecture, oss, ubi, true).findAll { it.key != 'build_date' }.each { k, v -> +void addCopyDockerContextTask(final Architecture architecture, final boolean oss) { + task(taskName("copy", architecture, oss, "DockerContext"), type: Sync) { + expansions(architecture, oss, false, true).findAll { it.key != 'build_date' }.each { k, v -> inputs.property(k, { v.toString() }) } - into buildPath(architecture, oss, ubi) + into buildPath(architecture, oss) - with dockerBuildContext(architecture, oss, ubi, true) + with dockerBuildContext(architecture, oss, false, true) if (architecture == Architecture.AARCH64) { if (oss) { @@ -222,23 +238,23 @@ task integTest(type: Test) { check.dependsOn integTest -void addBuildDockerImage(final Architecture architecture, final boolean oss, final boolean ubi) { - final Task buildDockerImageTask = task(taskName("build", architecture, oss, ubi, "DockerImage"), type: DockerBuildTask) { - TaskProvider copyContextTask = tasks.named(taskName("copy", architecture, oss, ubi, "DockerContext")) +void addBuildDockerImage(final Architecture architecture, final boolean oss) { + final Task buildDockerImageTask = task(taskName("build", architecture, oss, "DockerImage"), type: DockerBuildTask) { + TaskProvider copyContextTask = tasks.named(taskName("copy", architecture, oss, "DockerContext")) dependsOn(copyContextTask) dockerContext.fileProvider(copyContextTask.map { it.destinationDir }) if (oss) { tags = [ - "docker.elastic.co/elasticsearch/elasticsearch-oss${ubi ? '-ubi7' : ''}:${VersionProperties.elasticsearch}", - "elasticsearch-oss${ubi ? '-ubi7' : ''}:test" + "docker.elastic.co/elasticsearch/elasticsearch-oss:${VersionProperties.elasticsearch}", + "elasticsearch-oss:test" ] } else { tags = [ - "elasticsearch${ubi ? '-ubi7' : ''}:${VersionProperties.elasticsearch}", - "docker.elastic.co/elasticsearch/elasticsearch${ubi ? '-ubi7' : ''}:${VersionProperties.elasticsearch}", - "docker.elastic.co/elasticsearch/elasticsearch${ubi ? '-ubi7' : ''}-full:${VersionProperties.elasticsearch}", - "elasticsearch${ubi ? '-ubi7' : ''}:test", + "elasticsearch:${VersionProperties.elasticsearch}", + "docker.elastic.co/elasticsearch/elasticsearch:${VersionProperties.elasticsearch}", + "docker.elastic.co/elasticsearch/elasticsearch-full:${VersionProperties.elasticsearch}", + "elasticsearch:test", ] } } @@ -246,12 +262,13 @@ void addBuildDockerImage(final Architecture architecture, final boolean oss, fin assemble.dependsOn(buildDockerImageTask) } +// Note that the build process for UBI is different enough that +// we only provide a task for generating the build context, not actually +// building the image. See the `ubi-docker-build-context` sub-project. for (final Architecture architecture : Architecture.values()) { for (final boolean oss : [false, true]) { - for (final boolean ubi : [false, true]) { - addCopyDockerContextTask(architecture, oss, ubi) - addBuildDockerImage(architecture, oss, ubi) - } + addCopyDockerContextTask(architecture, oss) + addBuildDockerImage(architecture, oss) } } @@ -264,7 +281,7 @@ if (tasks.findByName("composePull")) { /* * The export subprojects write out the generated Docker images to disk, so - * that they can be easily reloaded, for example into a VM. + * that they can be easily reloaded, for example into a VM for distribution testing */ subprojects { Project subProject -> if (subProject.name.endsWith('-export')) { @@ -272,10 +289,9 @@ subprojects { Project subProject -> final Architecture architecture = subProject.name.contains('aarch64-') ? Architecture.AARCH64 : Architecture.X64 final boolean oss = subProject.name.contains('oss-') - final boolean ubi = subProject.name.contains('ubi-') - def exportTaskName = taskName("export", architecture, oss, ubi, "DockerImage") - def buildTaskName = taskName("build", architecture, oss, ubi, "DockerImage") + def exportTaskName = taskName("export", architecture, oss, "DockerImage") + def buildTaskName = taskName("build", architecture, oss, "DockerImage") def tarFile = "${parent.projectDir}/build/elasticsearch${architecture == Architecture.AARCH64 ? '-aarch64' : ''}${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar" final Task exportDockerImageTask = task(exportTaskName, type: LoggedExec) { @@ -285,7 +301,7 @@ subprojects { Project subProject -> args "save", "-o", tarFile, - "elasticsearch${oss ? '-oss' : ''}${ubi ? '-ubi7' : ''}:test" + "elasticsearch${oss ? '-oss' : ''}:test" } exportDockerImageTask.dependsOn(parent.tasks.getByName(buildTaskName)) @@ -294,7 +310,7 @@ subprojects { Project subProject -> artifacts.add('default', file(tarFile)) { type 'tar' - name "elasticsearch${architecture == Architecture.AARCH64 ? '-aarch64' : ''}${oss ? '-oss' : ''}${ubi ? '-ubi7' : ''}" + name "elasticsearch${architecture == Architecture.AARCH64 ? '-aarch64' : ''}${oss ? '-oss' : ''}" builtBy exportTaskName } diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index f50c40b70b66f..5844d16ba0889 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -9,23 +9,27 @@ can also comment out blocks, like this one. See: https://docs.groovy-lang.org/latest/html/api/groovy/text/SimpleTemplateEngine.html + + We use control-flow tags in this file to conditionally render the content. The + layout/presentation here has been adjusted so that it looks reasonable when rendered, + at the slight expense of how it looks here. */ %> ################################################################################ # Build stage 0 `builder`: -# Extract elasticsearch artifact +# Extract Elasticsearch artifact ################################################################################ +<% if (is_ubi) { %> +ARG BASE_REGISTRY=nexus-docker-secure.levelup-nexus.svc.cluster.local:18082 +ARG BASE_IMAGE=redhat/ubi/ubi8 +ARG BASE_TAG=8.2 -FROM ${base_image} AS builder +FROM \${BASE_REGISTRY}/\${BASE_IMAGE}:\${BASE_TAG} AS builder -# Ensure some requires tools are installed. They may not be on some minimal base images. -# We run ${package_manager} commands in a loop, in order to prevent transient repository access errors -# from failing the build. We only loop on failure, using \$exit_code to track success or failure. -RUN for iter in {1..10}; do \\ - ${package_manager} install --setopt=tsflags=nodocs -y tar gzip && \\ - exit_code=0 && break || \\ - exit_code=\$? && echo "${package_manager} error: retry \$iter in 10s" && sleep 10 ; \\ - done; \\ - (exit \$exit_code) +# `tini` is a tiny but valid init for containers. This is used to cleanly +# control how ES and any child processes are shut down. +COPY tini /tini +<% } else { %> +FROM centos:7 AS builder # `tini` is a tiny but valid init for containers. This is used to cleanly # control how ES and any child processes are shut down. @@ -45,9 +49,8 @@ RUN set -eux ; \\ curl --retry 8 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\ sha256sum -c \${tini_bin}.sha256sum ; \\ rm \${tini_bin}.sha256sum ; \\ - mv \${tini_bin} /tini ; \\ - chmod +x /tini - + mv \${tini_bin} /tini +<% } %> RUN mkdir /usr/share/elasticsearch WORKDIR /usr/share/elasticsearch @@ -57,31 +60,25 @@ RUN tar zxf /opt/elasticsearch.tar.gz --strip-components=1 RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env RUN mkdir -p config config/jvm.options.d data logs RUN chmod 0775 config config/jvm.options.d data logs -COPY config/elasticsearch.yml config/log4j2.properties config/ +COPY ${config_dir}/elasticsearch.yml ${config_dir}/log4j2.properties config/ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties ################################################################################ -# Build stage 1 (the actual elasticsearch image): +# Build stage 1 (the actual Elasticsearch image): +# # Copy elasticsearch from stage 0 # Add entrypoint ################################################################################ - -FROM ${base_image} - +<% if (is_ubi) { %> +FROM \${BASE_REGISTRY}/\${BASE_IMAGE}:\${BASE_TAG} AS builder +<% } else { %> +FROM centos:7 +<% } %> ENV ELASTIC_CONTAINER true -COPY --from=builder --chown=0:0 /tini /tini - -# We run ${package_manager} commands in a loop, in order to prevent transient repository access errors -# from failing the build. We only loop on failure, using \$exit_code to track success or failure. -RUN for iter in {1..10}; do \\ - ${package_manager} update --setopt=tsflags=nodocs -y && \\ - ${package_manager} install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \\ - ${package_manager} clean all && \\ - exit_code=0 && break || \\ - exit_code=\$? && echo "${package_manager} error: retry \$iter in 10s" && sleep 10 ; \\ - done; \\ - (exit \$exit_code) +RUN yum update --setopt=tsflags=nodocs -y && \\ + yum install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \\ + yum clean all RUN groupadd -g 1000 elasticsearch && \\ adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\ @@ -90,6 +87,8 @@ RUN groupadd -g 1000 elasticsearch && \\ WORKDIR /usr/share/elasticsearch COPY --from=builder --chown=1000:0 /usr/share/elasticsearch /usr/share/elasticsearch +COPY --from=builder --chown=0:0 /tini /tini +RUN chmod +x /tini # Replace OpenJDK's built-in CA certificate keystore with the one from the OS # vendor. The latter is superior in several ways. @@ -98,8 +97,7 @@ RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk ENV PATH /usr/share/elasticsearch/bin:\$PATH -COPY bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh - +COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN chmod g=u /etc/passwd && \\ chmod 0775 /usr/local/bin/docker-entrypoint.sh @@ -133,8 +131,9 @@ USER elasticsearch:root ENTRYPOINT ["/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] # Dummy overridable parameter parsed by entrypoint CMD ["eswrapper"] - +<% if (is_ubi) { %> HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9200 || exit 1 +<% } %> ################################################################################ # End of multi-stage Dockerfile ################################################################################ diff --git a/distribution/docker/src/docker/ubi/.gitignore b/distribution/docker/src/docker/ubi/.gitignore new file mode 100644 index 0000000000000..cda31c3986d13 --- /dev/null +++ b/distribution/docker/src/docker/ubi/.gitignore @@ -0,0 +1,2 @@ +# Ignore any locally downloaded or dropped releases +*.tar.gz diff --git a/distribution/docker/src/docker/ubi/Jenkinsfile b/distribution/docker/src/docker/ubi/Jenkinsfile new file mode 100644 index 0000000000000..7422f1f7a2a9c --- /dev/null +++ b/distribution/docker/src/docker/ubi/Jenkinsfile @@ -0,0 +1,2 @@ +@Library('DCCSCR@master') _ +dccscrPipeline(version: '${version}') diff --git a/distribution/docker/src/docker/ubi/README.md b/distribution/docker/src/docker/ubi/README.md new file mode 100644 index 0000000000000..14854a53615f4 --- /dev/null +++ b/distribution/docker/src/docker/ubi/README.md @@ -0,0 +1,37 @@ +# Elasticsearch + +**Elasticsearch** is a distributed, RESTful search and analytics engine capable of +solving a growing number of use cases. As the heart of the Elastic Stack, it +centrally stores your data so you can discover the expected and uncover the +unexpected. + +For more information about Elasticsearch, please visit +https://www.elastic.co/products/elasticsearch. + +### Installation instructions + +Please follow the documentation on [how to install Elasticsearch with Docker](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html). + +### Where to file issues and PRs + +- [Issues](https://github.com/elastic/elasticsearch/issues) +- [PRs](https://github.com/elastic/elasticsearch/pulls) + +### Where to get help + +- [Elasticsearch Discuss Forums](https://discuss.elastic.co/c/elasticsearch) +- [Elasticsearch Documentation](https://www.elastic.co/guide/en/elasticsearch/reference/master/index.html) + +### Still need help? + +You can learn more about the Elastic Community and also understand how to get more help +visiting [Elastic Community](https://www.elastic.co/community). + + +This software is governed by the [Elastic +License](https://github.com/elastic/elasticsearch/blob/7.8/licenses/ELASTIC-LICENSE.txt), +and includes the full set of [free +features](https://www.elastic.co/subscriptions). + +View the detailed release notes +[here](https://www.elastic.co/guide/en/elasticsearch/reference/7.8/es-release-notes.html). diff --git a/distribution/docker/ubi-docker-build-context/build.gradle b/distribution/docker/ubi-docker-build-context/build.gradle index 09c65f818a778..0321f7b344fa7 100644 --- a/distribution/docker/ubi-docker-build-context/build.gradle +++ b/distribution/docker/ubi-docker-build-context/build.gradle @@ -1,3 +1,5 @@ +import org.elasticsearch.gradle.Architecture + apply plugin: 'base' task buildUbiDockerBuildContext(type: Tar) { @@ -5,8 +7,7 @@ task buildUbiDockerBuildContext(type: Tar) { compression = Compression.GZIP archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch-ubi7" - // Non-local builds don't need to specify an architecture. - with dockerBuildContext(null, false, true, false) + with dockerBuildContext(Architecture.X64, false, true, false) } assemble.dependsOn buildUbiDockerBuildContext diff --git a/distribution/docker/ubi-docker-x64-export/build.gradle b/distribution/docker/ubi-docker-x64-export/build.gradle deleted file mode 100644 index 537b5a093683e..0000000000000 --- a/distribution/docker/ubi-docker-x64-export/build.gradle +++ /dev/null @@ -1,2 +0,0 @@ -// This file is intentionally blank. All configuration of the -// export is done in the parent project. diff --git a/settings.gradle b/settings.gradle index a638195b75fc6..56b675f8d559d 100644 --- a/settings.gradle +++ b/settings.gradle @@ -43,7 +43,6 @@ List projects = [ 'distribution:docker:oss-docker-build-context', 'distribution:docker:oss-docker-export', 'distribution:docker:ubi-docker-build-context', - 'distribution:docker:ubi-docker-export', 'distribution:packages:aarch64-oss-deb', 'distribution:packages:oss-deb', 'distribution:packages:aarch64-deb', From 8a3f3bc47471c59fd6dfb49f324a839a0273c109 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 30 Jul 2020 14:56:39 +0100 Subject: [PATCH 03/32] Generate UBI downloads.json during context build (badly) --- distribution/docker/build.gradle | 36 +++++++++++++++++-- .../docker/src/docker/ubi/downloads.json | 20 +++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 distribution/docker/src/docker/ubi/downloads.json diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 4d96b35984228..aa2d44344d1f1 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -1,4 +1,3 @@ -import java.nio.file.Path import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor import org.elasticsearch.gradle.LoggedExec @@ -7,6 +6,9 @@ import org.elasticsearch.gradle.docker.DockerBuildTask import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin +import java.nio.file.Path +import java.security.MessageDigest + apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.test.fixtures' apply plugin: 'elasticsearch.internal-distribution-download' @@ -78,11 +80,39 @@ RUN curl --retry 8 -S -L \\ 'build_date' : BuildParams.buildDate, 'git_revision' : BuildParams.gitRevision, 'license' : oss ? 'Apache-2.0' : 'Elastic-License', - 'source_elasticsearch': sourceElasticsearch.trim(), - 'version' : VersionProperties.elasticsearch + 'source_elasticsearch': sourceElasticsearch, + 'version' : VersionProperties.elasticsearch, + 'version_hash' : ubi ? getChecksum(configurations.dockerSource.singleFile) : '' ] } +private static String getChecksum(File file) { + MessageDigest digest = MessageDigest.getInstance("SHA-512") + + FileInputStream fis = new FileInputStream(file) + byte[] byteArray = new byte[1024] + int bytesCount = 0 + + //Read file data and update in message digest + while ((bytesCount = fis.read(byteArray)) != -1) { + digest.update(byteArray, 0, bytesCount) + } + + fis.close() + + //Get the hash's bytes + byte[] bytes = digest.digest() + + //This bytes[] has bytes in decimal format; + //Convert it to hexadecimal format + StringBuilder sb = new StringBuilder() + for (int i = 0; i < bytes.length; i++) { + sb.append(Integer.toString((bytes[i] & 0xff) + 0x100, 16).substring(1)) + } + + return sb.toString() +} + private static String buildPath(final Architecture architecture, final boolean oss) { String prefix = "" if (architecture == Architecture.AARCH64) { diff --git a/distribution/docker/src/docker/ubi/downloads.json b/distribution/docker/src/docker/ubi/downloads.json new file mode 100644 index 0000000000000..d3c4b378697a4 --- /dev/null +++ b/distribution/docker/src/docker/ubi/downloads.json @@ -0,0 +1,20 @@ +{ + "resources": [ + { + "url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-x86_64.tar.gz", + "filename": "elasticsearch-${version}-linux-x86_64.tar.gz", + "validation": { + "type": "sha512", + "value": "${version_hash}" + } + }, + { + "url": "https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64", + "filename": "tini", + "validation": { + "type": "sha256", + "value": "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c" + } + } + ] +} From ff2ecd7534173e56f0003541d6bb1a55d8e8786c Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 30 Jul 2020 15:15:13 +0100 Subject: [PATCH 04/32] Revert unnecessary changes --- .../org/elasticsearch/gradle/Architecture.java | 6 ------ .../InternalDistributionDownloadPlugin.java | 4 ++-- distribution/docker/build.gradle | 18 ++---------------- 3 files changed, 4 insertions(+), 24 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java index 0cce78e4ec8e6..f230d9af86e11 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java @@ -19,8 +19,6 @@ package org.elasticsearch.gradle; -import java.util.Locale; - public enum Architecture { X64, @@ -39,8 +37,4 @@ public static Architecture current() { } } - @Override - public String toString() { - return super.toString().toLowerCase(Locale.ROOT); - } } diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java index bc1372fb641f3..9399f1f062ca0 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java @@ -127,7 +127,7 @@ private static String distributionProjectName(ElasticsearchDistribution distribu final String archString = platform == ElasticsearchDistribution.Platform.WINDOWS || architecture == Architecture.X64 ? "" - : "-" + architecture; + : "-" + architecture.toString().toLowerCase(); if (distribution.getFlavor() == ElasticsearchDistribution.Flavor.OSS) { projectName += "oss-"; @@ -139,7 +139,7 @@ private static String distributionProjectName(ElasticsearchDistribution distribu switch (distribution.getType()) { case ARCHIVE: - projectName += platform + archString + (platform == ElasticsearchDistribution.Platform.WINDOWS + projectName += platform.toString() + archString + (platform == ElasticsearchDistribution.Platform.WINDOWS ? "-zip" : "-tar"); break; diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index aa2d44344d1f1..be53ae88a5a91 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -114,25 +114,11 @@ private static String getChecksum(File file) { } private static String buildPath(final Architecture architecture, final boolean oss) { - String prefix = "" - if (architecture == Architecture.AARCH64) { - prefix += "aarch64-" - } - if (oss) { - prefix += "oss-" - } - return "build/${prefix}docker" + return "build/${architecture == Architecture.AARCH64 ? 'aarch64-' : ''}${oss ? 'oss-' : ''}docker" } private static String taskName(final String prefix, final Architecture architecture, final boolean oss, final String suffix) { - String infix = "" - if (architecture == Architecture.AARCH64) { - infix += "Aarch64" - } - if (oss) { - infix += "Oss" - } - return prefix + infix + suffix + return "${prefix}${architecture == Architecture.AARCH64 ? 'Aarch64' : ''}${oss ? 'Oss' : ''}${suffix}" } project.ext { From ad9140f8021fcef9944a8fd194cf14648bd9f98e Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 30 Jul 2020 15:22:26 +0100 Subject: [PATCH 05/32] Fix UBI version in archive name --- distribution/docker/ubi-docker-build-context/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/docker/ubi-docker-build-context/build.gradle b/distribution/docker/ubi-docker-build-context/build.gradle index 0321f7b344fa7..72b0845531e08 100644 --- a/distribution/docker/ubi-docker-build-context/build.gradle +++ b/distribution/docker/ubi-docker-build-context/build.gradle @@ -6,7 +6,7 @@ task buildUbiDockerBuildContext(type: Tar) { archiveExtension = 'tar.gz' compression = Compression.GZIP archiveClassifier = "docker-build-context" - archiveBaseName = "elasticsearch-ubi7" + archiveBaseName = "elasticsearch-ubi8" with dockerBuildContext(Architecture.X64, false, true, false) } From 2171b241f1e92c8d5cdfadf899bbc5a5f0e99dd0 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 30 Jul 2020 16:19:34 +0100 Subject: [PATCH 06/32] Hack around gradle issues with configurations and project context --- distribution/docker/build.gradle | 22 ++++++++++++++----- .../docker/docker-build-context/build.gradle | 2 +- .../oss-docker-build-context/build.gradle | 2 +- .../ubi-docker-build-context/build.gradle | 2 +- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index be53ae88a5a91..f9f9ac5556d8a 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -31,7 +31,7 @@ dependencies { ossDockerSource project(path: ":distribution:archives:oss-linux-tar") } -ext.expansions = { Architecture architecture, boolean oss, boolean ubi, boolean local -> +ext.expansions = { Architecture architecture, boolean oss, boolean ubi, boolean local, Project p -> String classifier if (local || ubi) { switch (architecture) { @@ -82,7 +82,7 @@ RUN curl --retry 8 -S -L \\ 'license' : oss ? 'Apache-2.0' : 'Elastic-License', 'source_elasticsearch': sourceElasticsearch, 'version' : VersionProperties.elasticsearch, - 'version_hash' : ubi ? getChecksum(configurations.dockerSource.singleFile) : '' + 'version_hash' : ubi ? getChecksum(p.configurations.dockerSource.singleFile) : '' ] } @@ -122,9 +122,9 @@ private static String taskName(final String prefix, final Architecture architect } project.ext { - dockerBuildContext = { Architecture architecture, boolean oss, boolean ubi, boolean local -> + dockerBuildContext = { Architecture architecture, boolean oss, boolean ubi, boolean local, Project p -> copySpec { - final Map varExpansions = expansions(architecture, oss, ubi, local) + final Map varExpansions = expansions(architecture, oss, ubi, local, p) final Path projectDir = project.projectDir.toPath() if (ubi) { @@ -164,12 +164,12 @@ project.ext { void addCopyDockerContextTask(final Architecture architecture, final boolean oss) { task(taskName("copy", architecture, oss, "DockerContext"), type: Sync) { - expansions(architecture, oss, false, true).findAll { it.key != 'build_date' }.each { k, v -> + expansions(architecture, oss, false, true, project).findAll { it.key != 'build_date' }.each { k, v -> inputs.property(k, { v.toString() }) } into buildPath(architecture, oss) - with dockerBuildContext(architecture, oss, false, true) + with dockerBuildContext(architecture, oss, false, true, project) if (architecture == Architecture.AARCH64) { if (oss) { @@ -300,6 +300,16 @@ if (tasks.findByName("composePull")) { * that they can be easily reloaded, for example into a VM for distribution testing */ subprojects { Project subProject -> + if (subProject.name.endsWith('-context')) { + configurations { + dockerSource + } + + dependencies { + dockerSource project(path: ":distribution:archives:linux-tar") + } + } + if (subProject.name.endsWith('-export')) { apply plugin: 'distribution' diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle index 15604ce94c04f..0bf27b43190c2 100644 --- a/distribution/docker/docker-build-context/build.gradle +++ b/distribution/docker/docker-build-context/build.gradle @@ -6,7 +6,7 @@ task buildDockerBuildContext(type: Tar) { archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch" // Non-local builds don't need to specify an architecture. - with dockerBuildContext(null, false, false, false) + with dockerBuildContext(null, false, false, false, project) } assemble.dependsOn buildDockerBuildContext diff --git a/distribution/docker/oss-docker-build-context/build.gradle b/distribution/docker/oss-docker-build-context/build.gradle index 172bfe0c7e781..a77f4a78d875b 100644 --- a/distribution/docker/oss-docker-build-context/build.gradle +++ b/distribution/docker/oss-docker-build-context/build.gradle @@ -6,7 +6,7 @@ task buildOssDockerBuildContext(type: Tar) { archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch-oss" // Non-local builds don't need to specify an architecture. - with dockerBuildContext(null, true, false, false) + with dockerBuildContext(null, true, false, false, project) } assemble.dependsOn buildOssDockerBuildContext diff --git a/distribution/docker/ubi-docker-build-context/build.gradle b/distribution/docker/ubi-docker-build-context/build.gradle index 72b0845531e08..f59286dda0f62 100644 --- a/distribution/docker/ubi-docker-build-context/build.gradle +++ b/distribution/docker/ubi-docker-build-context/build.gradle @@ -7,7 +7,7 @@ task buildUbiDockerBuildContext(type: Tar) { compression = Compression.GZIP archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch-ubi8" - with dockerBuildContext(Architecture.X64, false, true, false) + with dockerBuildContext(Architecture.X64, false, true, false, project) } assemble.dependsOn buildUbiDockerBuildContext From 10247102e6cf11b0d9462d57e8720bff28cadaf9 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 30 Jul 2020 16:41:51 +0100 Subject: [PATCH 07/32] Revert "Hack around gradle issues with configurations and project context" This reverts commit 2171b241f1e92c8d5cdfadf899bbc5a5f0e99dd0. --- distribution/docker/build.gradle | 22 +++++-------------- .../docker/docker-build-context/build.gradle | 2 +- .../oss-docker-build-context/build.gradle | 2 +- .../ubi-docker-build-context/build.gradle | 2 +- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index f9f9ac5556d8a..be53ae88a5a91 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -31,7 +31,7 @@ dependencies { ossDockerSource project(path: ":distribution:archives:oss-linux-tar") } -ext.expansions = { Architecture architecture, boolean oss, boolean ubi, boolean local, Project p -> +ext.expansions = { Architecture architecture, boolean oss, boolean ubi, boolean local -> String classifier if (local || ubi) { switch (architecture) { @@ -82,7 +82,7 @@ RUN curl --retry 8 -S -L \\ 'license' : oss ? 'Apache-2.0' : 'Elastic-License', 'source_elasticsearch': sourceElasticsearch, 'version' : VersionProperties.elasticsearch, - 'version_hash' : ubi ? getChecksum(p.configurations.dockerSource.singleFile) : '' + 'version_hash' : ubi ? getChecksum(configurations.dockerSource.singleFile) : '' ] } @@ -122,9 +122,9 @@ private static String taskName(final String prefix, final Architecture architect } project.ext { - dockerBuildContext = { Architecture architecture, boolean oss, boolean ubi, boolean local, Project p -> + dockerBuildContext = { Architecture architecture, boolean oss, boolean ubi, boolean local -> copySpec { - final Map varExpansions = expansions(architecture, oss, ubi, local, p) + final Map varExpansions = expansions(architecture, oss, ubi, local) final Path projectDir = project.projectDir.toPath() if (ubi) { @@ -164,12 +164,12 @@ project.ext { void addCopyDockerContextTask(final Architecture architecture, final boolean oss) { task(taskName("copy", architecture, oss, "DockerContext"), type: Sync) { - expansions(architecture, oss, false, true, project).findAll { it.key != 'build_date' }.each { k, v -> + expansions(architecture, oss, false, true).findAll { it.key != 'build_date' }.each { k, v -> inputs.property(k, { v.toString() }) } into buildPath(architecture, oss) - with dockerBuildContext(architecture, oss, false, true, project) + with dockerBuildContext(architecture, oss, false, true) if (architecture == Architecture.AARCH64) { if (oss) { @@ -300,16 +300,6 @@ if (tasks.findByName("composePull")) { * that they can be easily reloaded, for example into a VM for distribution testing */ subprojects { Project subProject -> - if (subProject.name.endsWith('-context')) { - configurations { - dockerSource - } - - dependencies { - dockerSource project(path: ":distribution:archives:linux-tar") - } - } - if (subProject.name.endsWith('-export')) { apply plugin: 'distribution' diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle index 0bf27b43190c2..15604ce94c04f 100644 --- a/distribution/docker/docker-build-context/build.gradle +++ b/distribution/docker/docker-build-context/build.gradle @@ -6,7 +6,7 @@ task buildDockerBuildContext(type: Tar) { archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch" // Non-local builds don't need to specify an architecture. - with dockerBuildContext(null, false, false, false, project) + with dockerBuildContext(null, false, false, false) } assemble.dependsOn buildDockerBuildContext diff --git a/distribution/docker/oss-docker-build-context/build.gradle b/distribution/docker/oss-docker-build-context/build.gradle index a77f4a78d875b..172bfe0c7e781 100644 --- a/distribution/docker/oss-docker-build-context/build.gradle +++ b/distribution/docker/oss-docker-build-context/build.gradle @@ -6,7 +6,7 @@ task buildOssDockerBuildContext(type: Tar) { archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch-oss" // Non-local builds don't need to specify an architecture. - with dockerBuildContext(null, true, false, false, project) + with dockerBuildContext(null, true, false, false) } assemble.dependsOn buildOssDockerBuildContext diff --git a/distribution/docker/ubi-docker-build-context/build.gradle b/distribution/docker/ubi-docker-build-context/build.gradle index f59286dda0f62..72b0845531e08 100644 --- a/distribution/docker/ubi-docker-build-context/build.gradle +++ b/distribution/docker/ubi-docker-build-context/build.gradle @@ -7,7 +7,7 @@ task buildUbiDockerBuildContext(type: Tar) { compression = Compression.GZIP archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch-ubi8" - with dockerBuildContext(Architecture.X64, false, true, false, project) + with dockerBuildContext(Architecture.X64, false, true, false) } assemble.dependsOn buildUbiDockerBuildContext From 4a87d279010d91135f468b90bd1fc271066bde19 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 31 Jul 2020 11:28:05 +0100 Subject: [PATCH 08/32] Support ubi and ubi-minimal --- .../org/elasticsearch/gradle/Variant.java | 27 ++++ distribution/docker/build.gradle | 136 ++++++++++-------- .../docker/docker-build-context/build.gradle | 4 +- .../oss-docker-build-context/build.gradle | 4 +- distribution/docker/src/docker/Dockerfile | 31 ++-- .../ubi-docker-build-context/build.gradle | 5 +- .../build.gradle | 13 ++ settings.gradle | 1 + 8 files changed, 145 insertions(+), 76 deletions(-) create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/Variant.java create mode 100644 distribution/docker/ubi-minimal-docker-build-context/build.gradle diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Variant.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Variant.java new file mode 100644 index 0000000000000..7bb023b7af145 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Variant.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle; + +public enum Variant { + DEFAULT, + OSS, + UBI, + UBI_MINIMAL +} diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index be53ae88a5a91..63f9e5b6d28df 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -1,5 +1,6 @@ import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.ElasticsearchDistribution.Flavor +import org.elasticsearch.gradle.Variant import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.docker.DockerBuildTask @@ -8,6 +9,7 @@ import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin import java.nio.file.Path import java.security.MessageDigest +import java.util.stream.Collectors apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.test.fixtures' @@ -31,18 +33,15 @@ dependencies { ossDockerSource project(path: ":distribution:archives:oss-linux-tar") } -ext.expansions = { Architecture architecture, boolean oss, boolean ubi, boolean local -> +ext.expansions = { Architecture architecture, Variant variant, boolean local -> String classifier - if (local || ubi) { - switch (architecture) { - case Architecture.AARCH64: - classifier = "linux-aarch64" - break - case Architecture.X64: - classifier = "linux-x86_64" - break - default: - throw new IllegalArgumentException("Unrecognized architecture [" + architecture + "], must be one of (aarch64|x64)") + if (local || variant == Variant.UBI) { + if (architecture == Architecture.AARCH64) { + classifier = "linux-aarch64" + } else if (architecture == Architecture.X64) { + classifier = "linux-x86_64" + } else { + throw new IllegalArgumentException("Unsupported architecture [" + architecture + "]") } } else { /* When sourcing the Elasticsearch build remotely, the same Dockerfile needs @@ -51,7 +50,24 @@ ext.expansions = { Architecture architecture, boolean oss, boolean ubi, boolean classifier = "linux-\$(arch)" } - final String elasticsearch = "elasticsearch-${oss ? 'oss-' : ''}${VersionProperties.elasticsearch}-${classifier}.tar.gz" + final String elasticsearch = "elasticsearch-${variant == Variant.OSS ? 'oss-' : ''}${VersionProperties.elasticsearch}-${classifier}.tar.gz" + + String buildArgs = '#' + if (variant == Variant.UBI) { + buildArgs = """ +ARG BASE_REGISTRY=nexus-docker-secure.levelup-nexus.svc.cluster.local:18082 +ARG BASE_IMAGE=redhat/ubi/ubi8 +ARG BASE_TAG=8.2 +ARG TARBALL=${elasticsearch} +""" + } + + String baseImage = 'centos:7' + if (variant == Variant.UBI) { + baseImage = '${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS builder' + } else if (variant == Variant.UBI_MINIMAL) { + baseImage = 'registry.access.redhat.com/ubi8/ubi-minimal:8.2' + } /* Both the following Dockerfile commands put the resulting artifact at * the same location, regardless of classifier, so that the commands that @@ -60,11 +76,8 @@ ext.expansions = { Architecture architecture, boolean oss, boolean ubi, boolean String sourceElasticsearch if (local) { sourceElasticsearch = "COPY ${elasticsearch} /opt/elasticsearch.tar.gz" - } else if (ubi) { - sourceElasticsearch = """ -ARG TARBALL=${elasticsearch} -COPY \${TARBALL} /opt/elasticsearch.tar.gz -""" + } else if (variant == Variant.UBI) { + sourceElasticsearch = 'COPY ${TARBALL} /opt/elasticsearch.tar.gz' } else { sourceElasticsearch = """ RUN curl --retry 8 -S -L \\ @@ -74,15 +87,18 @@ RUN curl --retry 8 -S -L \\ } return [ - 'is_ubi' : ubi, - 'bin_dir' : ubi ? 'scripts' : 'bin', - 'config_dir' : ubi ? 'scripts' : 'config', + 'base_image' : baseImage, + 'bin_dir' : variant == Variant.UBI ? 'scripts' : 'bin', + 'build_args' : buildArgs, 'build_date' : BuildParams.buildDate, + 'config_dir' : variant == Variant.UBI ? 'scripts' : 'config', 'git_revision' : BuildParams.gitRevision, - 'license' : oss ? 'Apache-2.0' : 'Elastic-License', + 'license' : variant == Variant.OSS ? 'Apache-2.0' : 'Elastic-License', + 'package_manager' : variant == Variant.UBI_MINIMAL ? 'microdnf' : 'yum', 'source_elasticsearch': sourceElasticsearch, + 'variant' : variant.name().toLowerCase(), 'version' : VersionProperties.elasticsearch, - 'version_hash' : ubi ? getChecksum(configurations.dockerSource.singleFile) : '' + 'version_hash' : variant == Variant.UBI ? getChecksum(configurations.dockerSource.singleFile) : '' ] } @@ -113,21 +129,23 @@ private static String getChecksum(File file) { return sb.toString() } -private static String buildPath(final Architecture architecture, final boolean oss) { - return "build/${architecture == Architecture.AARCH64 ? 'aarch64-' : ''}${oss ? 'oss-' : ''}docker" +private static String buildPath(Architecture architecture, Variant variant) { + String variantString = variant == Variant.DEFAULT ? '' : variant.name().toLowerCase().replace('_', '-') + '-' + return "build/${architecture == Architecture.AARCH64 ? 'aarch64-' : ''}${variantString}docker" } -private static String taskName(final String prefix, final Architecture architecture, final boolean oss, final String suffix) { - return "${prefix}${architecture == Architecture.AARCH64 ? 'Aarch64' : ''}${oss ? 'Oss' : ''}${suffix}" +private static String taskName(String prefix, Architecture architecture, Variant variant, String suffix) { + String variantString = variant == Variant.DEFAULT ? '' : variant.name().tokenize("_")*.toLowerCase()*.capitalize().join() + return "${prefix}${architecture == Architecture.AARCH64 ? 'Aarch64' : ''}${variantString}${suffix}" } project.ext { - dockerBuildContext = { Architecture architecture, boolean oss, boolean ubi, boolean local -> + dockerBuildContext = { Architecture architecture, Variant variant, boolean local -> copySpec { - final Map varExpansions = expansions(architecture, oss, ubi, local) + final Map varExpansions = expansions(architecture, variant, local) final Path projectDir = project.projectDir.toPath() - if (ubi) { + if (variant == Variant.UBI) { into('scripts') { from projectDir.resolve("src/docker/bin") from(projectDir.resolve("src/docker/config")) { @@ -149,7 +167,7 @@ project.ext { */ duplicatesStrategy = DuplicatesStrategy.INCLUDE from projectDir.resolve("src/docker/config") - if (oss) { + if (variant == Variant.OSS) { from projectDir.resolve("src/docker/config/oss") } } @@ -162,23 +180,23 @@ project.ext { } } -void addCopyDockerContextTask(final Architecture architecture, final boolean oss) { - task(taskName("copy", architecture, oss, "DockerContext"), type: Sync) { - expansions(architecture, oss, false, true).findAll { it.key != 'build_date' }.each { k, v -> +void addCopyDockerContextTask(Architecture architecture, Variant variant) { + task(taskName("copy", architecture, variant, "DockerContext"), type: Sync) { + expansions(architecture, variant, true).findAll { it.key != 'build_date' }.each { k, v -> inputs.property(k, { v.toString() }) } - into buildPath(architecture, oss) + into buildPath(architecture, variant) - with dockerBuildContext(architecture, oss, false, true) + with dockerBuildContext(architecture, variant, true) if (architecture == Architecture.AARCH64) { - if (oss) { + if (variant == Variant.OSS) { from configurations.aarch64OssDockerSource } else { from configurations.aarch64DockerSource } } else { - if (oss) { + if (variant == Variant.OSS) { from configurations.ossDockerSource } else { from configurations.dockerSource @@ -254,23 +272,24 @@ task integTest(type: Test) { check.dependsOn integTest -void addBuildDockerImage(final Architecture architecture, final boolean oss) { - final Task buildDockerImageTask = task(taskName("build", architecture, oss, "DockerImage"), type: DockerBuildTask) { - TaskProvider copyContextTask = tasks.named(taskName("copy", architecture, oss, "DockerContext")) +void addBuildDockerImage(Architecture architecture, Variant variant) { + final Task buildDockerImageTask = task(taskName("build", architecture, variant, "DockerImage"), type: DockerBuildTask) { + TaskProvider copyContextTask = tasks.named(taskName("copy", architecture, variant, "DockerContext")) dependsOn(copyContextTask) dockerContext.fileProvider(copyContextTask.map { it.destinationDir }) - if (oss) { + if (variant == Variant.OSS) { tags = [ "docker.elastic.co/elasticsearch/elasticsearch-oss:${VersionProperties.elasticsearch}", "elasticsearch-oss:test" ] } else { + String suffix = variant == Variant.UBI ? '-ubi8' : variant == Variant.UBI_MINIMAL ? '-ubi8-minimal' : '' tags = [ - "elasticsearch:${VersionProperties.elasticsearch}", - "docker.elastic.co/elasticsearch/elasticsearch:${VersionProperties.elasticsearch}", - "docker.elastic.co/elasticsearch/elasticsearch-full:${VersionProperties.elasticsearch}", - "elasticsearch:test", + "elasticsearch${suffix}:${VersionProperties.elasticsearch}", + "docker.elastic.co/elasticsearch/elasticsearch${suffix}:${VersionProperties.elasticsearch}", + "docker.elastic.co/elasticsearch/elasticsearch-full${suffix}:${VersionProperties.elasticsearch}", + "elasticsearch${suffix}:test", ] } } @@ -278,13 +297,13 @@ void addBuildDockerImage(final Architecture architecture, final boolean oss) { assemble.dependsOn(buildDockerImageTask) } -// Note that the build process for UBI is different enough that -// we only provide a task for generating the build context, not actually -// building the image. See the `ubi-docker-build-context` sub-project. for (final Architecture architecture : Architecture.values()) { - for (final boolean oss : [false, true]) { - addCopyDockerContextTask(architecture, oss) - addBuildDockerImage(architecture, oss) + for (Variant variant : Variant.values()) { + if (architecture != Architecture.X64 && variant == Variant.UBI) { + continue + } + addCopyDockerContextTask(architecture, variant) + addBuildDockerImage(architecture, variant) } } @@ -305,10 +324,15 @@ subprojects { Project subProject -> final Architecture architecture = subProject.name.contains('aarch64-') ? Architecture.AARCH64 : Architecture.X64 final boolean oss = subProject.name.contains('oss-') + // We don't export UBI at the moment + final Variant variant = oss ? Variant.OSS : Variant.DEFAULT + + final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' + final String suffix = oss ? '-oss' : '' - def exportTaskName = taskName("export", architecture, oss, "DockerImage") - def buildTaskName = taskName("build", architecture, oss, "DockerImage") - def tarFile = "${parent.projectDir}/build/elasticsearch${architecture == Architecture.AARCH64 ? '-aarch64' : ''}${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar" + def exportTaskName = taskName("export", architecture, variant, "DockerImage") + def buildTaskName = taskName("build", architecture, variant, "DockerImage") + def tarFile = "${parent.projectDir}/build/elasticsearch${arch}${suffix}_test.${VersionProperties.elasticsearch}.docker.tar" final Task exportDockerImageTask = task(exportTaskName, type: LoggedExec) { inputs.file("${parent.projectDir}/build/markers/${buildTaskName}.marker") @@ -317,7 +341,7 @@ subprojects { Project subProject -> args "save", "-o", tarFile, - "elasticsearch${oss ? '-oss' : ''}:test" + "elasticsearch${suffix}:test" } exportDockerImageTask.dependsOn(parent.tasks.getByName(buildTaskName)) @@ -326,7 +350,7 @@ subprojects { Project subProject -> artifacts.add('default', file(tarFile)) { type 'tar' - name "elasticsearch${architecture == Architecture.AARCH64 ? '-aarch64' : ''}${oss ? '-oss' : ''}" + name "elasticsearch${arch}${suffix}" builtBy exportTaskName } diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle index 15604ce94c04f..45f36f1b61eeb 100644 --- a/distribution/docker/docker-build-context/build.gradle +++ b/distribution/docker/docker-build-context/build.gradle @@ -1,3 +1,5 @@ +import org.elasticsearch.gradle.Variant + apply plugin: 'base' task buildDockerBuildContext(type: Tar) { @@ -6,7 +8,7 @@ task buildDockerBuildContext(type: Tar) { archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch" // Non-local builds don't need to specify an architecture. - with dockerBuildContext(null, false, false, false) + with dockerBuildContext(null, Variant.DEFAULT, false) } assemble.dependsOn buildDockerBuildContext diff --git a/distribution/docker/oss-docker-build-context/build.gradle b/distribution/docker/oss-docker-build-context/build.gradle index 172bfe0c7e781..3cae2cf8e1306 100644 --- a/distribution/docker/oss-docker-build-context/build.gradle +++ b/distribution/docker/oss-docker-build-context/build.gradle @@ -1,3 +1,5 @@ +import org.elasticsearch.gradle.Variant + apply plugin: 'base' task buildOssDockerBuildContext(type: Tar) { @@ -6,7 +8,7 @@ task buildOssDockerBuildContext(type: Tar) { archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch-oss" // Non-local builds don't need to specify an architecture. - with dockerBuildContext(null, true, false, false) + with dockerBuildContext(null, Variant.OSS, false) } assemble.dependsOn buildOssDockerBuildContext diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 5844d16ba0889..f95ad1ae87f20 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -18,19 +18,18 @@ # Build stage 0 `builder`: # Extract Elasticsearch artifact ################################################################################ -<% if (is_ubi) { %> -ARG BASE_REGISTRY=nexus-docker-secure.levelup-nexus.svc.cluster.local:18082 -ARG BASE_IMAGE=redhat/ubi/ubi8 -ARG BASE_TAG=8.2 - -FROM \${BASE_REGISTRY}/\${BASE_IMAGE}:\${BASE_TAG} AS builder +${build_args} +FROM ${base_image} AS builder +<% if (variant == 'ubi_minimal') { %> +# Install required packages to extract the Elasticsearch distribution +RUN ${package_manager} install -y tar gzip +<% } %> +<% if (variant == 'ubi') { %> # `tini` is a tiny but valid init for containers. This is used to cleanly # control how ES and any child processes are shut down. COPY tini /tini <% } else { %> -FROM centos:7 AS builder - # `tini` is a tiny but valid init for containers. This is used to cleanly # control how ES and any child processes are shut down. # @@ -69,16 +68,14 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties # Copy elasticsearch from stage 0 # Add entrypoint ################################################################################ -<% if (is_ubi) { %> -FROM \${BASE_REGISTRY}/\${BASE_IMAGE}:\${BASE_TAG} AS builder -<% } else { %> -FROM centos:7 -<% } %> +FROM ${base_image} + ENV ELASTIC_CONTAINER true -RUN yum update --setopt=tsflags=nodocs -y && \\ - yum install --setopt=tsflags=nodocs -y nc shadow-utils zip unzip && \\ - yum clean all +RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\ + ${package_manager} install --setopt=tsflags=nodocs -y \\ + nc shadow-utils zip unzip <%= variant == 'ubi_minimal' ? 'findutils' : '' %> && \\ + ${package_manager} clean all RUN groupadd -g 1000 elasticsearch && \\ adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\ @@ -131,7 +128,7 @@ USER elasticsearch:root ENTRYPOINT ["/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] # Dummy overridable parameter parsed by entrypoint CMD ["eswrapper"] -<% if (is_ubi) { %> +<% if (variant == 'ubi') { %> HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9200 || exit 1 <% } %> ################################################################################ diff --git a/distribution/docker/ubi-docker-build-context/build.gradle b/distribution/docker/ubi-docker-build-context/build.gradle index 72b0845531e08..dc6f315e1c884 100644 --- a/distribution/docker/ubi-docker-build-context/build.gradle +++ b/distribution/docker/ubi-docker-build-context/build.gradle @@ -1,4 +1,5 @@ import org.elasticsearch.gradle.Architecture +import org.elasticsearch.gradle.Variant apply plugin: 'base' @@ -7,7 +8,9 @@ task buildUbiDockerBuildContext(type: Tar) { compression = Compression.GZIP archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch-ubi8" - with dockerBuildContext(Architecture.X64, false, true, false) + with dockerBuildContext(Architecture.X64, Variant.UBI, false) } +buildUbiDockerBuildContext.dependsOn project.parent.tasks.named('copyUbiDockerContext') + assemble.dependsOn buildUbiDockerBuildContext diff --git a/distribution/docker/ubi-minimal-docker-build-context/build.gradle b/distribution/docker/ubi-minimal-docker-build-context/build.gradle new file mode 100644 index 0000000000000..5703b22642172 --- /dev/null +++ b/distribution/docker/ubi-minimal-docker-build-context/build.gradle @@ -0,0 +1,13 @@ +import org.elasticsearch.gradle.Variant + +apply plugin: 'base' + +task buildUbiMinimalDockerBuildContext(type: Tar) { + archiveExtension = 'tar.gz' + compression = Compression.GZIP + archiveClassifier = "docker-build-context" + archiveBaseName = "elasticsearch-ubi8-minimal" + with dockerBuildContext(null, Variant.UBI_MINIMAL, false) +} + +assemble.dependsOn buildUbiMinimalDockerBuildContext diff --git a/settings.gradle b/settings.gradle index 56b675f8d559d..63abcbac25b00 100644 --- a/settings.gradle +++ b/settings.gradle @@ -42,6 +42,7 @@ List projects = [ 'distribution:docker:docker-export', 'distribution:docker:oss-docker-build-context', 'distribution:docker:oss-docker-export', + 'distribution:docker:ubi-minimal-docker-build-context', 'distribution:docker:ubi-docker-build-context', 'distribution:packages:aarch64-oss-deb', 'distribution:packages:oss-deb', From 051daff7831db9e2c21458553ab25d37cffef36c Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 11 Aug 2020 10:39:20 +0100 Subject: [PATCH 09/32] Tweaks --- distribution/docker/build.gradle | 42 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 76637812a6b38..8b2feaea49ee3 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -98,36 +98,36 @@ RUN curl --retry 8 -S -L \\ 'source_elasticsearch': sourceElasticsearch, 'variant' : variant.name().toLowerCase(), 'version' : VersionProperties.elasticsearch, - 'version_hash' : variant == Variant.UBI ? getChecksum(configurations.dockerSource.singleFile) : '' + 'version_hash' : variant == Variant.UBI ? '' : '' ] } -private static String getChecksum(File file) { - MessageDigest digest = MessageDigest.getInstance("SHA-512") +//private static String getChecksum(File file) { +// MessageDigest digest = MessageDigest.getInstance("SHA-512") - FileInputStream fis = new FileInputStream(file) - byte[] byteArray = new byte[1024] - int bytesCount = 0 +// FileInputStream fis = new FileInputStream(file) +// byte[] byteArray = new byte[1024] +// int bytesCount = 0 - //Read file data and update in message digest - while ((bytesCount = fis.read(byteArray)) != -1) { - digest.update(byteArray, 0, bytesCount) - } +// //Read file data and update in message digest +// while ((bytesCount = fis.read(byteArray)) != -1) { +// digest.update(byteArray, 0, bytesCount) +// } - fis.close() +// fis.close() - //Get the hash's bytes - byte[] bytes = digest.digest() +// //Get the hash's bytes +// byte[] bytes = digest.digest() - //This bytes[] has bytes in decimal format; - //Convert it to hexadecimal format - StringBuilder sb = new StringBuilder() - for (int i = 0; i < bytes.length; i++) { - sb.append(Integer.toString((bytes[i] & 0xff) + 0x100, 16).substring(1)) - } +// //This bytes[] has bytes in decimal format; +// //Convert it to hexadecimal format +// StringBuilder sb = new StringBuilder() +// for (int i = 0; i < bytes.length; i++) { +// sb.append(Integer.toString((bytes[i] & 0xff) + 0x100, 16).substring(1)) +// } - return sb.toString() -} +// return sb.toString() +//} private static String buildPath(Architecture architecture, Variant variant) { String variantString = variant == Variant.DEFAULT ? '' : variant.name().toLowerCase().replace('_', '-') + '-' From d544309b5c38e9f128d5c994d10e676515436031 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 14 Aug 2020 20:13:11 +0100 Subject: [PATCH 10/32] Fixes --- distribution/docker/build.gradle | 7 ++----- distribution/docker/src/docker/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 8b2feaea49ee3..37822d4d29b0e 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -58,13 +58,12 @@ ext.expansions = { Architecture architecture, Variant variant, boolean local -> ARG BASE_REGISTRY=nexus-docker-secure.levelup-nexus.svc.cluster.local:18082 ARG BASE_IMAGE=redhat/ubi/ubi8 ARG BASE_TAG=8.2 -ARG TARBALL=${elasticsearch} """ } String baseImage = 'centos:8' if (variant == Variant.UBI) { - baseImage = '${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG} AS builder' + baseImage = '${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}' } else if (variant == Variant.UBI_MINIMAL) { baseImage = 'registry.access.redhat.com/ubi8/ubi-minimal:8.2' } @@ -74,10 +73,8 @@ ARG TARBALL=${elasticsearch} * follow in the Dockerfile don't have to know about the runtime * architecture. */ String sourceElasticsearch - if (local) { + if (local || variant == Variant.UBI) { sourceElasticsearch = "COPY ${elasticsearch} /opt/elasticsearch.tar.gz" - } else if (variant == Variant.UBI) { - sourceElasticsearch = 'COPY ${TARBALL} /opt/elasticsearch.tar.gz' } else { sourceElasticsearch = """ RUN curl --retry 8 -S -L \\ diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 3a9600d0247ae..9db96b9d9dc93 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -55,7 +55,7 @@ WORKDIR /usr/share/elasticsearch ${source_elasticsearch} -RUN tar zxf /opt/elasticsearch.tar.gz --strip-components=1 +RUN tar -zxf /opt/elasticsearch.tar.gz --strip-components=1 RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env RUN mkdir -p config config/jvm.options.d data logs RUN chmod 0775 config config/jvm.options.d data logs From 82f8bdaebcf45fc8ab45ff10cfa1820991a25f9f Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 26 Oct 2020 18:32:50 +0000 Subject: [PATCH 11/32] More changes port-merge --- .../java/org/elasticsearch/gradle/DockerBase.java | 4 +++- distribution/docker/build.gradle | 2 +- .../docker/docker-build-context/build.gradle | 1 - .../ironbank-docker-build-context/build.gradle | 13 +++++++++++++ .../docker/ubi-docker-build-context/build.gradle | 2 +- .../ubi-minimal-docker-build-context/build.gradle | 13 ------------- 6 files changed, 18 insertions(+), 17 deletions(-) create mode 100644 distribution/docker/ironbank-docker-build-context/build.gradle delete mode 100644 distribution/docker/ubi-minimal-docker-build-context/build.gradle diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java index cab250e3417b4..2a60a873079a4 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java @@ -25,7 +25,9 @@ public enum DockerBase { CENTOS("centos:8"), // "latest" here is intentional, since the image name specifies "8" - UBI("docker.elastic.co/ubi8/ubi-minimal:latest"); + UBI("docker.elastic.co/ubi8/ubi-minimal:latest"), + // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build + IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}"); private final String image; diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 736012d7ae718..0e677664e97bf 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -47,7 +47,7 @@ ext.expansions = { Architecture architecture, boolean oss, DockerBase base, bool final String elasticsearch = "elasticsearch-" + (oss ? 'oss-' : '') + - "VersionProperties.elasticsearch}-${classifier}.tar" + + "${VersionProperties.elasticsearch}-${classifier}.tar" + '.gz' String buildArgs = '#' diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle index 0f5a9e5d6773f..10bf433b10a92 100644 --- a/distribution/docker/docker-build-context/build.gradle +++ b/distribution/docker/docker-build-context/build.gradle @@ -7,7 +7,6 @@ tasks.register("buildDockerBuildContext", Tar) { compression = Compression.GZIP archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch" - // Non-local builds don't need to specify an architecture. with dockerBuildContext(null, false, DockerBase.CENTOS, false) } diff --git a/distribution/docker/ironbank-docker-build-context/build.gradle b/distribution/docker/ironbank-docker-build-context/build.gradle new file mode 100644 index 0000000000000..4b282343bd9a5 --- /dev/null +++ b/distribution/docker/ironbank-docker-build-context/build.gradle @@ -0,0 +1,13 @@ +import org.elasticsearch.gradle.DockerBase + +apply plugin: 'base' + +tasks.register("buildIronBankAarch64DockerBuildContext", Tar) { + archiveExtension = 'tar.gz' + compression = Compression.GZIP + archiveClassifier = "docker-build-context" + archiveBaseName = "elasticsearch-ironbank" + with dockerBuildContext(null, false, DockerBase.IRON_BANK, false) +} + +tasks.named("assemble").configure {dependsOn "buildIronBankAarch64DockerBuildContext"} diff --git a/distribution/docker/ubi-docker-build-context/build.gradle b/distribution/docker/ubi-docker-build-context/build.gradle index 3fb9ba3a2b42f..129aba139b99f 100644 --- a/distribution/docker/ubi-docker-build-context/build.gradle +++ b/distribution/docker/ubi-docker-build-context/build.gradle @@ -10,4 +10,4 @@ task buildUbiDockerBuildContext(type: Tar) { with dockerBuildContext(null, false, DockerBase.UBI, false) } -assemble.dependsOn buildUbiDockerBuildContext +tasks.named("assemble").configure {dependsOn "buildUbiDockerBuildContext"} diff --git a/distribution/docker/ubi-minimal-docker-build-context/build.gradle b/distribution/docker/ubi-minimal-docker-build-context/build.gradle deleted file mode 100644 index 5703b22642172..0000000000000 --- a/distribution/docker/ubi-minimal-docker-build-context/build.gradle +++ /dev/null @@ -1,13 +0,0 @@ -import org.elasticsearch.gradle.Variant - -apply plugin: 'base' - -task buildUbiMinimalDockerBuildContext(type: Tar) { - archiveExtension = 'tar.gz' - compression = Compression.GZIP - archiveClassifier = "docker-build-context" - archiveBaseName = "elasticsearch-ubi8-minimal" - with dockerBuildContext(null, Variant.UBI_MINIMAL, false) -} - -assemble.dependsOn buildUbiMinimalDockerBuildContext From cd9accfeea73836d27f05e3029feb0d116680c72 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 26 Oct 2020 20:40:15 +0000 Subject: [PATCH 12/32] Lots of fixes, plus Dockerfile newline squashing --- distribution/docker/build.gradle | 21 ++++++++++--- .../docker/docker-build-context/build.gradle | 1 + .../build.gradle | 7 +++-- distribution/docker/src/docker/Dockerfile | 31 +++++++++++++++++-- .../src/docker/{ubi => iron_bank}/.gitignore | 0 .../src/docker/{ubi => iron_bank}/Jenkinsfile | 0 .../src/docker/{ubi => iron_bank}/README.md | 0 .../docker/{ubi => iron_bank}/downloads.json | 0 settings.gradle | 1 + 9 files changed, 52 insertions(+), 9 deletions(-) rename distribution/docker/src/docker/{ubi => iron_bank}/.gitignore (100%) rename distribution/docker/src/docker/{ubi => iron_bank}/Jenkinsfile (100%) rename distribution/docker/src/docker/{ubi => iron_bank}/README.md (100%) rename distribution/docker/src/docker/{ubi => iron_bank}/downloads.json (100%) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 0e677664e97bf..a0c39f22775e8 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -7,6 +7,8 @@ import org.elasticsearch.gradle.docker.DockerBuildTask import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin +import java.nio.file.Path + apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.test.fixtures' apply plugin: 'elasticsearch.internal-distribution-download' @@ -76,20 +78,30 @@ RUN curl --retry 8 -S -L \\ return [ 'base_image' : base.getImage(), - 'bin_dir' : variant == DockerBase.IRON_BANK ? 'scripts' : 'bin', + 'bin_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'bin', 'build_args' : buildArgs, 'build_date' : BuildParams.buildDate, - 'config_dir' : variant == DockerBase.IRON_BANK ? 'scripts' : 'config', + 'config_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'config', 'git_revision' : BuildParams.gitRevision, 'license' : oss ? 'Apache-2.0' : 'Elastic-License', 'package_manager' : base == DockerBase.IRON_BANK ? 'microdnf' : 'yum', 'source_elasticsearch': sourceElasticsearch, 'docker_base' : base.name().toLowerCase(), 'version' : VersionProperties.elasticsearch, - 'version_hash' : variant == DockerBase.IRON_BANK ? '' : '', + 'version_hash' : base == DockerBase.IRON_BANK ? '' : '', ] } +/** + * This filter squashes long runs of newlines so that the output + * is a little more aesthetically pleasing. + */ +class MarkdownFilter extends FilterReader { + MarkdownFilter(Reader input) { + super(new StringReader(input.text.replaceAll("\n{2,}", "\n\n"))) + } +} + private static String buildPath(Architecture architecture, boolean oss, DockerBase base) { return 'build/' + (architecture == Architecture.AARCH64 ? 'aarch64-' : '') + @@ -134,7 +146,7 @@ project.ext { // build-context sub-modules. duplicatesStrategy = DuplicatesStrategy.INCLUDE from projectDir.resolve("src/docker/config") - if (variant == Variant.OSS) { + if (oss) { from projectDir.resolve("src/docker/config/oss") } } @@ -142,6 +154,7 @@ project.ext { from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) { expand(varExpansions) + filter MarkdownFilter } } } diff --git a/distribution/docker/docker-build-context/build.gradle b/distribution/docker/docker-build-context/build.gradle index 10bf433b10a92..0f5a9e5d6773f 100644 --- a/distribution/docker/docker-build-context/build.gradle +++ b/distribution/docker/docker-build-context/build.gradle @@ -7,6 +7,7 @@ tasks.register("buildDockerBuildContext", Tar) { compression = Compression.GZIP archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch" + // Non-local builds don't need to specify an architecture. with dockerBuildContext(null, false, DockerBase.CENTOS, false) } diff --git a/distribution/docker/ironbank-docker-build-context/build.gradle b/distribution/docker/ironbank-docker-build-context/build.gradle index 4b282343bd9a5..6164be661f4d8 100644 --- a/distribution/docker/ironbank-docker-build-context/build.gradle +++ b/distribution/docker/ironbank-docker-build-context/build.gradle @@ -1,13 +1,14 @@ +import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.DockerBase apply plugin: 'base' -tasks.register("buildIronBankAarch64DockerBuildContext", Tar) { +tasks.register("buildIronBankDockerBuildContext", Tar) { archiveExtension = 'tar.gz' compression = Compression.GZIP archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch-ironbank" - with dockerBuildContext(null, false, DockerBase.IRON_BANK, false) + with dockerBuildContext(Architecture.X64, false, DockerBase.IRON_BANK, false) } -tasks.named("assemble").configure {dependsOn "buildIronBankAarch64DockerBuildContext"} +tasks.named("assemble").configure {dependsOn "buildIronBankDockerBuildContext"} diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 1551dc4954500..5e77f0644debb 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -3,6 +3,7 @@ # # Beginning of multi stage Dockerfile ################################################################################ + <% /* This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes have to be escaped in order for them to appear in the final Dockerfile. You @@ -13,7 +14,11 @@ We use control-flow tags in this file to conditionally render the content. The layout/presentation here has been adjusted so that it looks reasonable when rendered, at the slight expense of how it looks here. + + Note that this file is also filtered to squash together newlines, so we can + add as many newlines here as necessary to improve legibility. */ %> + <% if (docker_base == "ubi") { %> ################################################################################ # Build stage 0 `builder`: @@ -43,7 +48,8 @@ RUN set -eux ; \\ rm \${tini_bin}.sha256sum ; \\ mv \${tini_bin} /bin/tini ; \\ chmod +x /bin/tini -<% else if (docker_base == 'iron_bank') { %> + +<% } else if (docker_base == 'iron_bank') { %> ${build_args} FROM ${base_image} AS builder @@ -51,7 +57,11 @@ FROM ${base_image} AS builder # `tini` is a tiny but valid init for containers. This is used to cleanly # control how ES and any child processes are shut down. COPY tini /tini + <% } else { %> + +<% /* CentOS builds are actaully a custom base image with a minimal set of dependencies */ %> + ################################################################################ # Stage 1. Build curl statically. Installing it from RPM on CentOS pulls in too # many dependencies. @@ -201,6 +211,7 @@ COPY --from=curl /work/curl /rootfs/usr/bin/curl # Step 3. Fetch the Elasticsearch distribution and configure it for Docker ################################################################################ FROM ${base_image} AS builder + <% } %> RUN mkdir /usr/share/elasticsearch @@ -219,6 +230,7 @@ COPY ${config_dir}/elasticsearch.yml ${config_dir}/log4j2.properties config/ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties <% if (docker_base == "ubi" || docker_base == "iron_bank") { %> + ################################################################################ # Build stage 1 (the actual Elasticsearch image): # @@ -228,7 +240,8 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties FROM ${base_image} -<% if docker_base == "ubi" %> +<% if (docker_base == "ubi") { %> + RUN for iter in {1..10}; do \\ ${package_manager} update --setopt=tsflags=nodocs -y && \\ ${package_manager} install --setopt=tsflags=nodocs -y \\ @@ -238,18 +251,27 @@ RUN for iter in {1..10}; do \\ sleep 10; \\ done; \\ (exit \$exit_code) + %> } else { %> + +<% +/* Reviews of the Iron Bank Dockerfile said that they preferred simpler, */ +/* scripting so this version doesn't have the retry loop featured above. */ +%> RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\ ${package_manager} install --setopt=tsflags=nodocs -y \\ nc shadow-utils zip unzip && \\ ${package_manager} clean all + <% } %> RUN groupadd -g 1000 elasticsearch && \\ adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\ chmod 0775 /usr/share/elasticsearch && \\ chown -R 1000:0 /usr/share/elasticsearch + <% } else { %> + ################################################################################ # Stage 4. Build the final image, using the rootfs above as the basis, and # copying in the Elasticsearch distribution @@ -264,12 +286,14 @@ RUN addgroup -g 1000 elasticsearch && \\ addgroup elasticsearch root && \\ chmod 0775 /usr/share/elasticsearch && \\ chgrp 0 /usr/share/elasticsearch + <% } %> ENV ELASTIC_CONTAINER true WORKDIR /usr/share/elasticsearch COPY --from=builder --chown=1000:0 /usr/share/elasticsearch /usr/share/elasticsearch + <% if (docker_base == "ubi" || docker_base == "iron_bank") { %> COPY --from=builder --chown=0:0 /bin/tini /bin/tini <% } %> @@ -314,6 +338,7 @@ LABEL org.label-schema.build-date="${build_date}" \\ org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\ org.opencontainers.image.vendor="Elastic" \\ org.opencontainers.image.version="${version}" + <% if (docker_base == 'ubi') { %> LABEL name="Elasticsearch" \\ maintainer="infra@elastic.co" \\ @@ -334,9 +359,11 @@ ENTRYPOINT ["/bin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] # Dummy overridable parameter parsed by entrypoint CMD ["eswrapper"] + <% if (docker_base == 'iron_bank') { %> HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9200 || exit 1 <% } %> + ################################################################################ # End of multi-stage Dockerfile ################################################################################ diff --git a/distribution/docker/src/docker/ubi/.gitignore b/distribution/docker/src/docker/iron_bank/.gitignore similarity index 100% rename from distribution/docker/src/docker/ubi/.gitignore rename to distribution/docker/src/docker/iron_bank/.gitignore diff --git a/distribution/docker/src/docker/ubi/Jenkinsfile b/distribution/docker/src/docker/iron_bank/Jenkinsfile similarity index 100% rename from distribution/docker/src/docker/ubi/Jenkinsfile rename to distribution/docker/src/docker/iron_bank/Jenkinsfile diff --git a/distribution/docker/src/docker/ubi/README.md b/distribution/docker/src/docker/iron_bank/README.md similarity index 100% rename from distribution/docker/src/docker/ubi/README.md rename to distribution/docker/src/docker/iron_bank/README.md diff --git a/distribution/docker/src/docker/ubi/downloads.json b/distribution/docker/src/docker/iron_bank/downloads.json similarity index 100% rename from distribution/docker/src/docker/ubi/downloads.json rename to distribution/docker/src/docker/iron_bank/downloads.json diff --git a/settings.gradle b/settings.gradle index 62035638d9660..d6002a24c6495 100644 --- a/settings.gradle +++ b/settings.gradle @@ -38,6 +38,7 @@ List projects = [ 'distribution:docker:docker-aarch64-export', 'distribution:docker:docker-build-context', 'distribution:docker:docker-export', + 'distribution:docker:ironbank-docker-build-context', 'distribution:docker:oss-docker-aarch64-build-context', 'distribution:docker:oss-docker-aarch64-export', 'distribution:docker:oss-docker-build-context', From 01cf6cc784c6a6cb8fe8a882b1b7c3b6195d16fe Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 27 Oct 2020 11:17:03 +0000 Subject: [PATCH 13/32] More fixes --- .../org/elasticsearch/gradle/Variant.java | 27 ------------------- distribution/docker/build.gradle | 9 +++---- distribution/docker/src/docker/Dockerfile | 7 ++--- 3 files changed, 7 insertions(+), 36 deletions(-) delete mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/Variant.java diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Variant.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Variant.java deleted file mode 100644 index 7bb023b7af145..0000000000000 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/Variant.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.gradle; - -public enum Variant { - DEFAULT, - OSS, - UBI, - UBI_MINIMAL -} diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index a0c39f22775e8..bf3df08efb7c8 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -47,10 +47,7 @@ ext.expansions = { Architecture architecture, boolean oss, DockerBase base, bool classifier = "linux-\$(arch)" } - final String elasticsearch = "elasticsearch-" + - (oss ? 'oss-' : '') + - "${VersionProperties.elasticsearch}-${classifier}.tar" + - '.gz' + final String elasticsearch = "elasticsearch-${oss ? 'oss-' : ''}$VersionProperties.elasticsearch-${classifier}.tar.gz" String buildArgs = '#' if (base == DockerBase.IRON_BANK) { @@ -67,7 +64,7 @@ ARG BASE_TAG=8.2 * architecture. */ String sourceElasticsearch if (local || base == DockerBase.IRON_BANK) { - sourceElasticsearch = "COPY ${elasticsearch} /opt/elasticsearch.tar.gz" + sourceElasticsearch = "COPY $elasticsearch /opt/elasticsearch.tar.gz" } else { sourceElasticsearch = """ RUN curl --retry 8 -S -L \\ @@ -84,7 +81,7 @@ RUN curl --retry 8 -S -L \\ 'config_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'config', 'git_revision' : BuildParams.gitRevision, 'license' : oss ? 'Apache-2.0' : 'Elastic-License', - 'package_manager' : base == DockerBase.IRON_BANK ? 'microdnf' : 'yum', + 'package_manager' : base == DockerBase.UBI ? 'microdnf' : 'yum', 'source_elasticsearch': sourceElasticsearch, 'docker_base' : base.name().toLowerCase(), 'version' : VersionProperties.elasticsearch, diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 5e77f0644debb..54a44b5913bdc 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -56,7 +56,8 @@ FROM ${base_image} AS builder # `tini` is a tiny but valid init for containers. This is used to cleanly # control how ES and any child processes are shut down. -COPY tini /tini +COPY tini /bin/tini +RUN chmod 0755 /bin/tini <% } else { %> @@ -305,7 +306,7 @@ RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk ENV PATH /usr/share/elasticsearch/bin:\$PATH -COPY bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh # 1. The JDK's directories' permissions don't allow `java` to be executed under a different # group to the default. Fix this. @@ -339,7 +340,7 @@ LABEL org.label-schema.build-date="${build_date}" \\ org.opencontainers.image.vendor="Elastic" \\ org.opencontainers.image.version="${version}" -<% if (docker_base == 'ubi') { %> +<% if (docker_base == 'ubi' || docker_base == 'iron_bank') { %> LABEL name="Elasticsearch" \\ maintainer="infra@elastic.co" \\ vendor="Elastic" \\ From 295e0f173e6c7daeb6fc888d1e651c931a9b6aff Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 27 Oct 2020 11:17:39 +0000 Subject: [PATCH 14/32] Fix --- distribution/docker/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index bf3df08efb7c8..81844cd1fff96 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -47,7 +47,7 @@ ext.expansions = { Architecture architecture, boolean oss, DockerBase base, bool classifier = "linux-\$(arch)" } - final String elasticsearch = "elasticsearch-${oss ? 'oss-' : ''}$VersionProperties.elasticsearch-${classifier}.tar.gz" + final String elasticsearch = "elasticsearch-${oss ? 'oss-' : ''}${VersionProperties.elasticsearch}-${classifier}.tar.gz" String buildArgs = '#' if (base == DockerBase.IRON_BANK) { From 9363ecc56cfa2b5e55d09d8e192ef4c279f64748 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 27 Oct 2020 11:29:42 +0000 Subject: [PATCH 15/32] Fix filter name --- distribution/docker/build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 81844cd1fff96..14cbf7c552b65 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -93,8 +93,8 @@ RUN curl --retry 8 -S -L \\ * This filter squashes long runs of newlines so that the output * is a little more aesthetically pleasing. */ -class MarkdownFilter extends FilterReader { - MarkdownFilter(Reader input) { +class SquashNewlinesFilter extends FilterReader { + SquashNewlinesFilter(Reader input) { super(new StringReader(input.text.replaceAll("\n{2,}", "\n\n"))) } } @@ -138,7 +138,7 @@ project.ext { } into('config') { - // Oss and default distribution can have different configuration, therefore we want to + // The OSS and default distribution can have different configuration, therefore we want to // allow overriding the default configuration by creating config files in oss or default // build-context sub-modules. duplicatesStrategy = DuplicatesStrategy.INCLUDE @@ -151,7 +151,7 @@ project.ext { from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) { expand(varExpansions) - filter MarkdownFilter + filter SquashNewlinesFilter } } } From 597108016bd94f57e5a3772cc7687916c7234d26 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 29 Oct 2020 12:17:05 +0000 Subject: [PATCH 16/32] Tweaks --- distribution/docker/build.gradle | 1 - distribution/docker/src/docker/Dockerfile | 2 +- distribution/docker/src/docker/iron_bank/downloads.json | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 14cbf7c552b65..613c3595dc8b2 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -85,7 +85,6 @@ RUN curl --retry 8 -S -L \\ 'source_elasticsearch': sourceElasticsearch, 'docker_base' : base.name().toLowerCase(), 'version' : VersionProperties.elasticsearch, - 'version_hash' : base == DockerBase.IRON_BANK ? '' : '', ] } diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index 54a44b5913bdc..e2ad6a73f2efa 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -256,7 +256,7 @@ RUN for iter in {1..10}; do \\ %> } else { %> <% -/* Reviews of the Iron Bank Dockerfile said that they preferred simpler, */ +/* Reviews of the Iron Bank Dockerfile said that they preferred simpler */ /* scripting so this version doesn't have the retry loop featured above. */ %> RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\ diff --git a/distribution/docker/src/docker/iron_bank/downloads.json b/distribution/docker/src/docker/iron_bank/downloads.json index d3c4b378697a4..63f61a289b0e7 100644 --- a/distribution/docker/src/docker/iron_bank/downloads.json +++ b/distribution/docker/src/docker/iron_bank/downloads.json @@ -5,7 +5,7 @@ "filename": "elasticsearch-${version}-linux-x86_64.tar.gz", "validation": { "type": "sha512", - "value": "${version_hash}" + "value": "" } }, { From b3f02117d06a086900915f95d3cc017676116257 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 5 Nov 2020 09:29:38 +0000 Subject: [PATCH 17/32] Address review feedback --- distribution/docker/build.gradle | 10 +++++----- .../docker/ironbank-docker-build-context/build.gradle | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 613c3595dc8b2..712493e142060 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -32,7 +32,7 @@ dependencies { ext.expansions = { Architecture architecture, boolean oss, DockerBase base, boolean local -> String classifier - if (local || base == DockerBase.IRON_BANK) { + if (local) { if (architecture == Architecture.AARCH64) { classifier = "linux-aarch64" } else if (architecture == Architecture.X64) { @@ -63,7 +63,7 @@ ARG BASE_TAG=8.2 * follow in the Dockerfile don't have to know about the runtime * architecture. */ String sourceElasticsearch - if (local || base == DockerBase.IRON_BANK) { + if (local) { sourceElasticsearch = "COPY $elasticsearch /opt/elasticsearch.tar.gz" } else { sourceElasticsearch = """ @@ -84,7 +84,7 @@ RUN curl --retry 8 -S -L \\ 'package_manager' : base == DockerBase.UBI ? 'microdnf' : 'yum', 'source_elasticsearch': sourceElasticsearch, 'docker_base' : base.name().toLowerCase(), - 'version' : VersionProperties.elasticsearch, + 'version' : VersionProperties.elasticsearch ] } @@ -103,7 +103,7 @@ private static String buildPath(Architecture architecture, boolean oss, DockerBa (architecture == Architecture.AARCH64 ? 'aarch64-' : '') + (oss ? 'oss-' : '') + (base == DockerBase.UBI ? 'ubi-' : '') + - (base == DockerBase.UBI ? 'ubi-' : base == DockerBase.IRON_BANK ? 'ironbank-' : '') + + (base == DockerBase.UBI ? 'ubi-' : (base == DockerBase.IRON_BANK ? 'ironbank-' : '')) + 'docker' } @@ -111,7 +111,7 @@ private static String taskName(String prefix, Architecture architecture, boolean return prefix + (architecture == Architecture.AARCH64 ? 'Aarch64' : '') + (oss ? 'Oss' : '') + - (base == DockerBase.UBI ? 'Ubi' : base == DockerBase.IRON_BANK ? 'IronBank' : '') + + (base == DockerBase.UBI ? 'Ubi' : (base == DockerBase.IRON_BANK ? 'IronBank' : '')) + suffix } diff --git a/distribution/docker/ironbank-docker-build-context/build.gradle b/distribution/docker/ironbank-docker-build-context/build.gradle index 6164be661f4d8..00ddbe6ac374f 100644 --- a/distribution/docker/ironbank-docker-build-context/build.gradle +++ b/distribution/docker/ironbank-docker-build-context/build.gradle @@ -8,7 +8,7 @@ tasks.register("buildIronBankDockerBuildContext", Tar) { compression = Compression.GZIP archiveClassifier = "docker-build-context" archiveBaseName = "elasticsearch-ironbank" - with dockerBuildContext(Architecture.X64, false, DockerBase.IRON_BANK, false) + // We always treat Iron Bank builds as local, because that is how they + // are built + with dockerBuildContext(Architecture.X64, false, DockerBase.IRON_BANK, true) } - -tasks.named("assemble").configure {dependsOn "buildIronBankDockerBuildContext"} From b1b074d06e99f25d4271eee70463706ec682692c Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 6 Nov 2020 12:15:01 +0000 Subject: [PATCH 18/32] Add debugging line --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index e6cb24b2c38fb..b914c1514317d 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -392,6 +392,9 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex startElasticsearch(); stopElasticsearch(); + // (@rory) this is for debugging + Platforms.onLinux(() -> logger.warn(sh.run("ls -l /tmp").stdout)); + Result result = sh.run("echo y | " + installation.executables().nodeTool + " unsafe-bootstrap"); assertThat(result.stdout, containsString("Master node was successfully bootstrapped")); } From d446dce76739494c05857b79d77fa5102b655c22 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 6 Nov 2020 13:39:00 +0000 Subject: [PATCH 19/32] More debugging --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index b914c1514317d..238fde42dff32 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -393,7 +393,11 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex stopElasticsearch(); // (@rory) this is for debugging - Platforms.onLinux(() -> logger.warn(sh.run("ls -l /tmp").stdout)); + Platforms.onLinux(() -> { + logger.warn(sh.run("df").stdout); + logger.warn(sh.run("ls -ld /tmp").stdout); + logger.warn(sh.run("ls -l /tmp").stdout); + }); Result result = sh.run("echo y | " + installation.executables().nodeTool + " unsafe-bootstrap"); assertThat(result.stdout, containsString("Master node was successfully bootstrapped")); From 40fd5965ce9cd884fc676ef5290d7cf803c41917 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 6 Nov 2020 15:12:30 +0000 Subject: [PATCH 20/32] More debugging --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 238fde42dff32..492249fffa0d8 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -394,9 +394,7 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex // (@rory) this is for debugging Platforms.onLinux(() -> { - logger.warn(sh.run("df").stdout); - logger.warn(sh.run("ls -ld /tmp").stdout); - logger.warn(sh.run("ls -l /tmp").stdout); + logger.warn(sh.run("ps ax | grep elasticsearch").stdout); }); Result result = sh.run("echo y | " + installation.executables().nodeTool + " unsafe-bootstrap"); From aa44b5d79f628f10117f458a9b3b05d442e70a11 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 6 Nov 2020 15:26:37 +0000 Subject: [PATCH 21/32] Formatting --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 492249fffa0d8..3f089d9d77a0e 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -393,9 +393,7 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex stopElasticsearch(); // (@rory) this is for debugging - Platforms.onLinux(() -> { - logger.warn(sh.run("ps ax | grep elasticsearch").stdout); - }); + Platforms.onLinux(() -> { logger.warn(sh.run("ps ax | grep elasticsearch").stdout); }); Result result = sh.run("echo y | " + installation.executables().nodeTool + " unsafe-bootstrap"); assertThat(result.stdout, containsString("Master node was successfully bootstrapped")); From 3f7034be2a9e0962529f5e97c90470fb5fa28e6f Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 6 Nov 2020 15:58:16 +0000 Subject: [PATCH 22/32] More debugging --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 3f089d9d77a0e..8064636aeeda1 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -393,7 +393,7 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex stopElasticsearch(); // (@rory) this is for debugging - Platforms.onLinux(() -> { logger.warn(sh.run("ps ax | grep elasticsearch").stdout); }); + Platforms.onLinux(() -> { logger.warn(sh.run("ls -l /tmp/node.lock; ls -ld /tmp").stdout); }); Result result = sh.run("echo y | " + installation.executables().nodeTool + " unsafe-bootstrap"); assertThat(result.stdout, containsString("Master node was successfully bootstrapped")); From b3b7028955dcaf0d70a4183dab0cdbd305b03dd3 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 9 Nov 2020 12:37:51 +0000 Subject: [PATCH 23/32] More debugging --- .../test/java/org/elasticsearch/packaging/util/Archives.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java index f6b3c643d8b5d..7b99e76fd3fd2 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java @@ -405,8 +405,13 @@ public static void stopElasticsearch(Installation installation) throws Exception + "Unregister-EventSubscriber -Force" ); }); + Platforms.onLinux(() -> logger.warn(sh.run("ps ax | grep " + pid).stdout)); if (Files.exists(pidFile)) { Files.delete(pidFile); + if (Files.exists(pidFile)) { + Platforms.onLinux(() -> logger.warn(sh.run("tail -30 " + installation.logs.resolve("elasticsearch.log")).stdout)); + } + assertThat(pidFile, fileDoesNotExist()); } } From b1a3644d1f3f8389555ebed95fc3b7ac6221643b Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 9 Nov 2020 19:37:36 +0000 Subject: [PATCH 24/32] Debugging --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 4 ++++ .../java/org/elasticsearch/packaging/util/Archives.java | 6 +----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 8064636aeeda1..54892573fabd5 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -46,6 +46,7 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.startsWith; import static org.junit.Assume.assumeThat; @@ -395,6 +396,9 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex // (@rory) this is for debugging Platforms.onLinux(() -> { logger.warn(sh.run("ls -l /tmp/node.lock; ls -ld /tmp").stdout); }); + Path pidFile = installation.home.resolve("elasticsearch.pid"); + assertThat(pidFile, fileDoesNotExist()); + Result result = sh.run("echo y | " + installation.executables().nodeTool + " unsafe-bootstrap"); assertThat(result.stdout, containsString("Master node was successfully bootstrapped")); } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java index 7b99e76fd3fd2..fdc42a2f780c0 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java @@ -394,7 +394,7 @@ public static void stopElasticsearch(Installation installation) throws Exception assertThat(pid, is(not(emptyOrNullString()))); final Shell sh = new Shell(); - Platforms.onLinux(() -> sh.run("kill -SIGTERM " + pid + "; tail --pid=" + pid + " -f /dev/null")); + Platforms.onLinux(() -> sh.run("kill -SIGTERM " + pid + " && tail --pid=" + pid + " -f /dev/null")); Platforms.onWindows(() -> { sh.run("Get-Process -Id " + pid + " | Stop-Process -Force; Wait-Process -Id " + pid); @@ -408,10 +408,6 @@ public static void stopElasticsearch(Installation installation) throws Exception Platforms.onLinux(() -> logger.warn(sh.run("ps ax | grep " + pid).stdout)); if (Files.exists(pidFile)) { Files.delete(pidFile); - if (Files.exists(pidFile)) { - Platforms.onLinux(() -> logger.warn(sh.run("tail -30 " + installation.logs.resolve("elasticsearch.log")).stdout)); - } - assertThat(pidFile, fileDoesNotExist()); } } From eda4eed18f6d686458d8f8afe061fa845cf2df3b Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 10 Nov 2020 10:30:12 +0000 Subject: [PATCH 25/32] Debugging --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 54892573fabd5..b1b89af4116a0 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -399,6 +399,8 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex Path pidFile = installation.home.resolve("elasticsearch.pid"); assertThat(pidFile, fileDoesNotExist()); + assertThat(Path.of("/tmp/node.lock"), fileDoesNotExist()); + Result result = sh.run("echo y | " + installation.executables().nodeTool + " unsafe-bootstrap"); assertThat(result.stdout, containsString("Master node was successfully bootstrapped")); } From 7a8b2f3131e2f30707bc936d9cfc963268893c7b Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 10 Nov 2020 10:33:19 +0000 Subject: [PATCH 26/32] Debugging --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index b1b89af4116a0..499515910b25e 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -388,6 +388,8 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex Path relativeDataPath = installation.data.relativize(installation.home); append(installation.config("elasticsearch.yml"), "path.data: " + relativeDataPath); + logger.warn("Settings [path.data] to [" + relativeDataPath + "]"); + sh.setWorkingDirectory(getRootTempDir()); startElasticsearch(); From 2f297e404e182f4302eb1704e4d25013f4a91263 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 10 Nov 2020 10:33:45 +0000 Subject: [PATCH 27/32] Debugging --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 499515910b25e..8b467b7789411 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -401,8 +401,6 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex Path pidFile = installation.home.resolve("elasticsearch.pid"); assertThat(pidFile, fileDoesNotExist()); - assertThat(Path.of("/tmp/node.lock"), fileDoesNotExist()); - Result result = sh.run("echo y | " + installation.executables().nodeTool + " unsafe-bootstrap"); assertThat(result.stdout, containsString("Master node was successfully bootstrapped")); } From b36f021254b2814356fbeffd4feae107004e824d Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 10 Nov 2020 10:45:18 +0000 Subject: [PATCH 28/32] Debugging --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 8b467b7789411..fb2c36f6ba216 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -396,10 +396,7 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex stopElasticsearch(); // (@rory) this is for debugging - Platforms.onLinux(() -> { logger.warn(sh.run("ls -l /tmp/node.lock; ls -ld /tmp").stdout); }); - - Path pidFile = installation.home.resolve("elasticsearch.pid"); - assertThat(pidFile, fileDoesNotExist()); + Platforms.onLinux(() -> { logger.warn(sh.run("ls -l /tmp/node.lock; echo $USER / $UID / $GROUPS").stdout); }); Result result = sh.run("echo y | " + installation.executables().nodeTool + " unsafe-bootstrap"); assertThat(result.stdout, containsString("Master node was successfully bootstrapped")); From fbde6b1238f1c95d36ca36ee58cb50f7211b6092 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 10 Nov 2020 10:53:48 +0000 Subject: [PATCH 29/32] Run elasticsearch-node as the archive user --- .../org/elasticsearch/packaging/test/ArchiveTests.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index fb2c36f6ba216..c640d6fa184ee 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -34,6 +34,7 @@ import static java.nio.file.StandardOpenOption.APPEND; import static java.nio.file.StandardOpenOption.CREATE; +import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER; import static org.elasticsearch.packaging.util.Archives.installArchive; import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation; import static org.elasticsearch.packaging.util.FileExistenceMatchers.fileDoesNotExist; @@ -388,17 +389,12 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex Path relativeDataPath = installation.data.relativize(installation.home); append(installation.config("elasticsearch.yml"), "path.data: " + relativeDataPath); - logger.warn("Settings [path.data] to [" + relativeDataPath + "]"); - sh.setWorkingDirectory(getRootTempDir()); startElasticsearch(); stopElasticsearch(); - // (@rory) this is for debugging - Platforms.onLinux(() -> { logger.warn(sh.run("ls -l /tmp/node.lock; echo $USER / $UID / $GROUPS").stdout); }); - - Result result = sh.run("echo y | " + installation.executables().nodeTool + " unsafe-bootstrap"); + Result result = sh.run("echo y | sudo -E -u " + ARCHIVE_OWNER + " " + installation.executables().nodeTool + " unsafe-bootstrap"); assertThat(result.stdout, containsString("Master node was successfully bootstrapped")); } From ea8948edda5c032b617fd4428a9ea5d390e8cfe7 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 10 Nov 2020 11:29:04 +0000 Subject: [PATCH 30/32] Windows fix --- .../org/elasticsearch/packaging/test/ArchiveTests.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index c640d6fa184ee..392c1a1275829 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.packaging.test; import org.apache.http.client.fluent.Request; +import org.elasticsearch.packaging.util.Distribution; import org.elasticsearch.packaging.util.FileUtils; import org.elasticsearch.packaging.util.Installation; import org.elasticsearch.packaging.util.Platforms; @@ -394,7 +395,12 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex startElasticsearch(); stopElasticsearch(); - Result result = sh.run("echo y | sudo -E -u " + ARCHIVE_OWNER + " " + installation.executables().nodeTool + " unsafe-bootstrap"); + String nodeTool = installation.executables().nodeTool.toString(); + if (Platforms.WINDOWS == false) { + nodeTool = "sudo -E -u " + ARCHIVE_OWNER + " " + nodeTool; + } + + Result result = sh.run("echo y | " + nodeTool + " unsafe-bootstrap"); assertThat(result.stdout, containsString("Master node was successfully bootstrapped")); } From dd5ee8c441080f257d0ed81556bc67e1f6b497b1 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 11 Nov 2020 16:18:45 +0000 Subject: [PATCH 31/32] Revert debugging changes --- .../java/org/elasticsearch/packaging/test/ArchiveTests.java | 2 -- .../test/java/org/elasticsearch/packaging/util/Archives.java | 1 - 2 files changed, 3 deletions(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 392c1a1275829..d7dc6294046b1 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.packaging.test; import org.apache.http.client.fluent.Request; -import org.elasticsearch.packaging.util.Distribution; import org.elasticsearch.packaging.util.FileUtils; import org.elasticsearch.packaging.util.Installation; import org.elasticsearch.packaging.util.Platforms; @@ -48,7 +47,6 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.emptyString; import static org.hamcrest.Matchers.startsWith; import static org.junit.Assume.assumeThat; diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java index fdc42a2f780c0..e5072344dcd98 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java @@ -405,7 +405,6 @@ public static void stopElasticsearch(Installation installation) throws Exception + "Unregister-EventSubscriber -Force" ); }); - Platforms.onLinux(() -> logger.warn(sh.run("ps ax | grep " + pid).stdout)); if (Files.exists(pidFile)) { Files.delete(pidFile); } From cfe7b4cba1de2230227e8ed34191f65d6bb467d5 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 11 Nov 2020 16:36:08 +0000 Subject: [PATCH 32/32] Fixes --- distribution/docker/build.gradle | 5 ++++- distribution/docker/src/docker/iron_bank/README.md | 4 ++-- .../src/docker/iron_bank/{downloads.json => download.json} | 0 3 files changed, 6 insertions(+), 3 deletions(-) rename distribution/docker/src/docker/iron_bank/{downloads.json => download.json} (100%) diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 3c135677c74bf..9b8d41d52de9d 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -73,6 +73,8 @@ RUN curl --retry 8 -S -L \\ """.trim() } + def (major,minor) = VersionProperties.elasticsearch.split("\\.") + return [ 'base_image' : base.getImage(), 'bin_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'bin', @@ -84,7 +86,8 @@ RUN curl --retry 8 -S -L \\ 'package_manager' : base == DockerBase.UBI ? 'microdnf' : 'yum', 'source_elasticsearch': sourceElasticsearch, 'docker_base' : base.name().toLowerCase(), - 'version' : VersionProperties.elasticsearch + 'version' : VersionProperties.elasticsearch, + 'major_minor_version' : "${major}.${minor}" ] } diff --git a/distribution/docker/src/docker/iron_bank/README.md b/distribution/docker/src/docker/iron_bank/README.md index 14854a53615f4..6767a6faa66f8 100644 --- a/distribution/docker/src/docker/iron_bank/README.md +++ b/distribution/docker/src/docker/iron_bank/README.md @@ -29,9 +29,9 @@ visiting [Elastic Community](https://www.elastic.co/community). This software is governed by the [Elastic -License](https://github.com/elastic/elasticsearch/blob/7.8/licenses/ELASTIC-LICENSE.txt), +License](https://github.com/elastic/elasticsearch/blob/${major_minor_version}/licenses/ELASTIC-LICENSE.txt), and includes the full set of [free features](https://www.elastic.co/subscriptions). View the detailed release notes -[here](https://www.elastic.co/guide/en/elasticsearch/reference/7.8/es-release-notes.html). +[here](https://www.elastic.co/guide/en/elasticsearch/reference/${major_minor_version}/es-release-notes.html). diff --git a/distribution/docker/src/docker/iron_bank/downloads.json b/distribution/docker/src/docker/iron_bank/download.json similarity index 100% rename from distribution/docker/src/docker/iron_bank/downloads.json rename to distribution/docker/src/docker/iron_bank/download.json