Skip to content

HADOOP-19398: Upgrade Dockerfile base to jammy #7532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

HADOOP-19398: Upgrade Dockerfile base to jammy #7532

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a2b51ab
HADOOP-19398: Upgrade Dockerfile base to jammy
Jan 23, 2025
7611ab4
HADOOP-19398: Remove python 2.7 from arch64 image
Jan 23, 2025
6561167
fixed Dockerfile LegacyKeyValueFormat warnings
Mar 23, 2025
5446fbd
Merge branch 'trunk' into HADOOP-19398-upgrade-Dockerfile-base-to-jam…
Mar 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions dev-support/Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,10 @@ pipeline {
'''
}

dir("${WORKSPACE}/ubuntu-focal") {
dir("${WORKSPACE}/ubuntu-jammy") {
sh '''#!/usr/bin/env bash

cp -Rp ${WORKSPACE}/src ${WORKSPACE}/ubuntu-focal
cp -Rp ${WORKSPACE}/src ${WORKSPACE}/ubuntu-jammy
'''
}
}
Expand Down Expand Up @@ -251,12 +251,12 @@ pipeline {
}
}

// We want to use Ubuntu Focal as our main CI and thus, this stage
// We want to use Ubuntu Jammy as our main CI and thus, this stage
// isn't optional (runs for all the PRs).
stage ('precommit-run Ubuntu focal') {
stage ('precommit-run Ubuntu jammy') {
environment {
SOURCEDIR = "${WORKSPACE}/ubuntu-focal/src"
PATCHDIR = "${WORKSPACE}/ubuntu-focal/out"
SOURCEDIR = "${WORKSPACE}/ubuntu-jammy/src"
PATCHDIR = "${WORKSPACE}/ubuntu-jammy/out"
DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
IS_OPTIONAL = 0
}
Expand All @@ -281,8 +281,8 @@ pipeline {
usernameVariable: 'GITHUB_USER')]) {
sh '''#!/usr/bin/env bash

# Copy the artifacts of Ubuntu focal build to workspace
cp -Rp "${WORKSPACE}/ubuntu-focal/out" "${WORKSPACE}"
# Copy the artifacts of Ubuntu jammy build to workspace
cp -Rp "${WORKSPACE}/ubuntu-jammy/out" "${WORKSPACE}"

# Send Github status
chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
Expand Down
6 changes: 3 additions & 3 deletions dev-support/bin/create-release
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -508,16 +508,16 @@ function dockermode
echo "RUN groupadd --non-unique -g ${group_id} ${user_name}; exit 0;"
echo "RUN useradd -g ${group_id} -u ${user_id} -m ${user_name}; exit 0;"
echo "RUN chown -R ${user_name} /home/${user_name}; exit 0;"
echo "ENV HOME /home/${user_name}"
echo "ENV HOME=/home/${user_name}"
echo "RUN mkdir -p /maven"
echo "RUN chown -R ${user_name} /maven"

# we always force build with the OpenJDK JDK
# but with the correct version
if [[ "$CPU_ARCH" = "aarch64" || "$CPU_ARCH" = "arm64" ]]; then
echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-arm64"
echo "ENV JAVA_HOME=/usr/lib/jvm/java-${JVM_VERSION}-openjdk-arm64"
else
echo "ENV JAVA_HOME /usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
echo "ENV JAVA_HOME=/usr/lib/jvm/java-${JVM_VERSION}-openjdk-amd64"
fi

echo "USER ${user_name}"
Expand Down
32 changes: 16 additions & 16 deletions dev-support/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# Dockerfile for installing the necessary dependencies for building Hadoop.
# See BUILDING.txt.

FROM ubuntu:focal
FROM ubuntu:jammy

WORKDIR /root

Expand All @@ -29,8 +29,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras

ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_TERSE=true

######
# Platform package dependency resolver
Expand All @@ -49,7 +49,7 @@ RUN apt-get -q update \
&& echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \
&& wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc \
&& apt-get -q update \
&& apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \
&& apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:jammy) \
&& apt-get clean \
&& update-java-alternatives -s java-1.8.0-openjdk-amd64 \
&& rm -rf /var/lib/apt/lists/*
Expand All @@ -61,38 +61,38 @@ ENV PYTHONIOENCODING=utf-8
######
# Set env vars required to build Hadoop
######
ENV MAVEN_HOME /usr
ENV MAVEN_HOME=/usr
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64

#######
# Set env vars for SpotBugs 4.2.2
#######
ENV SPOTBUGS_HOME /opt/spotbugs
ENV SPOTBUGS_HOME=/opt/spotbugs

#######
# Set env vars for Google Protobuf 3.21.12
#######
ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin"
ENV PROTOBUF_HOME=/opt/protobuf
ENV PATH="${PATH}:/opt/protobuf/bin"

###
# Avoid out of memory errors in builds
###
ENV MAVEN_OPTS -Xms256m -Xmx3072m
ENV MAVEN_OPTS="-Xms256m -Xmx3072m"

# Skip gpg verification when downloading Yetus via yetus-wrapper
ENV HADOOP_SKIP_YETUS_VERIFICATION true
ENV HADOOP_SKIP_YETUS_VERIFICATION=true

####
# Install packages
####
RUN pkg-resolver/install-common-pkgs.sh
RUN pkg-resolver/install-spotbugs.sh ubuntu:focal
RUN pkg-resolver/install-boost.sh ubuntu:focal
RUN pkg-resolver/install-protobuf.sh ubuntu:focal
RUN pkg-resolver/install-hadolint.sh ubuntu:focal
RUN pkg-resolver/install-intel-isa-l.sh ubuntu:focal
RUN pkg-resolver/install-spotbugs.sh ubuntu:jammy
RUN pkg-resolver/install-boost.sh ubuntu:jammy
RUN pkg-resolver/install-protobuf.sh ubuntu:jammy
RUN pkg-resolver/install-hadolint.sh ubuntu:jammy
RUN pkg-resolver/install-intel-isa-l.sh ubuntu:jammy

###
# Everything past this point is either not needed for testing or breaks Yetus.
Expand Down
30 changes: 15 additions & 15 deletions dev-support/docker/Dockerfile_aarch64
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
# Dockerfile for installing the necessary dependencies for building Hadoop.
# See BUILDING.txt.

FROM ubuntu:focal
FROM ubuntu:jammy

WORKDIR /root

Expand All @@ -29,8 +29,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras

ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_TERSE=true

######
# Platform package dependency resolver
Expand All @@ -49,7 +49,7 @@ RUN apt-get -q update \
&& echo "deb https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" > /etc/apt/sources.list.d/adoptium.list \
&& wget -q -O - https://packages.adoptium.net/artifactory/api/gpg/key/public > /etc/apt/trusted.gpg.d/adoptium.asc \
&& apt-get -q update \
&& apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \
&& apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:jammy::arch64) \
&& apt-get clean \
&& update-java-alternatives -s java-1.8.0-openjdk-arm64 \
&& rm -rf /var/lib/apt/lists/*
Expand All @@ -61,39 +61,39 @@ ENV PYTHONIOENCODING=utf-8
######
# Set env vars required to build Hadoop
######
ENV MAVEN_HOME /usr
ENV MAVEN_HOME=/usr
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-arm64

#######
# Set env vars for SpotBugs 4.2.2
#######
ENV SPOTBUGS_HOME /opt/spotbugs
ENV SPOTBUGS_HOME=/opt/spotbugs

#######
# Set env vars for Google Protobuf 3.21.12
#######
ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin"
ENV PROTOBUF_HOME=/opt/protobuf
ENV PATH="${PATH}:/opt/protobuf/bin"

###
# Avoid out of memory errors in builds
###
ENV MAVEN_OPTS -Xms256m -Xmx3072m
ENV MAVEN_OPTS="-Xms256m -Xmx3072m"

# Skip gpg verification when downloading Yetus via yetus-wrapper
ENV HADOOP_SKIP_YETUS_VERIFICATION true
ENV HADOOP_SKIP_YETUS_VERIFICATION=true

# Force PhantomJS to be in 'headless' mode, do not connect to Xwindow
ENV QT_QPA_PLATFORM offscreen
ENV QT_QPA_PLATFORM=offscreen

####
# Install packages
####
RUN pkg-resolver/install-common-pkgs.sh
RUN pkg-resolver/install-spotbugs.sh ubuntu:focal::arch64
RUN pkg-resolver/install-boost.sh ubuntu:focal::arch64
RUN pkg-resolver/install-protobuf.sh ubuntu:focal::arch64
RUN pkg-resolver/install-spotbugs.sh ubuntu:jammy::arch64
RUN pkg-resolver/install-boost.sh ubuntu:jammy::arch64
RUN pkg-resolver/install-protobuf.sh ubuntu:jammy::arch64

###
# Everything past this point is either not needed for testing or breaks Yetus.
Expand Down
32 changes: 16 additions & 16 deletions dev-support/docker/Dockerfile_centos_7
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,37 +61,37 @@ SHELL ["/bin/bash", "--login", "-c"]
# Set the environment variables needed for CMake
# to find and use GCC 9 for compilation
######
ENV GCC_HOME "/opt/rh/devtoolset-9"
ENV CC "${GCC_HOME}/root/usr/bin/gcc"
ENV CXX "${GCC_HOME}/root/usr/bin/g++"
ENV SHLVL 1
ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
ENV PCP_DIR "${GCC_HOME}/root"
ENV MANPATH "${GCC_HOME}/root/usr/share/man:"
ENV PATH "${GCC_HOME}/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
ENV GCC_HOME="/opt/rh/devtoolset-9"
ENV CC="${GCC_HOME}/root/usr/bin/gcc"
ENV CXX="${GCC_HOME}/root/usr/bin/g++"
ENV SHLVL=1
ENV LD_LIBRARY_PATH="${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
ENV PCP_DIR="${GCC_HOME}/root"
ENV MANPATH="${GCC_HOME}/root/usr/share/man:"
ENV PATH="${GCC_HOME}/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
ENV PKG_CONFIG_PATH="${GCC_HOME}/root/usr/lib64/pkgconfig"
ENV INFOPATH="${GCC_HOME}/root/usr/share/info"

# TODO: Set locale

######
# Set env vars required to build Hadoop
######
ENV MAVEN_HOME /opt/maven
ENV PATH "${PATH}:${MAVEN_HOME}/bin"
ENV MAVEN_HOME=/opt/maven
ENV PATH="${PATH}:${MAVEN_HOME}/bin"
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0

#######
# Set env vars for SpotBugs
#######
ENV SPOTBUGS_HOME /opt/spotbugs
ENV SPOTBUGS_HOME=/opt/spotbugs

#######
# Set env vars for Google Protobuf 3.21.12
#######
ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin"
ENV PROTOBUF_HOME=/opt/protobuf
ENV PATH="${PATH}:/opt/protobuf/bin"

######
# Install packages
Expand Down
42 changes: 21 additions & 21 deletions dev-support/docker/Dockerfile_centos_8
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,42 +69,42 @@ SHELL ["/bin/bash", "--login", "-c"]
# Set the environment variables needed for CMake
# to find and use GCC 9 for compilation
######
ENV GCC_HOME "/opt/rh/gcc-toolset-9"
ENV CC "${GCC_HOME}/root/usr/bin/gcc"
ENV CXX "${GCC_HOME}/root/usr/bin/g++"
ENV MODULES_RUN_QUARANTINE "LD_LIBRARY_PATH LD_PRELOAD"
ENV MODULES_CMD "/usr/share/Modules/libexec/modulecmd.tcl"
ENV SHLVL 1
ENV MODULEPATH "/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
ENV MODULEPATH_modshare "/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
ENV MODULESHOME "/usr/share/Modules"
ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
ENV PCP_DIR "${GCC_HOME}/root"
ENV MANPATH "${GCC_HOME}/root/usr/share/man::"
ENV PATH "${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
ENV GCC_HOME="/opt/rh/gcc-toolset-9"
ENV CC="${GCC_HOME}/root/usr/bin/gcc"
ENV CXX="${GCC_HOME}/root/usr/bin/g++"
ENV MODULES_RUN_QUARANTINE="LD_LIBRARY_PATH LD_PRELOAD"
ENV MODULES_CMD="/usr/share/Modules/libexec/modulecmd.tcl"
ENV SHLVL=1
ENV MODULEPATH="/etc/scl/modulefiles:/usr/share/Modules/modulefiles:/etc/modulefiles:/usr/share/modulefiles"
ENV MODULEPATH_modshare="/usr/share/modulefiles:1:/usr/share/Modules/modulefiles:1:/etc/modulefiles:1"
ENV MODULESHOME="/usr/share/Modules"
ENV LD_LIBRARY_PATH="${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
ENV PCP_DIR="${GCC_HOME}/root"
ENV MANPATH="${GCC_HOME}/root/usr/share/man::"
ENV PATH="${GCC_HOME}/root/usr/bin:/usr/share/Modules/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
ENV PKG_CONFIG_PATH="${GCC_HOME}/root/usr/lib64/pkgconfig"
ENV INFOPATH="${GCC_HOME}/root/usr/share/info"

# TODO: Set locale

######
# Set env vars required to build Hadoop
######
ENV MAVEN_HOME /opt/maven
ENV PATH "${PATH}:${MAVEN_HOME}/bin"
ENV MAVEN_HOME=/opt/maven
ENV PATH="${PATH}:${MAVEN_HOME}/bin"
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0

#######
# Set env vars for SpotBugs
#######
ENV SPOTBUGS_HOME /opt/spotbugs
ENV SPOTBUGS_HOME=/opt/spotbugs

#######
# Set env vars for Google Protobuf 3.21.12
#######
ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin"
ENV PROTOBUF_HOME=/opt/protobuf
ENV PATH="${PATH}:/opt/protobuf/bin"

######
# Install packages
Expand Down
18 changes: 9 additions & 9 deletions dev-support/docker/Dockerfile_debian_10
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras

ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_TERSE=true

######
# Platform package dependency resolver
Expand All @@ -56,28 +56,28 @@ RUN apt-get -q update \
######
# Set env vars required to build Hadoop
######
ENV MAVEN_HOME /usr
ENV MAVEN_HOME=/usr
# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64

#######
# Set env vars for SpotBugs 4.2.2
#######
ENV SPOTBUGS_HOME /opt/spotbugs
ENV SPOTBUGS_HOME=/opt/spotbugs

#######
# Set env vars for Google Protobuf 3.21.12
#######
ENV PROTOBUF_HOME /opt/protobuf
ENV PATH "${PATH}:/opt/protobuf/bin"
ENV PROTOBUF_HOME=/opt/protobuf
ENV PATH="${PATH}:/opt/protobuf/bin"

###
# Avoid out of memory errors in builds
###
ENV MAVEN_OPTS -Xms256m -Xmx3072m
ENV MAVEN_OPTS="-Xms256m -Xmx3072m"

# Skip gpg verification when downloading Yetus via yetus-wrapper
ENV HADOOP_SKIP_YETUS_VERIFICATION true
ENV HADOOP_SKIP_YETUS_VERIFICATION=true

####
# Install packages
Expand Down
Loading