Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HBASE-28693 Change flaky tests to run with jdk17 #6031

Merged
merged 1 commit into from
Jun 27, 2024
Merged

HBASE-28693 Change flaky tests to run with jdk17 #6031

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 15 additions & 17 deletions dev-support/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,46 +75,46 @@ ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8
#

FROM base_image AS spotbugs_download_image
ENV SPOTBUGS_VERSION '4.7.3'
ENV SPOTBUGS_URL "https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/${SPOTBUGS_VERSION}/spotbugs-${SPOTBUGS_VERSION}.tgz"
ENV SPOTBUGS_SHA512 '09a9fe0e5a6ec8e9d6d116c361b5c34c9d0560c0271241f02fadee911952adfcd69dc184f6de1cc4d4a8fe2c84c162689ea9a691dcae0779935eedf390fcc4ad'
ENV SPOTBUGS_VERSION='4.7.3'
ENV SPOTBUGS_URL="https://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/${SPOTBUGS_VERSION}/spotbugs-${SPOTBUGS_VERSION}.tgz"
ENV SPOTBUGS_SHA512='09a9fe0e5a6ec8e9d6d116c361b5c34c9d0560c0271241f02fadee911952adfcd69dc184f6de1cc4d4a8fe2c84c162689ea9a691dcae0779935eedf390fcc4ad'
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output /tmp/spotbugs.tgz "${SPOTBUGS_URL}" && \
echo "${SPOTBUGS_SHA512} */tmp/spotbugs.tgz" | sha512sum -c -

FROM base_image AS hadolint_download_image
ENV HADOLINT_VERSION '2.10.0'
ENV HADOLINT_URL "https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VERSION}/hadolint-Linux-x86_64"
ENV HADOLINT_SHA512 '4816c95243bedf15476d2225f487fc17465495fb2031e1a4797d82a26db83a1edb63e4fed084b80cef17d5eb67eb45508caadaf7cd0252fb061187113991a338'
ENV HADOLINT_VERSION='2.10.0'
ENV HADOLINT_URL="https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VERSION}/hadolint-Linux-x86_64"
ENV HADOLINT_SHA512='4816c95243bedf15476d2225f487fc17465495fb2031e1a4797d82a26db83a1edb63e4fed084b80cef17d5eb67eb45508caadaf7cd0252fb061187113991a338'
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output /tmp/hadolint "${HADOLINT_URL}" && \
echo "${HADOLINT_SHA512} */tmp/hadolint" | sha512sum -c -

FROM base_image AS maven_download_image
ENV MAVEN_VERSION='3.8.6'
ENV MAVEN_URL "https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
ENV MAVEN_SHA512 'f790857f3b1f90ae8d16281f902c689e4f136ebe584aba45e4b1fa66c80cba826d3e0e52fdd04ed44b4c66f6d3fe3584a057c26dfcac544a60b301e6d0f91c26'
ENV MAVEN_URL="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
ENV MAVEN_SHA512='f790857f3b1f90ae8d16281f902c689e4f136ebe584aba45e4b1fa66c80cba826d3e0e52fdd04ed44b4c66f6d3fe3584a057c26dfcac544a60b301e6d0f91c26'
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output /tmp/maven.tar.gz "${MAVEN_URL}" && \
echo "${MAVEN_SHA512} */tmp/maven.tar.gz" | sha512sum -c -

FROM base_image AS openjdk8_download_image
ENV OPENJDK8_URL 'https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u352-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u352b08.tar.gz'
ENV OPENJDK8_SHA256 '1633bd7590cb1cd72f5a1378ae8294451028b274d798e2a4ac672059a2f00fee'
ENV OPENJDK8_URL='https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u352-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u352b08.tar.gz'
ENV OPENJDK8_SHA256='1633bd7590cb1cd72f5a1378ae8294451028b274d798e2a4ac672059a2f00fee'
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk8.tar.gz "${OPENJDK8_URL}" && \
echo "${OPENJDK8_SHA256} */tmp/adoptopenjdk8.tar.gz" | sha256sum -c -

FROM base_image AS openjdk11_download_image
ENV OPENJDK11_URL 'https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.17%2B8/OpenJDK11U-jdk_x64_linux_hotspot_11.0.17_8.tar.gz'
ENV OPENJDK11_SHA256 'b8d46ed08ef4859476fe6421a7690d899ed83dce63f13fd894f994043177ef3c'
ENV OPENJDK11_URL='https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.17%2B8/OpenJDK11U-jdk_x64_linux_hotspot_11.0.17_8.tar.gz'
ENV OPENJDK11_SHA256='b8d46ed08ef4859476fe6421a7690d899ed83dce63f13fd894f994043177ef3c'
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk11.tar.gz "${OPENJDK11_URL}" && \
echo "${OPENJDK11_SHA256} */tmp/adoptopenjdk11.tar.gz" | sha256sum -c -

FROM base_image AS openjdk17_download_image
ENV OPENJDK17_URL 'https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.10%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.10_7.tar.gz'
ENV OPENJDK17_SHA256 'a8fd07e1e97352e97e330beb20f1c6b351ba064ca7878e974c7d68b8a5c1b378'
ENV OPENJDK17_URL='https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.10%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.10_7.tar.gz'
ENV OPENJDK17_SHA256='a8fd07e1e97352e97e330beb20f1c6b351ba064ca7878e974c7d68b8a5c1b378'
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN curl --location --fail --silent --show-error --output /tmp/adoptopenjdk17.tar.gz "${OPENJDK17_URL}" && \
echo "${OPENJDK17_SHA256} */tmp/adoptopenjdk17.tar.gz" | sha256sum -c -
Expand Down Expand Up @@ -179,9 +179,7 @@ RUN mkdir -p /usr/lib/jvm && \
# these values to be specified here; the various --foo-path flags do not
# propigate as expected, while these are honored.
# TODO (nd): is this really true? investigate and file a ticket.
ENV SPOTBUGS_HOME '/opt/spotbugs'
ENV MAVEN_HOME '/opt/maven'
ENV MAVEN_OPTS '-Xmx3.6G'
ENV SPOTBUGS_HOME='/opt/spotbugs' MAVEN_HOME='/opt/maven'

CMD ["/bin/bash"]

Expand Down
19 changes: 10 additions & 9 deletions dev-support/flaky-tests/run-flaky-tests.Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@
// under the License.
pipeline {
agent {
node {
dockerfile {
dir 'dev-support/docker'
label 'hbase'
args '-v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro'
}
}
triggers {
Expand All @@ -31,20 +33,19 @@ pipeline {
}
environment {
ASF_NIGHTLIES = 'https://nightlies.apache.org'
JAVA_HOME = '/usr/lib/jvm/java-17'
}
parameters {
booleanParam(name: 'DEBUG', defaultValue: false, description: 'Produce a lot more meta-information.')
}
tools {
// this should match what the yetus nightly job for the branch will use
maven 'maven_latest'
jdk "jdk_1.8_latest"
}
stages {
stage ('run flaky tests') {
steps {
sh '''#!/usr/bin/env bash
set -e
MVN="${MAVEN_HOME}/bin/mvn"
# print the maven version and java version
${MVN} --version
declare -a curl_args=(--fail)
tmpdir=$(realpath target)
declare -a mvn_args=(--batch-mode -fn -Dbuild.id="${BUILD_ID}" -Dmaven.repo.local="${WORKSPACE}/local-repository" -Djava.io.tmpdir=${tmpdir})
Expand All @@ -56,7 +57,7 @@ pipeline {
curl "${curl_args[@]}" -o includes.txt "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/output/includes"
if [ -s includes.txt ]; then
rm -rf local-repository/org/apache/hbase
mvn clean "${mvn_args[@]}"
${MVN} clean "${mvn_args[@]}"
rm -rf "target/machine" && mkdir -p "target/machine"
if [ -x dev-support/gather_machine_environment.sh ]; then
"./dev-support/gather_machine_environment.sh" "target/machine"
Expand All @@ -65,11 +66,11 @@ pipeline {
else
echo "Skipped gathering machine environment because we couldn't read the script to do so."
fi
mvn -T0.25C package "${mvn_args[@]}" -Dtest="$(cat includes.txt)" -Dmaven.test.redirectTestOutputToFile=true -Dsurefire.firstPartForkCount=0.25C -Dsurefire.secondPartForkCount=0.25C
${MVN} -T0.25C package "${mvn_args[@]}" -Dtest="$(cat includes.txt)" -Dmaven.test.redirectTestOutputToFile=true -Dsurefire.firstPartForkCount=0.25C -Dsurefire.secondPartForkCount=0.25C
else
echo "set of flaky tests is currently empty."
fi
'''
'''
}
}
}
Expand Down