HBASE-28676 Make pre commit build for 3.x to only run jdk 17 test (#6001

) Signed-off-by: Nick Dimiduk <ndimiduk@apache.org>
apache · Jun 26, 2024 · 80516d7 · 80516d7
1 parent 7e8fd07
commit 80516d7
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 277 deletions.
diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub
@@ -56,8 +56,6 @@ pipeline {
  // stage works in its own subdirectory. there is an "output" under each of these
  // directories, which we retrieve after the build is complete.
  WORKDIR_REL_GENERAL_CHECK = 'yetus-general-check'
- WORKDIR_REL_JDK8_HADOOP3_CHECK = 'yetus-jdk8-hadoop3-check'
- WORKDIR_REL_JDK11_HADOOP3_CHECK = 'yetus-jdk11-hadoop3-check'
  WORKDIR_REL_JDK17_HADOOP3_CHECK = 'yetus-jdk17-hadoop3-check'
  ASF_NIGHTLIES = 'https://nightlies.apache.org'
  ASF_NIGHTLIES_BASE_ORI = "${ASF_NIGHTLIES}/hbase/${JOB_NAME}/${BUILD_NUMBER}"
@@ -82,8 +80,7 @@ pipeline {
  environment {
  // customized per parallel stage
  PLUGINS = "${GENERAL_CHECK_PLUGINS}"
- SET_JAVA_HOME = "/usr/lib/jvm/java-11"
- JAVA8_HOME = "/usr/lib/jvm/java-8"
+ SET_JAVA_HOME = "/usr/lib/jvm/java-17"
  HADOOP_PROFILE = '3.0'
  WORKDIR_REL = "${WORKDIR_REL_GENERAL_CHECK}"
  // identical for all parallel stages
@@ -202,278 +199,6 @@ pipeline {
  }
  }
  }
- stage ('yetus jdk8 Hadoop3 checks') {
- agent {
- node {
- label 'hbase'
- }
- }
- environment {
- // customized per parallel stage
- PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
- SET_JAVA_HOME = '/usr/lib/jvm/java-8'
- WORKDIR_REL = "${WORKDIR_REL_JDK8_HADOOP3_CHECK}"
- // identical for all parallel stages
- WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
- YETUSDIR = "${WORKDIR}/${YETUS_REL}"
- SOURCEDIR = "${WORKDIR}/${SRC_REL}"
- PATCHDIR = "${WORKDIR}/${PATCH_REL}"
- BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
- DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
- YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
- SKIP_ERRORPRONE = true
- }
- when {
- // this will return true if the pipeline is building a change request, such as a GitHub pull request.
- changeRequest()
- }
- steps {
- dir("${SOURCEDIR}") {
- checkout scm
- }
- dir("${YETUSDIR}") {
- sh'''#!/usr/bin/env bash
- wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
- tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
- rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
- '''
- }
- dir("${WORKDIR}") {
- withCredentials([
- usernamePassword(
- credentialsId: 'apache-hbase-at-github.com',
- passwordVariable: 'GITHUB_PASSWORD',
- usernameVariable: 'GITHUB_USER'
- )]) {
- script {
- def ret = sh(
- label: 'test-patch',
- returnStatus: true,
- script: '''#!/bin/bash -e
- hostname -a ; pwd ; ls -la
- printenv 2>&1 | sort
- echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
- "${YETUS_DRIVER}"
- '''
- )
- if (ret != 0) {
- // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
- // test output. See HBASE-26339 for more details.
- currentBuild.result = 'UNSTABLE'
- }
- }
- }
- }
- }
- post {
- always {
- junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml",
- allowEmptyResults: true, skipPublishingChecks: true
- sh label: 'zip surefire reports', script: '''#!/bin/bash -e
- if [ -d "${PATCHDIR}/archiver" ]; then
- count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
- if [[ 0 -ne ${count} ]]; then
- echo "zipping ${count} archived files"
- zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
- else
- echo "No archived files, skipping compressing."
- fi
- else
- echo "No archiver directory, skipping compressing."
- fi
- '''
- sshPublisher(publishers: [
- sshPublisherDesc(configName: 'Nightlies',
- transfers: [
- sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
- sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
- )
- ]
- )
- ])
- // remove the big test logs zip file, store the nightlies url in test_logs.txt
- sh '''#!/bin/bash -e
- if [ -f "${PATCHDIR}/test_logs.zip" ]; then
- echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
- rm -rf "${PATCHDIR}/test_logs.zip"
- python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
- else
- echo "No test_logs.zip, skipping"
- fi
- '''
- // Has to be relative to WORKSPACE.
- archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
- archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
- publishHTML target: [
- allowMissing: true,
- keepAll: true,
- alwaysLinkToLastBuild: true,
- // Has to be relative to WORKSPACE
- reportDir: "${WORKDIR_REL}/${PATCH_REL}",
- reportFiles: 'report.html',
- reportName: 'PR JDK8 Hadoop3 Check Report'
- ]
- }
- // Jenkins pipeline jobs fill slaves on PRs without this :(
- cleanup() {
- script {
- sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
- # See YETUS-764
- if [ -f "${PATCHDIR}/pidfile.txt" ]; then
- echo "test-patch process appears to still be running: killing"
- kill `cat "${PATCHDIR}/pidfile.txt"` || true
- sleep 10
- fi
- if [ -f "${PATCHDIR}/cidfile.txt" ]; then
- echo "test-patch container appears to still be running: killing"
- docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
- fi
- # See HADOOP-13951
- chmod -R u+rxw "${WORKSPACE}"
- '''
- dir ("${WORKDIR}") {
- deleteDir()
- }
- }
- }
- }
- }
- stage ('yetus jdk11 hadoop3 checks') {
- agent {
- node {
- label 'hbase'
- }
- }
- environment {
- // customized per parallel stage
- PLUGINS = "${JDK_SPECIFIC_PLUGINS}"
- SET_JAVA_HOME = '/usr/lib/jvm/java-11'
- WORKDIR_REL = "${WORKDIR_REL_JDK11_HADOOP3_CHECK}"
- // identical for all parallel stages
- WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
- YETUSDIR = "${WORKDIR}/${YETUS_REL}"
- SOURCEDIR = "${WORKDIR}/${SRC_REL}"
- PATCHDIR = "${WORKDIR}/${PATCH_REL}"
- BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
- DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
- YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
- SKIP_ERRORPRONE = true
- }
- when {
- // this will return true if the pipeline is building a change request, such as a GitHub pull request.
- changeRequest()
- }
- steps {
- dir("${SOURCEDIR}") {
- checkout scm
- }
- dir("${YETUSDIR}") {
- sh'''#!/usr/bin/env bash
- wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
- tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
- rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
- '''
- }
- dir("${WORKDIR}") {
- withCredentials([
- usernamePassword(
- credentialsId: 'apache-hbase-at-github.com',
- passwordVariable: 'GITHUB_PASSWORD',
- usernameVariable: 'GITHUB_USER'
- )]) {
- script {
- def ret = sh(
- label: 'test-patch',
- returnStatus: true,
- script: '''#!/bin/bash -e
- hostname -a ; pwd ; ls -la
- printenv 2>&1 | sort
- echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
- "${YETUS_DRIVER}"
- '''
- )
- if (ret != 0) {
- // mark the build as UNSTABLE instead of FAILURE, to avoid skipping the later publish of
- // test output. See HBASE-26339 for more details.
- currentBuild.result = 'UNSTABLE'
- }
- }
- }
- }
- }
- post {
- always {
- junit testResults: "${WORKDIR_REL}/${SRC_REL}/**/target/**/TEST-*.xml",
- allowEmptyResults: true, skipPublishingChecks: true
- sh label: 'zip surefire reports', script: '''#!/bin/bash -e
- if [ -d "${PATCHDIR}/archiver" ]; then
- count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
- if [[ 0 -ne ${count} ]]; then
- echo "zipping ${count} archived files"
- zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
- else
- echo "No archived files, skipping compressing."
- fi
- else
- echo "No archiver directory, skipping compressing."
- fi
- '''
- sshPublisher(publishers: [
- sshPublisherDesc(configName: 'Nightlies',
- transfers: [
- sshTransfer(remoteDirectory: "hbase/${JOB_NAME}/${BUILD_NUMBER}",
- sourceFiles: "${env.WORKDIR_REL}/${env.PATCH_REL}/test_logs.zip"
- )
- ]
- )
- ])
- // remove the big test logs zip file, store the nightlies url in test_logs.txt
- sh '''#!/bin/bash -e
- if [ -f "${PATCHDIR}/test_logs.zip" ]; then
- echo "Remove ${PATCHDIR}/test_logs.zip for saving space"
- rm -rf "${PATCHDIR}/test_logs.zip"
- python3 ${SOURCEDIR}/dev-support/gen_redirect_html.py "${ASF_NIGHTLIES_BASE}/${WORKDIR_REL}/${PATCH_REL}" > "${PATCHDIR}/test_logs.html"
- else
- echo "No test_logs.zip, skipping"
- fi
- '''
- // Has to be relative to WORKSPACE.
- archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
- archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
- publishHTML target: [
- allowMissing: true,
- keepAll: true,
- alwaysLinkToLastBuild: true,
- // Has to be relative to WORKSPACE
- reportDir: "${WORKDIR_REL}/${PATCH_REL}",
- reportFiles: 'report.html',
- reportName: 'PR JDK11 Hadoop3 Check Report'
- ]
- }
- // Jenkins pipeline jobs fill slaves on PRs without this :(
- cleanup() {
- script {
- sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
- # See YETUS-764
- if [ -f "${PATCHDIR}/pidfile.txt" ]; then
- echo "test-patch process appears to still be running: killing"
- kill `cat "${PATCHDIR}/pidfile.txt"` || true
- sleep 10
- fi
- if [ -f "${PATCHDIR}/cidfile.txt" ]; then
- echo "test-patch container appears to still be running: killing"
- docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
- fi
- # See HADOOP-13951
- chmod -R u+rxw "${WORKSPACE}"
- '''
- dir ("${WORKDIR}") {
- deleteDir()
- }
- }
- }
- }
- }
  stage ('yetus jdk17 hadoop3 checks') {
  agent {
  node {

diff --git a/hbase-build-configuration/pom.xml b/hbase-build-configuration/pom.xml
@@ -77,7 +77,18 @@
  <configuration>
  <release>${releaseTarget}</release>
  <showWarnings>true</showWarnings>
+ <fork>true</fork>
  <compilerArgs>
+ <arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED</arg>
+ <arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED</arg>
+ <arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED</arg>
+ <arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED</arg>
+ <arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED</arg>
+ <arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED</arg>
+ <arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED</arg>
+ <arg>-J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED</arg>
+ <arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED</arg>
+ <arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED</arg>
  <arg>-XDcompilePolicy=simple</arg>
  <!-- All -Xep need to be on single line see: https://github.com/google/error-prone/pull/1115 -->
  <arg>-Xplugin:ErrorProne -XepDisableWarningsInGeneratedCode -XepExcludedPaths:.*/target/.* -Xep:FallThrough:OFF -Xep:MutablePublicArray:OFF -Xep:ClassNewInstance:ERROR -Xep:MissingDefault:ERROR -Xep:BanJNDI:WARN</arg>

diff --git a/pom.xml b/pom.xml
@@ -809,7 +809,7 @@
  <tar.name>${project.build.finalName}.tar.gz</tar.name>
  <maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm</maven.build.timestamp.format>
  <buildDate>${maven.build.timestamp}</buildDate>
- <compileSource>1.8</compileSource>
+ <compileSource>17</compileSource>
  <releaseTarget>8</releaseTarget>
  <!-- Build dependencies -->
  <!-- The $revision feature is introduced in 3.5.0 -->