HBASE-28679 Upgrade yetus to a newer version

dev-support/Jenkinsfile

@@ -31,7 +31,7 @@ pipeline {
  disableConcurrentBuilds()
  }
  environment {
- YETUS_RELEASE = '0.12.0'
+ YETUS_RELEASE = '0.15.0'
  // where we'll write everything from different steps. Need a copy here so the final step can check for success/failure.
  OUTPUT_DIR_RELATIVE_GENERAL = 'output-general'
  OUTPUT_DIR_RELATIVE_JDK8_HADOOP2 = 'output-jdk8-hadoop2'
@@ -43,12 +43,12 @@ pipeline {
  PROJECT_PERSONALITY = 'https://raw.githubusercontent.com/apache/hbase/master/dev-support/hbase-personality.sh'
  PERSONALITY_FILE = 'tools/personality.sh'
  // This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
- AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc,dev-support/test-patch.sh'
- WHITESPACE_IGNORE_LIST = '.*/generated/.*'
+ AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
+ BLANKS_IGNORE_LIST = '.*/generated/.*'
  // output from surefire; sadly the archive function in yetus only works on file names.
  ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
  // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
- TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
+ TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
  EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${BRANCH_NAME}/lastSuccessfulBuild/artifact/output/excludes"
  // TODO does hadoopcheck need to be jdk specific?
  SHALLOW_CHECKS = 'all,-shadedjars,-unit' // run by the 'yetus general check'

dev-support/Jenkinsfile_GitHub

@@ -36,14 +36,16 @@ pipeline {
  YETUS_REL = 'yetus'
  DOCKERFILE_REL = "${SRC_REL}/dev-support/docker/Dockerfile"
  YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins_precommit_github_yetus.sh"
- // Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
- YETUS_VERSION = 'rel/0.12.0'
+ YETUS_VERSION = '0.15.0'
  GENERAL_CHECK_PLUGINS = 'all,-javadoc,-jira,-shadedjars,-unit'
  JDK_SPECIFIC_PLUGINS = 'compile,github,htmlout,javac,javadoc,maven,mvninstall,shadedjars,unit'
+ // This section of the docs tells folks not to use the javadoc tag. older branches have our old version of the check for said tag.
+ AUTHOR_IGNORE_LIST = 'src/main/asciidoc/_chapters/developer.adoc'
+ BLANKS_IGNORE_LIST = '.*/generated/.*'
  // output from surefire; sadly the archive function in yetus only works on file names.
  ARCHIVE_PATTERN_LIST = 'TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump'
  // These tests currently have known failures. Once they burn down to 0, remove from here so that new problems will cause a failure.
- TESTS_FILTER = 'cc,checkstyle,javac,javadoc,pylint,shellcheck,whitespace,perlcritic,ruby-lint,rubocop,mvnsite'
+ TESTS_FILTER = 'checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop'
  EXCLUDE_TESTS_URL = "${JENKINS_URL}/job/HBase-Find-Flaky-Tests/job/${CHANGE_TARGET}/lastSuccessfulBuild/artifact/output/excludes"
  // set build parallel
  BUILD_THREAD = 4
@@ -102,11 +104,11 @@ pipeline {
  checkout scm
  }
  dir("${YETUSDIR}") {
- checkout([
- $class : 'GitSCM',
- branches : [[name: "${YETUS_VERSION}"]],
- userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
- )
+ sh```#!/usr/bin/env bash
+ wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ ```
  }
  dir("${WORKDIR}") {
  withCredentials([
@@ -229,11 +231,11 @@ pipeline {
  checkout scm
  }
  dir("${YETUSDIR}") {
- checkout([
- $class : 'GitSCM',
- branches : [[name: "${YETUS_VERSION}"]],
- userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
- )
+ sh```#!/usr/bin/env bash
+ wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ ```
  }
  dir("${WORKDIR}") {
  withCredentials([
@@ -365,11 +367,11 @@ pipeline {
  checkout scm
  }
  dir("${YETUSDIR}") {
- checkout([
- $class : 'GitSCM',
- branches : [[name: "${YETUS_VERSION}"]],
- userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
- )
+ sh```#!/usr/bin/env bash
+ wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ ```
  }
  dir("${WORKDIR}") {
  withCredentials([
@@ -493,22 +495,19 @@ pipeline {
  SKIP_ERRORPRONE = true
  }
  when {
- allOf {
- // this will return true if the pipeline is building a change request, such as a GitHub pull request.
- changeRequest()
- expression { env.CHANGE_TARGET in ['master', 'branch-3'] }
- }
+ // this will return true if the pipeline is building a change request, such as a GitHub pull request.
+ changeRequest()
  }
  steps {
  dir("${SOURCEDIR}") {
  checkout scm
  }
  dir("${YETUSDIR}") {
- checkout([
- $class : 'GitSCM',
- branches : [[name: "${YETUS_VERSION}"]],
- userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
- )
+ sh```#!/usr/bin/env bash
+ wget https://dlcdn.apache.org/yetus/${YETUS_VERSION}/apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ tar --strip-components=1 -xzf apache-yetus-${YETUS_VERSION}-bin.tar.gz && \
+ rm apache-yetus-${YETUS_VERSION}-bin.tar.gz
+ ```
  }
  dir("${WORKDIR}") {
  withCredentials([

dev-support/docker/Dockerfile

@@ -53,6 +53,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \
  ruby=1:3.0* \
  ruby-dev=1:3.0* \
  shellcheck='0.8.0-*' \
+ libxml2-dev='2.9.13+dfsg-*' \
+ libxml2-utils='2.9.13+dfsg-*' \
  && \
  apt-get clean && \
  rm -rf /var/lib/apt/lists/* \

dev-support/hbase_nightly_yetus.sh

@@ -20,7 +20,7 @@ declare -i missing_env=0
 # Validate params
 for required_env in "TESTS" "PERSONALITY_FILE" "BASEDIR" "ARCHIVE_PATTERN_LIST" "OUTPUT_DIR_RELATIVE" \
  "OUTPUT_DIR" "PROJECT" "AUTHOR_IGNORE_LIST" \
- "WHITESPACE_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
+ "BLANKS_IGNORE_LIST" "BRANCH_NAME" "TESTS_FILTER" "DEBUG" \
  "USE_YETUS_PRERELEASE" "WORKSPACE" "YETUS_RELEASE"; do
  if [ -z "${!required_env}" ]; then
  echo "[ERROR] Required environment variable '${required_env}' is not set."
@@ -59,8 +59,8 @@ YETUS_ARGS=("--patch-dir=${OUTPUT_DIR}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--project=${PROJECT}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--resetrepo" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--author-ignore-list=${AUTHOR_IGNORE_LIST}" "${YETUS_ARGS[@]}")
-YETUS_ARGS=("--whitespace-eol-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}")
-YETUS_ARGS=("--whitespace-tabs-ignore-list=${WHITESPACE_IGNORE_LIST}" "${YETUS_ARGS[@]}")
+YETUS_ARGS=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}")
+YETUS_ARGS=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
@@ -106,11 +106,9 @@ if [[ -n "${JAVA8_HOME}" ]]; then
 fi
 
 if [[ true != "${USE_YETUS_PRERELEASE}" ]]; then
- YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/shelldocs" "${YETUS_ARGS[@]}")
  TESTPATCHBIN="${WORKSPACE}/yetus-${YETUS_RELEASE}/bin/test-patch"
 else
- YETUS_ARGS=("--shelldocs=${WORKSPACE}/yetus-git/shelldocs/shelldocs.py" "${YETUS_ARGS[@]}")
- TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/test-patch.sh"
+ TESTPATCHBIN="${WORKSPACE}/yetus-git/precommit/src/main/shell/test-patch.sh"
 fi
 echo "Launching yetus with command line:"
 echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"

dev-support/jenkins_precommit_github_yetus.sh

@@ -41,6 +41,8 @@ declare -a required_envs=(
  "SOURCEDIR"
  "TESTS_FILTER"
  "YETUSDIR"
+ "AUTHOR_IGNORE_LIST"
+ "BLANKS_IGNORE_LIST"
 )
 # Validate params
 for required_env in "${required_envs[@]}"; do
@@ -57,7 +59,7 @@ if [ ${missing_env} -gt 0 ]; then
 fi
 
 # TODO (HBASE-23900): cannot assume test-patch runs directly from sources
-TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
+TESTPATCHBIN="${YETUSDIR}/bin/test-patch"
 
 # this must be clean for every run
 rm -rf "${PATCHDIR}"
@@ -87,8 +89,8 @@ YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
 YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
 YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
 # enable writing back to Github
-YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}")
-YETUS_ARGS+=("--github-user=${GITHUB_USER}")
+YETUS_ARGS+=("--github-token=${GITHUB_PASSWORD}")
+YETUS_ARGS+=("--github-write-comment")
 # auto-kill any surefire stragglers during unit test runs
 YETUS_ARGS+=("--reapermode=kill")
 # set relatively high limits for ASF machines
@@ -109,8 +111,9 @@ YETUS_ARGS+=("--docker")
 YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
 YETUS_ARGS+=("--mvn-custom-repos")
 YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}")
-YETUS_ARGS+=("--whitespace-eol-ignore-list=.*/generated/.*")
-YETUS_ARGS+=("--whitespace-tabs-ignore-list=.*/generated/.*")
+YETUS_ARGS+=("--author-ignore-list=${AUTHOR_IGNORE_LIST}")
+YETUS_ARGS+=("--blanks-eol-ignore-list=${BLANKS_IGNORE_LIST}")
+YETUS_ARGS+=("--blanks-tabs-ignore-list=${BLANKS_IGNORE_LIST}*")
 YETUS_ARGS+=("--tests-filter=${TESTS_FILTER}")
 YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/hbase-personality.sh")
 YETUS_ARGS+=("--quick-hadoopcheck")