@@ -23,7 +23,7 @@ pipeline {
2323
2424 options {
2525 buildDiscarder(logRotator(numToKeepStr : ' 5'
26- timeout (time : 20 , unit : ' HOURS'
26+ timeout (time : 24 , unit : ' HOURS'
2727 timestamps()
2828 checkoutToSubdirectory(' src'
2929 }
@@ -32,7 +32,6 @@ pipeline {
3232 SOURCEDIR = ' src'
3333 // will also need to change notification section below
3434 PATCHDIR = ' out'
35- DOCKERFILE = " ${ SOURCEDIR} "
3635 YETUS = ' yetus'
3736 // Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
3837 YETUS_VERSION = ' f9ba0170a5787a5f4662d3769804fef0226a182f'
@@ -57,7 +56,17 @@ pipeline {
5756 }
5857 }
5958
60- stage (' precommit-run'
59+ // This is an optional stage which runs only when there's a change in
60+ // C++/C++ build/platform.
61+ // This stage serves as a means of cross platform validation, which is
62+ // really needed to ensure that any C++ related/platform change doesn't
63+ // break the Hadoop build on Centos 8.
64+ stage (' precommit-run Centos 8'
65+ environment {
66+ DOCKERFILE = " ${ SOURCEDIR} "
67+ IS_OPTIONAL = 1
68+ }
69+
6170 steps {
6271 withCredentials(
6372 [usernamePassword(credentialsId : ' apache-hadoop-at-github.com'
@@ -68,100 +77,61 @@ pipeline {
6877 usernameVariable : ' JIRA_USER'
6978 sh ''' #!/usr/bin/env bash
7079
71- set -e
72-
73- TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
74-
75- # this must be clean for every run
76- if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
77- rm -rf "${WORKSPACE}/${PATCHDIR}"
78- fi
79- mkdir -p "${WORKSPACE}/${PATCHDIR}"
80-
81- # if given a JIRA issue, process it. If CHANGE_URL is set
82- # (e.g., Github Branch Source plugin), process it.
83- # otherwise exit, because we don't want Hadoop to do a
84- # full build. We wouldn't normally do this check for smaller
85- # projects. :)
86- if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
87- YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
88- elif [[ -z "${CHANGE_URL}" ]]; then
89- echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
90- exit 0
91- fi
92-
93- YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
94-
95- # where the source is located
96- YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
97-
98- # our project defaults come from a personality file
99- YETUS_ARGS+=("--project=hadoop")
100- YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh")
101-
102- # lots of different output formats
103- YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
104- YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
105- YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
106-
107- # enable writing back to Github
108- YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
109-
110- # enable writing back to ASF JIRA
111- YETUS_ARGS+=(--jira-password="${JIRA_PASSWORD}")
112- YETUS_ARGS+=(--jira-user="${JIRA_USER}")
113-
114- # auto-kill any surefire stragglers during unit test runs
115- YETUS_ARGS+=("--reapermode=kill")
116-
117- # set relatively high limits for ASF machines
118- # changing these to higher values may cause problems
119- # with other jobs on systemd-enabled machines
120- YETUS_ARGS+=("--proclimit=5500")
121- YETUS_ARGS+=("--dockermemlimit=22g")
122-
123- # -1 spotbugs issues that show up prior to the patch being applied
124- YETUS_ARGS+=("--spotbugs-strict-precheck")
125-
126- # rsync these files back into the archive dir
127- YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
128-
129- # URL for user-side presentation in reports and such to our artifacts
130- # (needs to match the archive bits below)
131- YETUS_ARGS+=("--build-url-artifacts=artifact/out")
132-
133- # plugins to enable
134- YETUS_ARGS+=("--plugins=all")
135-
136- # don't let these tests cause -1s because we aren't really paying that
137- # much attention to them
138- YETUS_ARGS+=("--tests-filter=checkstyle")
139-
140- # run in docker mode and specifically point to our
141- # Dockerfile since we don't want to use the auto-pulled version.
142- YETUS_ARGS+=("--docker")
143- YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
144- YETUS_ARGS+=("--mvn-custom-repos")
80+ chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
81+ "${SOURCEDIR}/dev-support/jenkins.sh"
82+ '''
83+ }
84+ }
85+ }
14586
146- # effectively treat dev-suport as a custom maven module
147- YETUS_ARGS+=("--skip-dirs=dev-support")
87+ // This is an optional stage which runs only when there's a change in
88+ // C++/C++ build/platform.
89+ // This stage serves as a means of cross platform validation, which is
90+ // really needed to ensure that any C++ related/platform change doesn't
91+ // break the Hadoop build on Debian 10.
92+ stage (' precommit-run Debian 10'
93+ environment {
94+ DOCKERFILE = " ${ SOURCEDIR} "
95+ IS_OPTIONAL = 1
96+ }
14897
149- # help keep the ASF boxes clean
150- YETUS_ARGS+=("--sentinel")
98+ steps {
99+ withCredentials(
100+ [usernamePassword(credentialsId : ' apache-hadoop-at-github.com'
101+ passwordVariable : ' GITHUB_TOKEN'
102+ usernameVariable : ' GITHUB_USER'
103+ usernamePassword(credentialsId : ' hadoopqa-at-asf-jira'
104+ passwordVariable : ' JIRA_PASSWORD'
105+ usernameVariable : ' JIRA_USER'
106+ sh ''' #!/usr/bin/env bash
151107
152- # test with Java 8 and 11
153- YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
154- YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
155- YETUS_ARGS+=("--multijdktests=compile")
108+ chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
109+ "${SOURCEDIR}/dev-support/jenkins.sh"
110+ '''
111+ }
112+ }
113+ }
156114
157- # custom javadoc goals
158- YETUS_ARGS+=("--mvn-javadoc-goals=process-sources,javadoc:javadoc-no-fork")
115+ // We want to use Ubuntu Focal as our main CI and thus, this stage
116+ // isn't optional (runs for all the PRs).
117+ stage (' precommit-run Ubuntu focal'
118+ environment {
119+ DOCKERFILE = " ${ SOURCEDIR} "
120+ IS_OPTIONAL = 0
121+ }
159122
160- # write Yetus report as GitHub comment (YETUS-1102)
161- YETUS_ARGS+=("--github-write-comment")
162- YETUS_ARGS+=("--github-use-emoji-vote")
123+ steps {
124+ withCredentials(
125+ [usernamePassword(credentialsId : ' apache-hadoop-at-github.com'
126+ passwordVariable : ' GITHUB_TOKEN'
127+ usernameVariable : ' GITHUB_USER'
128+ usernamePassword(credentialsId : ' hadoopqa-at-asf-jira'
129+ passwordVariable : ' JIRA_PASSWORD'
130+ usernameVariable : ' JIRA_USER'
131+ sh ''' #!/usr/bin/env bash
163132
164- "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
133+ chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
134+ "${SOURCEDIR}/dev-support/jenkins.sh"
165135 '''
166136 }
167137 }
0 commit comments