Skip to content

Commit

Permalink
contrib/cray: Change the Verbs Jenkins pipeline test reporting to xUnit.
Browse files Browse the repository at this point in the history 
Move the SFT Testing stage to its own stage in the Jenkins pipeline.
Add a SFT Test baseline file that contains the expected results.

Signed-off-by: Tony Zinger <ajz@cray.com>
  • Loading branch information
@tonyzinger
tonyzinger committed Aug 17, 2018
1 parent 734a8d3 commit 2c8581c
Show file tree
Hide file tree
Showing 2 changed files with 178 additions and 39 deletions.
104 changes: 65 additions & 39 deletions contrib/cray/Jenkinsfile.verbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ pipeline {
options {
// Generic build options
timeout (time: 30, unit: 'MINUTES')
buildDiscarder(logRotator(numToKeepStr: '5'))
buildDiscarder(logRotator(numToKeepStr: '15'))

// Build options
disableConcurrentBuilds()
Expand Down Expand Up @@ -51,6 +51,7 @@ pipeline {
environment {
LD_LIBRARY_PATH = "$TMP_INSTALL_PATH/lib:$LD_LIBRARY_PATH"
}
failFast true
parallel {
stage('Unit tests') {
steps {
Expand All @@ -65,44 +66,6 @@ pipeline {
}
}
}
stage('SFT tests') {
environment {
SFT_BIN = "${ROOT_BUILD_PATH + '/libfabric-sft/stable/bin'}"
SFT_ADD_ARGS = '--dbg-cq-retry 1 --dbg-cq-delay 5'
SFT_MAX_JOB_TIME = '3'
SFT_NUM_JOBS = '4'
SFT_PARTITION = 'mellanox'
SFT_PROVIDER = 'verbs;ofi_rxm'
SFT_TEST_CMDS = 'sft_test_commands'
SFT_TEST_RESULTS = 'sft_test_results.xml'
SFT_TEST_RESULTS_CI = 'sft_ci_results.yaml'
SFT_TEST_RESULTS_BASE_DIR = "${ROOT_BUILD_PATH}" + "/sft_test_results/"
SFT_TEST_RESULTS_DIR = ""
}
steps {
echo 'SFT tests'
script {
def cur_date_time = new Date().format('yyyy_MM_dd_HH_mm_ss')
SFT_TEST_RESULTS_SUBDIR = "BUILD_" + env.BUILD_ID + "_DATE_" + "${cur_date_time}"
SFT_TEST_RESULTS_DIR = "${SFT_TEST_RESULTS_BASE_DIR}" + "${SFT_TEST_RESULTS_SUBDIR}"
}
sh "rm -f ${SFT_BIN}/core*"
sh "rm -rf ${SFT_TEST_RESULTS_DIR}"
sh "mkdir -p ${SFT_TEST_RESULTS_DIR}"
sh "cd ${SFT_BIN};./ci-all.sh --provider '${SFT_PROVIDER}' --partition ${SFT_PARTITION} -L ${SFT_TEST_RESULTS_DIR} --num-jobs ${SFT_NUM_JOBS} --max-job-time ${SFT_MAX_JOB_TIME} --output-cmds ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_CMDS} --results-file ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_CI} --additional-args '${SFT_ADD_ARGS}'"
}
post {
always {
sh "${SFT_BIN}/sft_parse_test_results.pm -d ${SFT_TEST_RESULTS_DIR} -o ${SFT_TEST_RESULTS}"
sh "gzip -r ${SFT_TEST_RESULTS_DIR}"
sh "gunzip ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS}"
sh "cp -r ${SFT_TEST_RESULTS_DIR} ."
archiveArtifacts artifacts: "${SFT_TEST_RESULTS_SUBDIR}/*"
junit testResults: "${SFT_TEST_RESULTS_SUBDIR}/${SFT_TEST_RESULTS}", healthScaleFactor: 0.0
sh "rm -rf ${SFT_TEST_RESULTS_DIR}"
}
}
}
stage('Fabtests') {
steps {
timeout (time: 20, unit: 'MINUTES') {
Expand All @@ -115,6 +78,69 @@ pipeline {
}
}
}
stage('SFT tests') {
environment {
LD_LIBRARY_PATH = "$TMP_INSTALL_PATH/lib:$LD_LIBRARY_PATH"
SFT_BIN = "${ROOT_BUILD_PATH + '/libfabric-sft/stable/bin'}"
SFT_ADD_ARGS = "--additional-args '--dbg-cq-retry 1 --dbg-cq-delay 5'"
// SFT_ADD_ARGS = " "
SFT_MAX_JOB_TIME = '3'
SFT_NUM_JOBS = '4'
// SFT_PARTITION = '--partition mellanox'
SFT_PARTITION = ' '
SFT_PROVIDER = 'verbs;ofi_rxm'
SFT_BASELINE_DIR = "contrib/cray/"
SFT_BASELINE_RESULTS_FILE = 'sft_test_results_baseline.txt'
SFT_PREVIOUS_BASELINE_RESULTS = 'sft_test_results_baseline.txt'
SFT_TEST_CMDS = 'sft_test_commands'
SFT_TEST_RESULTS = 'sft_test_results.xml'
SFT_TEST_RESULTS_EXPECTED = 'expected_'
SFT_TEST_RESULTS_PREFIX = 'BUILD_'
// SFT_TEST_RESULTS_PREVIOUS = 'previous_'
SFT_TEST_RESULTS_CI = 'sft_ci_results.yaml'
SFT_TEST_RESULTS_BASE_DIR = "${ROOT_BUILD_PATH}" + "/sft_test_results/"
SFT_TEST_RESULTS_DIR = ""
}
steps {
echo 'SFT tests'
script {
def cur_date_time = new Date().format('yyyy_MM_dd_HH_mm_ss')
SFT_TEST_RESULTS_SUBDIR = "${SFT_TEST_RESULTS_PREFIX}" + env.BUILD_ID + "_DATE_" + "${cur_date_time}"
SFT_TEST_RESULTS_DIR = "${SFT_TEST_RESULTS_BASE_DIR}" + "${SFT_TEST_RESULTS_SUBDIR}"
}
sh "rm -f ${SFT_BIN}/core*"
sh "rm -rf ${SFT_TEST_RESULTS_DIR}"
sh "mkdir -p ${SFT_TEST_RESULTS_DIR}"
sh "cd ${SFT_BIN};./ci-all.sh --provider '${SFT_PROVIDER}' ${SFT_PARTITION} -L ${SFT_TEST_RESULTS_DIR} --num-jobs ${SFT_NUM_JOBS} --max-job-time ${SFT_MAX_JOB_TIME} --output-cmds ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_CMDS} --results-file ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_CI} ${SFT_ADD_ARGS}"
}
post {
always {
// script {
// try {
// copyArtifacts(projectName: "${env.JOB_NAME}", selector: lastSuccessful(), filter: "${SFT_TEST_RESULTS_PREFIX}*/${SFT_BASELINE_RESULTS_FILE}");
// sh "cp ./${SFT_TEST_RESULTS_PREFIX}*/${SFT_BASELINE_RESULTS_FILE} ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_PREVIOUS}${SFT_BASELINE_RESULTS_FILE}"
// }
// catch (Exception e) {
// echo 'No rpms to archive'
// }
// }
sh "cp ./${SFT_BASELINE_DIR}/${SFT_BASELINE_RESULTS_FILE} ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_EXPECTED}${SFT_BASELINE_RESULTS_FILE}"
sh "${SFT_BIN}/sft_parse_test_results.pm -b ${SFT_TEST_RESULTS_EXPECTED}${SFT_BASELINE_RESULTS_FILE} -d ${SFT_TEST_RESULTS_DIR} -o ${SFT_TEST_RESULTS} -r ${SFT_BASELINE_RESULTS_FILE}"
sh "gzip -r ${SFT_TEST_RESULTS_DIR}"
sh "gunzip ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS}"
sh "gunzip ${SFT_TEST_RESULTS_DIR}/${SFT_BASELINE_RESULTS_FILE}"
sh "gunzip ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_EXPECTED}${SFT_BASELINE_RESULTS_FILE}"
// sh "gunzip ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_PREVIOUS}${SFT_BASELINE_RESULTS_FILE}"
sh "cp -r ${SFT_TEST_RESULTS_DIR} ."
archiveArtifacts artifacts: "${SFT_TEST_RESULTS_SUBDIR}/*"
step ([$class: 'XUnitBuilder',
thresholds: [
[$class: 'FailedThreshold', unstableThreshold: '0']],
tools: [[$class: 'JUnitType', pattern: "${SFT_TEST_RESULTS_SUBDIR}/${SFT_TEST_RESULTS}"]]])
sh "rm -rf ${SFT_TEST_RESULTS_DIR}"
}
}
}
stage("Deploy: Install") {
steps {
sh "make clean"
Expand Down
113 changes: 113 additions & 0 deletions contrib/cray/sft_test_results_baseline.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
# Expected test values can be: 'PASS', 'FAIL', SKIP' or 'INTERMITTENT'
# PASS: means that the test should pass
# if the test result is PASS, then Jenkins will report this test as PASSED
# if the test result is FAIL, then Jenkins will report this test as FAILED
# FAIL: means that the test currently fails, probably due to incorrect behavior of a feature
# if the test result is PASS, then Jenkins will report this test as PASSED
# if the test result is FAIL, then Jenkins will report this test as SKIPPED
# SKIP: means that the test should be skipped, due to an unimplemented feature
# if the test result is PASS, then Jenkins will report this test as PASSED
# if the test result is FAIL, then Jenkins will report this test as SKIPPED
# INTERMITTENT: means that the test should pass but it fails infrequently
# if the test result is PASS, then Jenkins will report this test as PASSED
# if the test result is FAIL, then Jenkins will report this test as SKIPPED

MESSAGE_All-to-All_FI_INJECT=PASS
MESSAGE_All-to-All_FI_INJECTDATA=PASS
MESSAGE_All-to-All_FI_SEND=PASS
MESSAGE_All-to-All_FI_SENDDATA=PASS
MESSAGE_All-to-All_FI_SENDMSG=PASS
MESSAGE_All-to-All_FI_SENDV=PASS
MESSAGE_All-to-One_FI_INJECT=PASS
MESSAGE_All-to-One_FI_INJECTDATA=PASS
MESSAGE_All-to-One_FI_SEND=PASS
MESSAGE_All-to-One_FI_SENDDATA=PASS
MESSAGE_All-to-One_FI_SENDMSG=PASS
MESSAGE_All-to-One_FI_SENDV=PASS
MESSAGE_One-to-All_FI_INJECT=PASS
MESSAGE_One-to-All_FI_INJECTDATA=PASS
MESSAGE_One-to-All_FI_SEND=PASS
MESSAGE_One-to-All_FI_SENDDATA=PASS
MESSAGE_One-to-All_FI_SENDMSG=PASS
MESSAGE_One-to-All_FI_SENDV=PASS
MESSAGE_Round-Robin_FI_INJECT=PASS
MESSAGE_Round-Robin_FI_INJECTDATA=PASS
MESSAGE_Round-Robin_FI_SEND=PASS
MESSAGE_Round-Robin_FI_SENDDATA=PASS
MESSAGE_Round-Robin_FI_SENDDATA_TRIGGER=SKIP
MESSAGE_Round-Robin_FI_SENDMSG=PASS
MESSAGE_Round-Robin_FI_SENDMSG_TRIGGER=SKIP
MESSAGE_Round-Robin_FI_SENDV=PASS
MESSAGE_Round-Robin_FI_SENDV_TRIGGER=SKIP
MESSAGE_Round-Robin_FI_SEND_TRIGGER=SKIP
RMA_All-to-All_FI_INJECT_WRITE=PASS
RMA_All-to-All_FI_INJECT_WRITEDATA=PASS
RMA_All-to-All_FI_READ=FAIL
RMA_All-to-All_FI_READMSG=FAIL
RMA_All-to-All_FI_READV=FAIL
RMA_All-to-All_FI_WRITE=PASS
RMA_All-to-All_FI_WRITEDATA=PASS
RMA_All-to-All_FI_WRITEMSG=PASS
RMA_All-to-All_FI_WRITEV=PASS
RMA_All-to-One_FI_INJECT_WRITE=PASS
RMA_All-to-One_FI_INJECT_WRITEDATA=PASS
RMA_All-to-One_FI_READ=FAIL
RMA_All-to-One_FI_READMSG=FAIL
RMA_All-to-One_FI_READV=FAIL
RMA_All-to-One_FI_WRITE=PASS
RMA_All-to-One_FI_WRITEDATA=PASS
RMA_All-to-One_FI_WRITEMSG=PASS
RMA_All-to-One_FI_WRITEV=PASS
RMA_One-to-All_FI_INJECT_WRITE=PASS
RMA_One-to-All_FI_INJECT_WRITEDATA=PASS
RMA_One-to-All_FI_READ=FAIL
RMA_One-to-All_FI_READMSG=FAIL
RMA_One-to-All_FI_READV=FAIL
RMA_One-to-All_FI_WRITE=PASS
RMA_One-to-All_FI_WRITEDATA=PASS
RMA_One-to-All_FI_WRITEMSG=PASS
RMA_One-to-All_FI_WRITEV=PASS
RMA_Round-Robin_FI_INJECT_WRITE=PASS
RMA_Round-Robin_FI_INJECT_WRITEDATA=PASS
RMA_Round-Robin_FI_READ=FAIL
RMA_Round-Robin_FI_READMSG=FAIL
RMA_Round-Robin_FI_READMSG_TRIGGER=SKIP
RMA_Round-Robin_FI_READV=FAIL
RMA_Round-Robin_FI_READV_TRIGGER=SKIP
RMA_Round-Robin_FI_READ_TRIGGER=SKIP
RMA_Round-Robin_FI_WRITE=PASS
RMA_Round-Robin_FI_WRITEDATA=PASS
RMA_Round-Robin_FI_WRITEDATA_TRIGGER=SKIP
RMA_Round-Robin_FI_WRITEMSG=PASS
RMA_Round-Robin_FI_WRITEMSG_TRIGGER=SKIP
RMA_Round-Robin_FI_WRITEV=PASS
RMA_Round-Robin_FI_WRITEV_TRIGGER=SKIP
RMA_Round-Robin_FI_WRITE_TRIGGER=SKIP
TAGGED_All-to-All_FI_TINJECT=PASS
TAGGED_All-to-All_FI_TINJECTDATA=PASS
TAGGED_All-to-All_FI_TSEND=PASS
TAGGED_All-to-All_FI_TSENDDATA=PASS
TAGGED_All-to-All_FI_TSENDMSG=PASS
TAGGED_All-to-All_FI_TSENDV=PASS
TAGGED_All-to-One_FI_TINJECT=PASS
TAGGED_All-to-One_FI_TINJECTDATA=PASS
TAGGED_All-to-One_FI_TSEND=PASS
TAGGED_All-to-One_FI_TSENDDATA=PASS
TAGGED_All-to-One_FI_TSENDMSG=PASS
TAGGED_All-to-One_FI_TSENDV=PASS
TAGGED_One-to-All_FI_TINJECT=PASS
TAGGED_One-to-All_FI_TINJECTDATA=PASS
TAGGED_One-to-All_FI_TSEND=PASS
TAGGED_One-to-All_FI_TSENDDATA=PASS
TAGGED_One-to-All_FI_TSENDMSG=PASS
TAGGED_One-to-All_FI_TSENDV=PASS
TAGGED_Round-Robin_FI_TINJECT=PASS
TAGGED_Round-Robin_FI_TINJECTDATA=PASS
TAGGED_Round-Robin_FI_TSEND=PASS
TAGGED_Round-Robin_FI_TSENDDATA=PASS
TAGGED_Round-Robin_FI_TSENDDATA_TRIGGER=SKIP
TAGGED_Round-Robin_FI_TSENDMSG=PASS
TAGGED_Round-Robin_FI_TSENDMSG_TRIGGER=SKIP
TAGGED_Round-Robin_FI_TSENDV=PASS
TAGGED_Round-Robin_FI_TSENDV_TRIGGER=SKIP
TAGGED_Round-Robin_FI_TSEND_TRIGGER=SKIP

0 comments on commit 2c8581c

Please sign in to comment.