From 2c8581c1274edc1cbd689144cd5e6e032f7a900d Mon Sep 17 00:00:00 2001 From: Tony Zinger Date: Fri, 17 Aug 2018 09:01:25 -0500 Subject: [PATCH] contrib/cray: Change the Verbs Jenkins pipeline test reporting to xUnit. Move the SFT Testing stage to its own stage in the Jenkins pipeline. Add a SFT Test baseline file that contains the expected results. Signed-off-by: Tony Zinger --- contrib/cray/Jenkinsfile.verbs | 104 ++++++++++++------- contrib/cray/sft_test_results_baseline.txt | 113 +++++++++++++++++++++ 2 files changed, 178 insertions(+), 39 deletions(-) create mode 100644 contrib/cray/sft_test_results_baseline.txt diff --git a/contrib/cray/Jenkinsfile.verbs b/contrib/cray/Jenkinsfile.verbs index 05bf0b38153..34fd33c481a 100644 --- a/contrib/cray/Jenkinsfile.verbs +++ b/contrib/cray/Jenkinsfile.verbs @@ -11,7 +11,7 @@ pipeline { options { // Generic build options timeout (time: 30, unit: 'MINUTES') - buildDiscarder(logRotator(numToKeepStr: '5')) + buildDiscarder(logRotator(numToKeepStr: '15')) // Build options disableConcurrentBuilds() @@ -51,6 +51,7 @@ pipeline { environment { LD_LIBRARY_PATH = "$TMP_INSTALL_PATH/lib:$LD_LIBRARY_PATH" } + failFast true parallel { stage('Unit tests') { steps { @@ -65,44 +66,6 @@ pipeline { } } } - stage('SFT tests') { - environment { - SFT_BIN = "${ROOT_BUILD_PATH + '/libfabric-sft/stable/bin'}" - SFT_ADD_ARGS = '--dbg-cq-retry 1 --dbg-cq-delay 5' - SFT_MAX_JOB_TIME = '3' - SFT_NUM_JOBS = '4' - SFT_PARTITION = 'mellanox' - SFT_PROVIDER = 'verbs;ofi_rxm' - SFT_TEST_CMDS = 'sft_test_commands' - SFT_TEST_RESULTS = 'sft_test_results.xml' - SFT_TEST_RESULTS_CI = 'sft_ci_results.yaml' - SFT_TEST_RESULTS_BASE_DIR = "${ROOT_BUILD_PATH}" + "/sft_test_results/" - SFT_TEST_RESULTS_DIR = "" - } - steps { - echo 'SFT tests' - script { - def cur_date_time = new Date().format('yyyy_MM_dd_HH_mm_ss') - SFT_TEST_RESULTS_SUBDIR = "BUILD_" + env.BUILD_ID + "_DATE_" + "${cur_date_time}" - SFT_TEST_RESULTS_DIR = "${SFT_TEST_RESULTS_BASE_DIR}" + "${SFT_TEST_RESULTS_SUBDIR}" - } - sh "rm -f ${SFT_BIN}/core*" - sh "rm -rf ${SFT_TEST_RESULTS_DIR}" - sh "mkdir -p ${SFT_TEST_RESULTS_DIR}" - sh "cd ${SFT_BIN};./ci-all.sh --provider '${SFT_PROVIDER}' --partition ${SFT_PARTITION} -L ${SFT_TEST_RESULTS_DIR} --num-jobs ${SFT_NUM_JOBS} --max-job-time ${SFT_MAX_JOB_TIME} --output-cmds ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_CMDS} --results-file ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_CI} --additional-args '${SFT_ADD_ARGS}'" - } - post { - always { - sh "${SFT_BIN}/sft_parse_test_results.pm -d ${SFT_TEST_RESULTS_DIR} -o ${SFT_TEST_RESULTS}" - sh "gzip -r ${SFT_TEST_RESULTS_DIR}" - sh "gunzip ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS}" - sh "cp -r ${SFT_TEST_RESULTS_DIR} ." - archiveArtifacts artifacts: "${SFT_TEST_RESULTS_SUBDIR}/*" - junit testResults: "${SFT_TEST_RESULTS_SUBDIR}/${SFT_TEST_RESULTS}", healthScaleFactor: 0.0 - sh "rm -rf ${SFT_TEST_RESULTS_DIR}" - } - } - } stage('Fabtests') { steps { timeout (time: 20, unit: 'MINUTES') { @@ -115,6 +78,69 @@ pipeline { } } } + stage('SFT tests') { + environment { + LD_LIBRARY_PATH = "$TMP_INSTALL_PATH/lib:$LD_LIBRARY_PATH" + SFT_BIN = "${ROOT_BUILD_PATH + '/libfabric-sft/stable/bin'}" + SFT_ADD_ARGS = "--additional-args '--dbg-cq-retry 1 --dbg-cq-delay 5'" +// SFT_ADD_ARGS = " " + SFT_MAX_JOB_TIME = '3' + SFT_NUM_JOBS = '4' +// SFT_PARTITION = '--partition mellanox' + SFT_PARTITION = ' ' + SFT_PROVIDER = 'verbs;ofi_rxm' + SFT_BASELINE_DIR = "contrib/cray/" + SFT_BASELINE_RESULTS_FILE = 'sft_test_results_baseline.txt' + SFT_PREVIOUS_BASELINE_RESULTS = 'sft_test_results_baseline.txt' + SFT_TEST_CMDS = 'sft_test_commands' + SFT_TEST_RESULTS = 'sft_test_results.xml' + SFT_TEST_RESULTS_EXPECTED = 'expected_' + SFT_TEST_RESULTS_PREFIX = 'BUILD_' +// SFT_TEST_RESULTS_PREVIOUS = 'previous_' + SFT_TEST_RESULTS_CI = 'sft_ci_results.yaml' + SFT_TEST_RESULTS_BASE_DIR = "${ROOT_BUILD_PATH}" + "/sft_test_results/" + SFT_TEST_RESULTS_DIR = "" + } + steps { + echo 'SFT tests' + script { + def cur_date_time = new Date().format('yyyy_MM_dd_HH_mm_ss') + SFT_TEST_RESULTS_SUBDIR = "${SFT_TEST_RESULTS_PREFIX}" + env.BUILD_ID + "_DATE_" + "${cur_date_time}" + SFT_TEST_RESULTS_DIR = "${SFT_TEST_RESULTS_BASE_DIR}" + "${SFT_TEST_RESULTS_SUBDIR}" + } + sh "rm -f ${SFT_BIN}/core*" + sh "rm -rf ${SFT_TEST_RESULTS_DIR}" + sh "mkdir -p ${SFT_TEST_RESULTS_DIR}" + sh "cd ${SFT_BIN};./ci-all.sh --provider '${SFT_PROVIDER}' ${SFT_PARTITION} -L ${SFT_TEST_RESULTS_DIR} --num-jobs ${SFT_NUM_JOBS} --max-job-time ${SFT_MAX_JOB_TIME} --output-cmds ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_CMDS} --results-file ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_CI} ${SFT_ADD_ARGS}" + } + post { + always { +// script { +// try { +// copyArtifacts(projectName: "${env.JOB_NAME}", selector: lastSuccessful(), filter: "${SFT_TEST_RESULTS_PREFIX}*/${SFT_BASELINE_RESULTS_FILE}"); +// sh "cp ./${SFT_TEST_RESULTS_PREFIX}*/${SFT_BASELINE_RESULTS_FILE} ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_PREVIOUS}${SFT_BASELINE_RESULTS_FILE}" +// } +// catch (Exception e) { +// echo 'No rpms to archive' +// } +// } + sh "cp ./${SFT_BASELINE_DIR}/${SFT_BASELINE_RESULTS_FILE} ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_EXPECTED}${SFT_BASELINE_RESULTS_FILE}" + sh "${SFT_BIN}/sft_parse_test_results.pm -b ${SFT_TEST_RESULTS_EXPECTED}${SFT_BASELINE_RESULTS_FILE} -d ${SFT_TEST_RESULTS_DIR} -o ${SFT_TEST_RESULTS} -r ${SFT_BASELINE_RESULTS_FILE}" + sh "gzip -r ${SFT_TEST_RESULTS_DIR}" + sh "gunzip ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS}" + sh "gunzip ${SFT_TEST_RESULTS_DIR}/${SFT_BASELINE_RESULTS_FILE}" + sh "gunzip ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_EXPECTED}${SFT_BASELINE_RESULTS_FILE}" +// sh "gunzip ${SFT_TEST_RESULTS_DIR}/${SFT_TEST_RESULTS_PREVIOUS}${SFT_BASELINE_RESULTS_FILE}" + sh "cp -r ${SFT_TEST_RESULTS_DIR} ." + archiveArtifacts artifacts: "${SFT_TEST_RESULTS_SUBDIR}/*" + step ([$class: 'XUnitBuilder', + thresholds: [ + [$class: 'FailedThreshold', unstableThreshold: '0']], + tools: [[$class: 'JUnitType', pattern: "${SFT_TEST_RESULTS_SUBDIR}/${SFT_TEST_RESULTS}"]]]) + sh "rm -rf ${SFT_TEST_RESULTS_DIR}" + } + } + } stage("Deploy: Install") { steps { sh "make clean" diff --git a/contrib/cray/sft_test_results_baseline.txt b/contrib/cray/sft_test_results_baseline.txt new file mode 100644 index 00000000000..f5ea3bf016a --- /dev/null +++ b/contrib/cray/sft_test_results_baseline.txt @@ -0,0 +1,113 @@ +# Expected test values can be: 'PASS', 'FAIL', SKIP' or 'INTERMITTENT' +# PASS: means that the test should pass +# if the test result is PASS, then Jenkins will report this test as PASSED +# if the test result is FAIL, then Jenkins will report this test as FAILED +# FAIL: means that the test currently fails, probably due to incorrect behavior of a feature +# if the test result is PASS, then Jenkins will report this test as PASSED +# if the test result is FAIL, then Jenkins will report this test as SKIPPED +# SKIP: means that the test should be skipped, due to an unimplemented feature +# if the test result is PASS, then Jenkins will report this test as PASSED +# if the test result is FAIL, then Jenkins will report this test as SKIPPED +# INTERMITTENT: means that the test should pass but it fails infrequently +# if the test result is PASS, then Jenkins will report this test as PASSED +# if the test result is FAIL, then Jenkins will report this test as SKIPPED + +MESSAGE_All-to-All_FI_INJECT=PASS +MESSAGE_All-to-All_FI_INJECTDATA=PASS +MESSAGE_All-to-All_FI_SEND=PASS +MESSAGE_All-to-All_FI_SENDDATA=PASS +MESSAGE_All-to-All_FI_SENDMSG=PASS +MESSAGE_All-to-All_FI_SENDV=PASS +MESSAGE_All-to-One_FI_INJECT=PASS +MESSAGE_All-to-One_FI_INJECTDATA=PASS +MESSAGE_All-to-One_FI_SEND=PASS +MESSAGE_All-to-One_FI_SENDDATA=PASS +MESSAGE_All-to-One_FI_SENDMSG=PASS +MESSAGE_All-to-One_FI_SENDV=PASS +MESSAGE_One-to-All_FI_INJECT=PASS +MESSAGE_One-to-All_FI_INJECTDATA=PASS +MESSAGE_One-to-All_FI_SEND=PASS +MESSAGE_One-to-All_FI_SENDDATA=PASS +MESSAGE_One-to-All_FI_SENDMSG=PASS +MESSAGE_One-to-All_FI_SENDV=PASS +MESSAGE_Round-Robin_FI_INJECT=PASS +MESSAGE_Round-Robin_FI_INJECTDATA=PASS +MESSAGE_Round-Robin_FI_SEND=PASS +MESSAGE_Round-Robin_FI_SENDDATA=PASS +MESSAGE_Round-Robin_FI_SENDDATA_TRIGGER=SKIP +MESSAGE_Round-Robin_FI_SENDMSG=PASS +MESSAGE_Round-Robin_FI_SENDMSG_TRIGGER=SKIP +MESSAGE_Round-Robin_FI_SENDV=PASS +MESSAGE_Round-Robin_FI_SENDV_TRIGGER=SKIP +MESSAGE_Round-Robin_FI_SEND_TRIGGER=SKIP +RMA_All-to-All_FI_INJECT_WRITE=PASS +RMA_All-to-All_FI_INJECT_WRITEDATA=PASS +RMA_All-to-All_FI_READ=FAIL +RMA_All-to-All_FI_READMSG=FAIL +RMA_All-to-All_FI_READV=FAIL +RMA_All-to-All_FI_WRITE=PASS +RMA_All-to-All_FI_WRITEDATA=PASS +RMA_All-to-All_FI_WRITEMSG=PASS +RMA_All-to-All_FI_WRITEV=PASS +RMA_All-to-One_FI_INJECT_WRITE=PASS +RMA_All-to-One_FI_INJECT_WRITEDATA=PASS +RMA_All-to-One_FI_READ=FAIL +RMA_All-to-One_FI_READMSG=FAIL +RMA_All-to-One_FI_READV=FAIL +RMA_All-to-One_FI_WRITE=PASS +RMA_All-to-One_FI_WRITEDATA=PASS +RMA_All-to-One_FI_WRITEMSG=PASS +RMA_All-to-One_FI_WRITEV=PASS +RMA_One-to-All_FI_INJECT_WRITE=PASS +RMA_One-to-All_FI_INJECT_WRITEDATA=PASS +RMA_One-to-All_FI_READ=FAIL +RMA_One-to-All_FI_READMSG=FAIL +RMA_One-to-All_FI_READV=FAIL +RMA_One-to-All_FI_WRITE=PASS +RMA_One-to-All_FI_WRITEDATA=PASS +RMA_One-to-All_FI_WRITEMSG=PASS +RMA_One-to-All_FI_WRITEV=PASS +RMA_Round-Robin_FI_INJECT_WRITE=PASS +RMA_Round-Robin_FI_INJECT_WRITEDATA=PASS +RMA_Round-Robin_FI_READ=FAIL +RMA_Round-Robin_FI_READMSG=FAIL +RMA_Round-Robin_FI_READMSG_TRIGGER=SKIP +RMA_Round-Robin_FI_READV=FAIL +RMA_Round-Robin_FI_READV_TRIGGER=SKIP +RMA_Round-Robin_FI_READ_TRIGGER=SKIP +RMA_Round-Robin_FI_WRITE=PASS +RMA_Round-Robin_FI_WRITEDATA=PASS +RMA_Round-Robin_FI_WRITEDATA_TRIGGER=SKIP +RMA_Round-Robin_FI_WRITEMSG=PASS +RMA_Round-Robin_FI_WRITEMSG_TRIGGER=SKIP +RMA_Round-Robin_FI_WRITEV=PASS +RMA_Round-Robin_FI_WRITEV_TRIGGER=SKIP +RMA_Round-Robin_FI_WRITE_TRIGGER=SKIP +TAGGED_All-to-All_FI_TINJECT=PASS +TAGGED_All-to-All_FI_TINJECTDATA=PASS +TAGGED_All-to-All_FI_TSEND=PASS +TAGGED_All-to-All_FI_TSENDDATA=PASS +TAGGED_All-to-All_FI_TSENDMSG=PASS +TAGGED_All-to-All_FI_TSENDV=PASS +TAGGED_All-to-One_FI_TINJECT=PASS +TAGGED_All-to-One_FI_TINJECTDATA=PASS +TAGGED_All-to-One_FI_TSEND=PASS +TAGGED_All-to-One_FI_TSENDDATA=PASS +TAGGED_All-to-One_FI_TSENDMSG=PASS +TAGGED_All-to-One_FI_TSENDV=PASS +TAGGED_One-to-All_FI_TINJECT=PASS +TAGGED_One-to-All_FI_TINJECTDATA=PASS +TAGGED_One-to-All_FI_TSEND=PASS +TAGGED_One-to-All_FI_TSENDDATA=PASS +TAGGED_One-to-All_FI_TSENDMSG=PASS +TAGGED_One-to-All_FI_TSENDV=PASS +TAGGED_Round-Robin_FI_TINJECT=PASS +TAGGED_Round-Robin_FI_TINJECTDATA=PASS +TAGGED_Round-Robin_FI_TSEND=PASS +TAGGED_Round-Robin_FI_TSENDDATA=PASS +TAGGED_Round-Robin_FI_TSENDDATA_TRIGGER=SKIP +TAGGED_Round-Robin_FI_TSENDMSG=PASS +TAGGED_Round-Robin_FI_TSENDMSG_TRIGGER=SKIP +TAGGED_Round-Robin_FI_TSENDV=PASS +TAGGED_Round-Robin_FI_TSENDV_TRIGGER=SKIP +TAGGED_Round-Robin_FI_TSEND_TRIGGER=SKIP