From d71b8740147946ec06c353836daac9934a0c732b Mon Sep 17 00:00:00 2001 From: XieYunshen <1084314248@qq.com> Date: Thu, 18 Mar 2021 20:08:36 +0800 Subject: [PATCH 01/10] use busybox run test on windows openblas --- paddle/scripts/paddle_build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index 07de8ff6c2f7e3..26a3e29b9f123f 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -502,7 +502,7 @@ echo ======================================== echo Running CPU unit tests in parallel way ... echo ======================================== -ctest.exe -E "(%disable_ut_quickly%)" -LE %nightly_label% --output-on-failure -C Release -j 8 --repeat until-pass:4 after-timeout:4 +%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\run_unittests_cpu.sh %NIGHTLY_MODE% goto:eof From 98ef7c4d3cea528f444769bc04996c56befecd44 Mon Sep 17 00:00:00 2001 From: XieYunshen <1084314248@qq.com> Date: Thu, 18 Mar 2021 20:57:53 +0800 Subject: [PATCH 02/10] fix error --- tools/windows/run_unittests_cpu.sh | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 tools/windows/run_unittests_cpu.sh diff --git a/tools/windows/run_unittests_cpu.sh b/tools/windows/run_unittests_cpu.sh new file mode 100644 index 00000000000000..41d942bbfb7011 --- /dev/null +++ b/tools/windows/run_unittests_cpu.sh @@ -0,0 +1,38 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set +x +NIGHTLY_MODE=$1 + +export PADDLE_ROOT="$(cd "$PWD/../" && pwd )" + +if [ ${NIGHTLY_MODE:-OFF} == "ON" ]; then + nightly_label="" +else + nightly_label="(RUN_TYPE=NIGHTLY|RUN_TYPE=DIST:NIGHTLY|RUN_TYPE=EXCLUSIVE:NIGHTLY)" + echo "=========================================" + echo "Unittests with nightly labels are only run at night" + echo "=========================================" +fi +if disable_ut_quickly=$(python ${PADDLE_ROOT}/tools/get_quick_disable_lt.py); then + echo "=========================================" + echo "The following unittests have been disabled:" + echo ${disable_ut_quickly} + echo "=========================================" +else + disable_ut_quickly='' +fi + +ctest -E "(%disable_ut_quickly%)" -LE %nightly_label% --output-on-failure -C Release -j 8 --repeat until-pass:4 after-timeout:4 From 5837c441e6c9d3950198a3e4af9df62a27c79a44 Mon Sep 17 00:00:00 2001 From: XieYunshen <1084314248@qq.com> Date: Fri, 19 Mar 2021 10:31:04 +0800 Subject: [PATCH 03/10] fix disable_quick and nightly lable issue --- tools/windows/run_unittests_cpu.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/windows/run_unittests_cpu.sh b/tools/windows/run_unittests_cpu.sh index 41d942bbfb7011..39ad1838904c25 100644 --- a/tools/windows/run_unittests_cpu.sh +++ b/tools/windows/run_unittests_cpu.sh @@ -32,7 +32,7 @@ if disable_ut_quickly=$(python ${PADDLE_ROOT}/tools/get_quick_disable_lt.py); th echo ${disable_ut_quickly} echo "=========================================" else - disable_ut_quickly='' + disable_ut_quickly='disable_ut' fi -ctest -E "(%disable_ut_quickly%)" -LE %nightly_label% --output-on-failure -C Release -j 8 --repeat until-pass:4 after-timeout:4 +ctest -E "${disable_ut_quickly}" -LE "${nightly_label}" --output-on-failure -C Release -j 8 --repeat until-pass:4 after-timeout:4 From ea5668b725e0d00ae89365f9f7016423d3b25f02 Mon Sep 17 00:00:00 2001 From: XieYunshen <1084314248@qq.com> Date: Mon, 22 Mar 2021 14:46:44 +0800 Subject: [PATCH 04/10] add retry on windows openblas --- tools/windows/run_unittests_cpu.sh | 109 ++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/tools/windows/run_unittests_cpu.sh b/tools/windows/run_unittests_cpu.sh index 39ad1838904c25..aed5ede3209a50 100644 --- a/tools/windows/run_unittests_cpu.sh +++ b/tools/windows/run_unittests_cpu.sh @@ -35,4 +35,111 @@ else disable_ut_quickly='disable_ut' fi -ctest -E "${disable_ut_quickly}" -LE "${nightly_label}" --output-on-failure -C Release -j 8 --repeat until-pass:4 after-timeout:4 +failed_test_lists='' +tmp_dir=`mktemp -d` +function collect_failed_tests() { + set +e + for file in `ls $tmp_dir`; do + grep -q 'The following tests FAILED:' $tmp_dir/$file + exit_code=$? + if [ $exit_code -ne 0 ]; then + failuretest='' + else + failuretest=`grep -A 10000 'The following tests FAILED:' $tmp_dir/$file | sed 's/The following tests FAILED://g'|sed '/^$/d'` + failed_test_lists="${failed_test_lists} + ${failuretest}" + fi + done + set -e +} + +function unittests_retry(){ + parallel_job=4 + is_retry_execuate=0 + wintest_error=1 + retry_time=3 + exec_times=0 + exec_retry_threshold=10 + retry_unittests=$(echo "${failed_test_lists}" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) + need_retry_ut_counts=$(echo "$retry_unittests" |awk -F ' ' '{print }'| sed '/^$/d' | wc -l) + retry_unittests_regular=$(echo "$retry_unittests" |awk -F ' ' '{print }' | awk 'BEGIN{ all_str=""}{if (all_str==""){all_str=$1}else{all_str=all_str"$|^"$1}} END{print "^"all_str"$"}') + tmpfile=$tmp_dir/$RANDOM + + if [ $need_retry_ut_counts -lt $exec_retry_threshold ];then + retry_unittests_record='' + while ( [ $exec_times -lt $retry_time ] ) + do + retry_unittests_record="$retry_unittests_record$failed_test_lists" + if ( [[ "$exec_times" == "0" ]] );then + cur_order='first' + elif ( [[ "$exec_times" == "1" ]] );then + cur_order='second' + if [[ "$failed_test_lists" == "" ]]; then + break + else + retry_unittests=$(echo "${failed_test_lists}" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) + retry_unittests_regular=$(echo "$retry_unittests" |awk -F ' ' '{print }' | awk 'BEGIN{ all_str=""}{if (all_str==""){all_str=$1}else{all_str=all_str"$|^"$1}} END{print "^"all_str"$"}') + fi + elif ( [[ "$exec_times" == "2" ]] );then + cur_order='third' + fi + echo "=========================================" + echo "This is the ${cur_order} time to re-run" + echo "=========================================" + echo "The following unittest will be re-run:" + echo "${retry_unittests}" + echo "=========================================" + rm -f $tmp_dir/* + failed_test_lists='' + (ctest -R "($retry_unittests_regular)" --output-on-failure -C Release -j $parallel_job| tee $tmpfile ) & + wait; + collect_failed_tests + exec_times=$(echo $exec_times | awk '{print $0+1}') + done + else + # There are more than 10 failed unit tests, so no unit test retry + is_retry_execuate=1 + fi + rm -f $tmp_dir/* +} + +function show_ut_retry_result() { + if [[ "$is_retry_execuate" != "0" ]];then + failed_test_lists_ult=`echo "${failed_test_lists}" | grep -Po '[^ ].*$'` + echo "=========================================" + echo "There are more than 10 failed unit tests, so no unit test retry!!!" + echo "=========================================" + echo "${failed_test_lists_ult}" + exit 8; + else + retry_unittests_ut_name=$(echo "$retry_unittests_record" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) + retry_unittests_record_judge=$(echo ${retry_unittests_ut_name}| tr ' ' '\n' | sort | uniq -c | awk '{if ($1 >=3) {print $2}}') + if [ -z "${retry_unittests_record_judge}" ];then + echo "========================================" + echo "There are failed tests, which have been successful after re-run:" + echo "========================================" + echo "The following tests have been re-ran:" + echo "${retry_unittests_record}" + else + failed_ut_re=$(echo "${retry_unittests_record_judge}" | awk 'BEGIN{ all_str=""}{if (all_str==""){all_str=$1}else{all_str=all_str"|"$1}} END{print all_str}') + echo "========================================" + echo "There are failed tests, which have been executed re-run,but success rate is less than 50%:" + echo "Summary Failed Tests... " + echo "========================================" + echo "The following tests FAILED: " + echo "${retry_unittests_record}" | grep -E "$failed_ut_re" + exit 8; + fi + fi +} + +set +e +(ctest -E "${disable_ut_quickly}" -LE "${nightly_label}" --output-on-failure -C Release -j 8 | tee $tmpfile) & +wait +collect_failed_tests +set -e +rm -f $tmp_dir/* +if [[ "$failed_test_lists" != "" ]]; then + unittests_retry + show_ut_retry_result +fi From e9dfd145eeebdc89b248ed0597ac8e0badf3203f Mon Sep 17 00:00:00 2001 From: XieYunshen <1084314248@qq.com> Date: Mon, 22 Mar 2021 18:42:11 +0800 Subject: [PATCH 05/10] fix bug --- tools/windows/run_unittests_cpu.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/windows/run_unittests_cpu.sh b/tools/windows/run_unittests_cpu.sh index aed5ede3209a50..5ebbbfc39583d3 100644 --- a/tools/windows/run_unittests_cpu.sh +++ b/tools/windows/run_unittests_cpu.sh @@ -134,6 +134,7 @@ function show_ut_retry_result() { } set +e +tmpfile=$tmp_dir/$RANDOM (ctest -E "${disable_ut_quickly}" -LE "${nightly_label}" --output-on-failure -C Release -j 8 | tee $tmpfile) & wait collect_failed_tests From 6a4bf2ada0a09327c3eafadbc168f3b5a0cc15ea Mon Sep 17 00:00:00 2001 From: XieYunshen <1084314248@qq.com> Date: Tue, 23 Mar 2021 16:46:04 +0800 Subject: [PATCH 06/10] use one file to run cpu and gpu tests --- paddle/scripts/paddle_build.bat | 6 +- tools/windows/run_unittests.sh | 101 +++++++++++--------- tools/windows/run_unittests_cpu.sh | 146 ----------------------------- 3 files changed, 63 insertions(+), 190 deletions(-) delete mode 100644 tools/windows/run_unittests_cpu.sh diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index 26a3e29b9f123f..18143fd033e8b0 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -51,6 +51,8 @@ if not defined WITH_CACHE set WITH_CACHE=OFF if not defined WITH_UNITY_BUILD set WITH_UNITY_BUILD=OFF if not defined INFERENCE_DEMO_INSTALL_DIR set INFERENCE_DEMO_INSTALL_DIR=%cache_dir:\=/%/inference_demo if not defined LOG_LEVEL set LOG_LEVEL=normal +if not defined PRECISION_TEST set PRECISION_TEST=OFF +if not defined NIGHTLY_MODE set PRECISION_TEST=OFF rem -------set cache build directory----------- rmdir build\python /s/q @@ -493,7 +495,7 @@ setlocal enabledelayedexpansion :: for /F %%# in ('cmd /C nvidia-smi -L ^|find "GPU" /C') do set CUDA_DEVICE_COUNT=%%# set CUDA_DEVICE_COUNT=1 -%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\run_unittests.sh %NIGHTLY_MODE% %PRECISION_TEST% +%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\run_unittests.sh %NIGHTLY_MODE% %PRECISION_TEST% %WITH_GPU% goto:eof @@ -502,7 +504,7 @@ echo ======================================== echo Running CPU unit tests in parallel way ... echo ======================================== -%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\run_unittests_cpu.sh %NIGHTLY_MODE% +%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\run_unittests.sh %NIGHTLY_MODE% %PRECISION_TEST% %WITH_GPU% goto:eof diff --git a/tools/windows/run_unittests.sh b/tools/windows/run_unittests.sh index dd4b21c80d910b..749deff7345681 100644 --- a/tools/windows/run_unittests.sh +++ b/tools/windows/run_unittests.sh @@ -16,6 +16,7 @@ set -e set +x NIGHTLY_MODE=$1 PRECISION_TEST=$2 +WITH_GPU=$3 export PADDLE_ROOT="$(cd "$PWD/../" && pwd )" if [ ${NIGHTLY_MODE:-OFF} == "ON" ]; then @@ -204,50 +205,52 @@ long_time_test="^best_fit_allocator_test$|\ ^test_strided_slice_op$|\ ^test_transpose_op$" -export FLAGS_call_stack_level=2 -export FLAGS_fraction_of_gpu_memory_to_use=0.92 -export CUDA_VISIBLE_DEVICES=0 +if [ ${WITH_GPU:-OFF} == "ON" ];then + export FLAGS_call_stack_level=2 + export FLAGS_fraction_of_gpu_memory_to_use=0.92 + export CUDA_VISIBLE_DEVICES=0 -UT_list=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d') -num=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d' | wc -l) -echo "Windows 1 card TestCases count is $num" -if [ ${PRECISION_TEST:-OFF} == "ON" ]; then - python ${PADDLE_ROOT}/tools/get_pr_ut.py - if [[ -f "ut_list" ]]; then - set +x - echo "PREC length: "`wc -l ut_list` - precision_cases=`cat ut_list` - set -x + UT_list=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d') + num=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d' | wc -l) + echo "Windows 1 card TestCases count is $num" + if [ ${PRECISION_TEST:-OFF} == "ON" ]; then + python ${PADDLE_ROOT}/tools/get_pr_ut.py + if [[ -f "ut_list" ]]; then + set +x + echo "PREC length: "`wc -l ut_list` + precision_cases=`cat ut_list` + set -x + fi fi -fi -set +e -if [ ${PRECISION_TEST:-OFF} == "ON" ] && [[ "$precision_cases" != "" ]];then - UT_list_prec='' - re=$(cat ut_list|awk -F ' ' '{print }' | awk 'BEGIN{ all_str=""}{if (all_str==""){all_str=$1}else{all_str=all_str"$|^"$1}} END{print "^"all_str"$"}') - for case in $UT_list; do - flag=$(echo $case|grep -oE $re) - if [ -n "$flag" ];then - if [ -z "$UT_list_prec" ];then - UT_list_prec=$case + set +e + if [ ${PRECISION_TEST:-OFF} == "ON" ] && [[ "$precision_cases" != "" ]];then + UT_list_prec='' + re=$(cat ut_list|awk -F ' ' '{print }' | awk 'BEGIN{ all_str=""}{if (all_str==""){all_str=$1}else{all_str=all_str"$|^"$1}} END{print "^"all_str"$"}') + for case in $UT_list; do + flag=$(echo $case|grep -oE $re) + if [ -n "$flag" ];then + if [ -z "$UT_list_prec" ];then + UT_list_prec=$case + else + UT_list_prec=$UT_list_prec'\n'$case + fi else - UT_list_prec=$UT_list_prec'\n'$case + echo $case "won't run in PRECISION_TEST mode." fi - else - echo $case "won't run in PRECISION_TEST mode." - fi - done - UT_list=$UT_list_prec -fi -set -e + done + UT_list=$UT_list_prec + fi + set -e -output=$(python ${PADDLE_ROOT}/tools/parallel_UT_rule.py "${UT_list}") -eight_parallel_job=$(echo $output | cut -d ";" -f 1) -tetrad_parallel_jog=$(echo $output | cut -d ";" -f 2) -non_parallel_job=$(echo $output | cut -d ";" -f 3) + output=$(python ${PADDLE_ROOT}/tools/parallel_UT_rule.py "${UT_list}") + eight_parallel_job=$(echo $output | cut -d ";" -f 1) + tetrad_parallel_jog=$(echo $output | cut -d ";" -f 2) + non_parallel_job=$(echo $output | cut -d ";" -f 3) -non_parallel_job_1=$(echo $non_parallel_job | cut -d "," -f 1) -non_parallel_job_2=$(echo $non_parallel_job | cut -d "," -f 2) + non_parallel_job_1=$(echo $non_parallel_job | cut -d "," -f 1) + non_parallel_job_2=$(echo $non_parallel_job | cut -d "," -f 2) +fi failed_test_lists='' tmp_dir=`mktemp -d` @@ -267,6 +270,12 @@ function collect_failed_tests() { set -e } +function run_unittest_cpu() { + tmpfile=$tmp_dir/$RANDOM + (ctest -E "${disable_ut_quickly}" -LE "${nightly_label}" --output-on-failure -C Release -j 8 | tee $tmpfile) & + wait; +} + function run_unittest() { test_case=$1 parallel_job=$2 @@ -285,7 +294,11 @@ function run_unittest() { } function unittests_retry(){ - parallel_job=1 + if [ "${WITH_GPU:-OFF}" == "ON" ];then + parallel_job=1 + else + parallel_job=4 + fi is_retry_execuate=0 wintest_error=1 retry_time=3 @@ -365,10 +378,14 @@ function show_ut_retry_result() { } set +e -run_unittest $eight_parallel_job 8 -run_unittest $tetrad_parallel_jog 4 -run_unittest $non_parallel_job_1 -run_unittest $non_parallel_job_2 +if [ "${WITH_GPU:-OFF}" == "ON" ];then + run_unittest $eight_parallel_job 8 + run_unittest $tetrad_parallel_jog 4 + run_unittest $non_parallel_job_1 + run_unittest $non_parallel_job_2 +else + run_unittest_cpu +fi collect_failed_tests set -e rm -f $tmp_dir/* diff --git a/tools/windows/run_unittests_cpu.sh b/tools/windows/run_unittests_cpu.sh deleted file mode 100644 index 5ebbbfc39583d3..00000000000000 --- a/tools/windows/run_unittests_cpu.sh +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e -set +x -NIGHTLY_MODE=$1 - -export PADDLE_ROOT="$(cd "$PWD/../" && pwd )" - -if [ ${NIGHTLY_MODE:-OFF} == "ON" ]; then - nightly_label="" -else - nightly_label="(RUN_TYPE=NIGHTLY|RUN_TYPE=DIST:NIGHTLY|RUN_TYPE=EXCLUSIVE:NIGHTLY)" - echo "=========================================" - echo "Unittests with nightly labels are only run at night" - echo "=========================================" -fi -if disable_ut_quickly=$(python ${PADDLE_ROOT}/tools/get_quick_disable_lt.py); then - echo "=========================================" - echo "The following unittests have been disabled:" - echo ${disable_ut_quickly} - echo "=========================================" -else - disable_ut_quickly='disable_ut' -fi - -failed_test_lists='' -tmp_dir=`mktemp -d` -function collect_failed_tests() { - set +e - for file in `ls $tmp_dir`; do - grep -q 'The following tests FAILED:' $tmp_dir/$file - exit_code=$? - if [ $exit_code -ne 0 ]; then - failuretest='' - else - failuretest=`grep -A 10000 'The following tests FAILED:' $tmp_dir/$file | sed 's/The following tests FAILED://g'|sed '/^$/d'` - failed_test_lists="${failed_test_lists} - ${failuretest}" - fi - done - set -e -} - -function unittests_retry(){ - parallel_job=4 - is_retry_execuate=0 - wintest_error=1 - retry_time=3 - exec_times=0 - exec_retry_threshold=10 - retry_unittests=$(echo "${failed_test_lists}" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) - need_retry_ut_counts=$(echo "$retry_unittests" |awk -F ' ' '{print }'| sed '/^$/d' | wc -l) - retry_unittests_regular=$(echo "$retry_unittests" |awk -F ' ' '{print }' | awk 'BEGIN{ all_str=""}{if (all_str==""){all_str=$1}else{all_str=all_str"$|^"$1}} END{print "^"all_str"$"}') - tmpfile=$tmp_dir/$RANDOM - - if [ $need_retry_ut_counts -lt $exec_retry_threshold ];then - retry_unittests_record='' - while ( [ $exec_times -lt $retry_time ] ) - do - retry_unittests_record="$retry_unittests_record$failed_test_lists" - if ( [[ "$exec_times" == "0" ]] );then - cur_order='first' - elif ( [[ "$exec_times" == "1" ]] );then - cur_order='second' - if [[ "$failed_test_lists" == "" ]]; then - break - else - retry_unittests=$(echo "${failed_test_lists}" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) - retry_unittests_regular=$(echo "$retry_unittests" |awk -F ' ' '{print }' | awk 'BEGIN{ all_str=""}{if (all_str==""){all_str=$1}else{all_str=all_str"$|^"$1}} END{print "^"all_str"$"}') - fi - elif ( [[ "$exec_times" == "2" ]] );then - cur_order='third' - fi - echo "=========================================" - echo "This is the ${cur_order} time to re-run" - echo "=========================================" - echo "The following unittest will be re-run:" - echo "${retry_unittests}" - echo "=========================================" - rm -f $tmp_dir/* - failed_test_lists='' - (ctest -R "($retry_unittests_regular)" --output-on-failure -C Release -j $parallel_job| tee $tmpfile ) & - wait; - collect_failed_tests - exec_times=$(echo $exec_times | awk '{print $0+1}') - done - else - # There are more than 10 failed unit tests, so no unit test retry - is_retry_execuate=1 - fi - rm -f $tmp_dir/* -} - -function show_ut_retry_result() { - if [[ "$is_retry_execuate" != "0" ]];then - failed_test_lists_ult=`echo "${failed_test_lists}" | grep -Po '[^ ].*$'` - echo "=========================================" - echo "There are more than 10 failed unit tests, so no unit test retry!!!" - echo "=========================================" - echo "${failed_test_lists_ult}" - exit 8; - else - retry_unittests_ut_name=$(echo "$retry_unittests_record" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) - retry_unittests_record_judge=$(echo ${retry_unittests_ut_name}| tr ' ' '\n' | sort | uniq -c | awk '{if ($1 >=3) {print $2}}') - if [ -z "${retry_unittests_record_judge}" ];then - echo "========================================" - echo "There are failed tests, which have been successful after re-run:" - echo "========================================" - echo "The following tests have been re-ran:" - echo "${retry_unittests_record}" - else - failed_ut_re=$(echo "${retry_unittests_record_judge}" | awk 'BEGIN{ all_str=""}{if (all_str==""){all_str=$1}else{all_str=all_str"|"$1}} END{print all_str}') - echo "========================================" - echo "There are failed tests, which have been executed re-run,but success rate is less than 50%:" - echo "Summary Failed Tests... " - echo "========================================" - echo "The following tests FAILED: " - echo "${retry_unittests_record}" | grep -E "$failed_ut_re" - exit 8; - fi - fi -} - -set +e -tmpfile=$tmp_dir/$RANDOM -(ctest -E "${disable_ut_quickly}" -LE "${nightly_label}" --output-on-failure -C Release -j 8 | tee $tmpfile) & -wait -collect_failed_tests -set -e -rm -f $tmp_dir/* -if [[ "$failed_test_lists" != "" ]]; then - unittests_retry - show_ut_retry_result -fi From 257efedb0a2401845751dc426e4c17744d3835b0 Mon Sep 17 00:00:00 2001 From: XieYunshen <1084314248@qq.com> Date: Thu, 25 Mar 2021 13:15:52 +0800 Subject: [PATCH 07/10] fix with grep warning --- tools/windows/run_unittests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/windows/run_unittests.sh b/tools/windows/run_unittests.sh index 749deff7345681..ad585b66eef4f7 100644 --- a/tools/windows/run_unittests.sh +++ b/tools/windows/run_unittests.sh @@ -349,7 +349,7 @@ function unittests_retry(){ function show_ut_retry_result() { if [[ "$is_retry_execuate" != "0" ]];then - failed_test_lists_ult=`echo "${failed_test_lists}" | grep -Po '[^ ].*$'` + failed_test_lists_ult=`echo "${failed_test_lists}" echo "=========================================" echo "There are more than 10 failed unit tests, so no unit test retry!!!" echo "=========================================" From aad8937bc38d9536ef870a2292c4d037c1ec8cef Mon Sep 17 00:00:00 2001 From: XieYunshen <1084314248@qq.com> Date: Thu, 25 Mar 2021 13:38:51 +0800 Subject: [PATCH 08/10] fix syntax error --- tools/windows/run_unittests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/windows/run_unittests.sh b/tools/windows/run_unittests.sh index ad585b66eef4f7..6ad64d8696cba7 100644 --- a/tools/windows/run_unittests.sh +++ b/tools/windows/run_unittests.sh @@ -349,7 +349,7 @@ function unittests_retry(){ function show_ut_retry_result() { if [[ "$is_retry_execuate" != "0" ]];then - failed_test_lists_ult=`echo "${failed_test_lists}" + failed_test_lists_ult=`echo "${failed_test_lists}"` echo "=========================================" echo "There are more than 10 failed unit tests, so no unit test retry!!!" echo "=========================================" From ea1bee0b064ca6f48549be408f57e1c15ba710a5 Mon Sep 17 00:00:00 2001 From: XieYunshen <1084314248@qq.com> Date: Mon, 29 Mar 2021 20:01:35 +0800 Subject: [PATCH 09/10] change run_unittest to run_unittest_gpu --- tools/windows/run_unittests.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/windows/run_unittests.sh b/tools/windows/run_unittests.sh index 6ad64d8696cba7..8f83eee5cdbaf4 100644 --- a/tools/windows/run_unittests.sh +++ b/tools/windows/run_unittests.sh @@ -276,7 +276,7 @@ function run_unittest_cpu() { wait; } -function run_unittest() { +function run_unittest_gpu() { test_case=$1 parallel_job=$2 if [ "$2" == "" ]; then @@ -379,10 +379,10 @@ function show_ut_retry_result() { set +e if [ "${WITH_GPU:-OFF}" == "ON" ];then - run_unittest $eight_parallel_job 8 - run_unittest $tetrad_parallel_jog 4 - run_unittest $non_parallel_job_1 - run_unittest $non_parallel_job_2 + run_unittest_gpu $eight_parallel_job 8 + run_unittest_gpu $tetrad_parallel_jog 4 + run_unittest_gpu $non_parallel_job_1 + run_unittest_gpu $non_parallel_job_2 else run_unittest_cpu fi From 11a5e53e5238b8673b3601093774c70778b97e41 Mon Sep 17 00:00:00 2001 From: YUNSHEN XIE <1084314248@qq.com> Date: Thu, 1 Apr 2021 16:08:00 +0800 Subject: [PATCH 10/10] Update run_unittests.sh fix error --- tools/windows/run_unittests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/windows/run_unittests.sh b/tools/windows/run_unittests.sh index a227fe6a11376c..6da2401fbe4608 100644 --- a/tools/windows/run_unittests.sh +++ b/tools/windows/run_unittests.sh @@ -379,10 +379,10 @@ function show_ut_retry_result() { set +e if [ "${WITH_GPU:-OFF}" == "ON" ];then - run_unittest_gpu $eight_parallel_job 8 - run_unittest_gpu $tetrad_parallel_jog 4 - run_unittest_gpu $non_parallel_job_1 - run_unittest_gpu $non_parallel_job_2 + run_unittest_gpu $cpu_parallel_job 12 + run_unittest_gpu $tetrad_parallel_job 4 + run_unittest_gpu $two_parallel_job 2 + run_unittest_gpu $non_parallel_job else run_unittest_cpu fi