Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use busybox run test on windows openblas #31728

Merged
merged 11 commits into from
Apr 2, 2021
Merged

use busybox run test on windows openblas #31728

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
d71b874
use busybox run test on windows openblas
XieYunshen Mar 18, 2021
98ef7c4
fix error
XieYunshen Mar 18, 2021
5837c44
fix disable_quick and nightly lable issue
XieYunshen Mar 19, 2021
ea5668b
add retry on windows openblas
XieYunshen Mar 22, 2021
e9dfd14
fix bug
XieYunshen Mar 22, 2021
6a4bf2a
use one file to run cpu and gpu tests
XieYunshen Mar 23, 2021
257efed
fix with grep warning
XieYunshen Mar 25, 2021
aad8937
fix syntax error
XieYunshen Mar 25, 2021
ea1bee0
change run_unittest to run_unittest_gpu
XieYunshen Mar 29, 2021
e15b450
Merge branch 'develop' into retry_on_win_openblas
XieYunshen Apr 1, 2021
11a5e53
Update run_unittests.sh
XieYunshen Apr 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions paddle/scripts/paddle_build.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ if not defined WITH_CACHE set WITH_CACHE=OFF
if not defined WITH_UNITY_BUILD set WITH_UNITY_BUILD=OFF
if not defined INFERENCE_DEMO_INSTALL_DIR set INFERENCE_DEMO_INSTALL_DIR=%cache_dir:\=/%/inference_demo
if not defined LOG_LEVEL set LOG_LEVEL=normal
if not defined PRECISION_TEST set PRECISION_TEST=OFF
if not defined NIGHTLY_MODE set PRECISION_TEST=OFF

rem -------set cache build directory-----------
rmdir build\python /s/q
Expand Down Expand Up @@ -493,7 +495,7 @@ setlocal enabledelayedexpansion
:: for /F %%# in ('cmd /C nvidia-smi -L ^|find "GPU" /C') do set CUDA_DEVICE_COUNT=%%#
set CUDA_DEVICE_COUNT=1

%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\run_unittests.sh %NIGHTLY_MODE% %PRECISION_TEST%
%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\run_unittests.sh %NIGHTLY_MODE% %PRECISION_TEST% %WITH_GPU%

goto:eof

Expand All @@ -502,7 +504,7 @@ echo ========================================
echo Running CPU unit tests in parallel way ...
echo ========================================

ctest.exe -E "(%disable_ut_quickly%)" -LE %nightly_label% --output-on-failure -C Release -j 8 --repeat until-pass:4 after-timeout:4
%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\run_unittests.sh %NIGHTLY_MODE% %PRECISION_TEST% %WITH_GPU%

goto:eof

Expand Down
105 changes: 61 additions & 44 deletions tools/windows/run_unittests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ set -e
set +x
NIGHTLY_MODE=$1
PRECISION_TEST=$2
WITH_GPU=$3

export PADDLE_ROOT="$(cd "$PWD/../" && pwd )"
if [ ${NIGHTLY_MODE:-OFF} == "ON" ]; then
Expand Down Expand Up @@ -204,50 +205,52 @@ long_time_test="^best_fit_allocator_test$|\
^test_strided_slice_op$|\
^test_transpose_op$"

export FLAGS_call_stack_level=2
export FLAGS_fraction_of_gpu_memory_to_use=0.92
export CUDA_VISIBLE_DEVICES=0
if [ ${WITH_GPU:-OFF} == "ON" ];then
export FLAGS_call_stack_level=2
export FLAGS_fraction_of_gpu_memory_to_use=0.92
export CUDA_VISIBLE_DEVICES=0

UT_list=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d')
num=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d' | wc -l)
echo "Windows 1 card TestCases count is $num"
if [ ${PRECISION_TEST:-OFF} == "ON" ]; then
python ${PADDLE_ROOT}/tools/get_pr_ut.py
if [[ -f "ut_list" ]]; then
set +x
echo "PREC length: "`wc -l ut_list`
precision_cases=`cat ut_list`
set -x
UT_list=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d')
num=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d' | wc -l)
echo "Windows 1 card TestCases count is $num"
if [ ${PRECISION_TEST:-OFF} == "ON" ]; then
python ${PADDLE_ROOT}/tools/get_pr_ut.py
if [[ -f "ut_list" ]]; then
set +x
echo "PREC length: "`wc -l ut_list`
precision_cases=`cat ut_list`
set -x
fi
fi
fi

set +e
if [ ${PRECISION_TEST:-OFF} == "ON" ] && [[ "$precision_cases" != "" ]];then
UT_list_prec=''
re=$(cat ut_list|awk -F ' ' '{print }' | awk 'BEGIN{ all_str=""}{if (all_str==""){all_str=$1}else{all_str=all_str"$|^"$1}} END{print "^"all_str"$"}')
for case in $UT_list; do
flag=$(echo $case|grep -oE $re)
if [ -n "$flag" ];then
if [ -z "$UT_list_prec" ];then
UT_list_prec=$case
set +e
if [ ${PRECISION_TEST:-OFF} == "ON" ] && [[ "$precision_cases" != "" ]];then
UT_list_prec=''
re=$(cat ut_list|awk -F ' ' '{print }' | awk 'BEGIN{ all_str=""}{if (all_str==""){all_str=$1}else{all_str=all_str"$|^"$1}} END{print "^"all_str"$"}')
for case in $UT_list; do
flag=$(echo $case|grep -oE $re)
if [ -n "$flag" ];then
if [ -z "$UT_list_prec" ];then
UT_list_prec=$case
else
UT_list_prec=$UT_list_prec'\n'$case
fi
else
UT_list_prec=$UT_list_prec'\n'$case
echo $case "won't run in PRECISION_TEST mode."
fi
else
echo $case "won't run in PRECISION_TEST mode."
fi
done
UT_list=$UT_list_prec
fi
set -e
done
UT_list=$UT_list_prec
fi
set -e

output=$(python ${PADDLE_ROOT}/tools/parallel_UT_rule.py "${UT_list}")
eight_parallel_job=$(echo $output | cut -d ";" -f 1)
tetrad_parallel_jog=$(echo $output | cut -d ";" -f 2)
non_parallel_job=$(echo $output | cut -d ";" -f 3)
output=$(python ${PADDLE_ROOT}/tools/parallel_UT_rule.py "${UT_list}")
eight_parallel_job=$(echo $output | cut -d ";" -f 1)
tetrad_parallel_jog=$(echo $output | cut -d ";" -f 2)
non_parallel_job=$(echo $output | cut -d ";" -f 3)

non_parallel_job_1=$(echo $non_parallel_job | cut -d "," -f 1)
non_parallel_job_2=$(echo $non_parallel_job | cut -d "," -f 2)
non_parallel_job_1=$(echo $non_parallel_job | cut -d "," -f 1)
non_parallel_job_2=$(echo $non_parallel_job | cut -d "," -f 2)
fi

failed_test_lists=''
tmp_dir=`mktemp -d`
Expand All @@ -267,7 +270,13 @@ function collect_failed_tests() {
set -e
}

function run_unittest() {
function run_unittest_cpu() {
tmpfile=$tmp_dir/$RANDOM
(ctest -E "${disable_ut_quickly}" -LE "${nightly_label}" --output-on-failure -C Release -j 8 | tee $tmpfile) &
wait;
}

function run_unittest_gpu() {
test_case=$1
parallel_job=$2
if [ "$2" == "" ]; then
Expand All @@ -285,7 +294,11 @@ function run_unittest() {
}

function unittests_retry(){
parallel_job=1
if [ "${WITH_GPU:-OFF}" == "ON" ];then
parallel_job=1
else
parallel_job=4
fi
is_retry_execuate=0
wintest_error=1
retry_time=3
Expand Down Expand Up @@ -336,7 +349,7 @@ function unittests_retry(){

function show_ut_retry_result() {
if [[ "$is_retry_execuate" != "0" ]];then
failed_test_lists_ult=`echo "${failed_test_lists}" | grep -Po '[^ ].*$'`
failed_test_lists_ult=`echo "${failed_test_lists}"`
echo "========================================="
echo "There are more than 10 failed unit tests, so no unit test retry!!!"
echo "========================================="
Expand Down Expand Up @@ -365,10 +378,14 @@ function show_ut_retry_result() {
}

set +e
run_unittest $eight_parallel_job 8
run_unittest $tetrad_parallel_jog 4
run_unittest $non_parallel_job_1
run_unittest $non_parallel_job_2
if [ "${WITH_GPU:-OFF}" == "ON" ];then
run_unittest_gpu $eight_parallel_job 8
run_unittest_gpu $tetrad_parallel_jog 4
run_unittest_gpu $non_parallel_job_1
run_unittest_gpu $non_parallel_job_2
else
run_unittest_cpu
fi
collect_failed_tests
set -e
rm -f $tmp_dir/*
Expand Down