Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix bugs of third party #56670

Merged
merged 8 commits into from
Aug 30, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 51 additions & 3 deletions paddle/scripts/paddle_build.bat
Original file line number Diff line number Diff line change
Expand Up @@ -402,8 +402,8 @@ if %day_now% NEQ %day_before% (
)

echo set -ex > cache.sh
echo md5_content=$(cat %work_dir:\=/%/cmake/external/*.cmake ^|md5sum ^| awk '{print $1}') >> cache.sh
echo echo ${md5_content}^>md5.txt >> cache.sh
echo md5_content=$(cat %work_dir:\=/%/cmake/external/*.cmake^|md5sum^|awk '{print $1}')$(git submodule status^|md5sum^|awk '{print $1}')>>cache.sh
echo echo ${md5_content}^>md5.txt>>cache.sh

%cache_dir%\tools\busybox64.exe cat cache.sh
%cache_dir%\tools\busybox64.exe bash cache.sh
Expand All @@ -415,6 +415,51 @@ if "%WITH_GPU%"=="ON" (
) else (
set sub_dir=cpu
)

@ECHO ON
cd /d %work_dir%
python -c "import wget;wget.download('https://paddle-windows.bj.bcebos.com/third_party_code/%sub_dir%/%md5%.tar.gz')" 2>nul
if !ERRORLEVEL! EQU 0 (
echo Getting source code of third party : extracting ...
tar -xf %md5%.tar.gz
del %md5%.tar.gz
if !errorlevel! EQU 0 (
echo Getting source code of third party : successful
)
) else (
git submodule update --init --recursive
set BCE_FILE=%cache_dir%\bce-python-sdk-new\BosClient.py
echo Uploading source code of third_party: checking bce ...
if not exist %cache_dir%\bce-python-sdk-new (
echo There is no bce in this PC, will install bce.
cd /d %cache_dir%
echo Download package from https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz
python -c "import wget;wget.download('https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz')"
python -c "import shutil;shutil.unpack_archive('bos_new.tar.gz', extract_dir='./bce-python-sdk-new',format='gztar')"
)
python -m pip install pycryptodome
python -m pip install bce-python-sdk==0.8.74
if !errorlevel! EQU 0 (
cd /d %work_dir%
echo Uploading source code of third party: compressing ...
tar -zcf %md5%.tar.gz ./third_party ./.git/modules
if !errorlevel! EQU 0 (
echo Uploading source code of third party: uploading ...
python !BCE_FILE! %md5%.tar.gz paddle-windows/third_party_code/%sub_dir% 1>nul
if !errorlevel! EQU 0 (
echo Upload source code of third party %md5% to bos paddle-windows/third_party_code/%sub_dir% successfully.
) else (
echo Failed upload source code of third party to bos, reason: upload failed.
)
) else (
echo Failed upload source code of third party to bos, reason: compress failed.
)
del %md5%.tar.gz
) else (
echo Failed upload source code of third party to bos, reason: install bce failed.
)
)

set THIRD_PARTY_HOME=%cache_dir:\=/%/third_party/%sub_dir%
set THIRD_PARTY_PATH=%THIRD_PARTY_HOME%/%md5%

Expand Down Expand Up @@ -450,7 +495,9 @@ if not exist %THIRD_PARTY_PATH% (
echo Found reusable third_party cache in %THIRD_PARTY_PATH%, will reuse it.
)


:cmake_impl
cd /d %work_dir%\%BUILD_DIR%
echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^
-DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DON_INFER=%ON_INFER% ^
-DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^
Expand Down Expand Up @@ -582,8 +629,9 @@ if "%UPLOAD_TP_FILE%"=="ON" (
echo Download package from https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz
python -c "import wget;wget.download('https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz')"
python -c "import shutil;shutil.unpack_archive('bos_new.tar.gz', extract_dir='./bce-python-sdk-new',format='gztar')"
python -m pip install bce-python-sdk==0.8.74
)
python -m pip install pycryptodome
python -m pip install bce-python-sdk==0.8.74
xuxinyi389 marked this conversation as resolved.
Show resolved Hide resolved
if !errorlevel! EQU 0 (
cd /d %THIRD_PARTY_HOME%
echo Uploading third_party: compressing ...
Expand Down