Fix bugs of third party (PaddlePaddle#56670)

* fix bugs of tp * fix bugs of tp * fix bugs * fix bugs * fix bugs of md5
BeingGod · Sep 9, 2023 · e7b6b58 · e7b6b58
1 parent cc97258
commit e7b6b58
Showing 1 changed file with 51 additions and 3 deletions.
diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat
@@ -402,8 +402,8 @@ if %day_now% NEQ %day_before% (
 )
 
 echo set -ex > cache.sh
-echo md5_content=$(cat %work_dir:\=/%/cmake/external/*.cmake  ^|md5sum ^| awk '{print $1}') >> cache.sh
-echo echo ${md5_content}^>md5.txt >> cache.sh
+echo md5_content=$(cat %work_dir:\=/%/cmake/external/*.cmake^|md5sum^|awk '{print $1}')$(git submodule status^|md5sum^|awk '{print $1}')>>cache.sh
+echo echo ${md5_content}^>md5.txt>>cache.sh
 
 %cache_dir%\tools\busybox64.exe cat cache.sh
 %cache_dir%\tools\busybox64.exe bash cache.sh
@@ -415,6 +415,51 @@ if "%WITH_GPU%"=="ON" (
 ) else (
     set sub_dir=cpu
 )
+
+@ECHO ON
+cd /d %work_dir%
+python -c "import wget;wget.download('https://paddle-windows.bj.bcebos.com/third_party_code/%sub_dir%/%md5%.tar.gz')" 2>nul
+if !ERRORLEVEL! EQU 0 (
+    echo Getting source code of third party : extracting ...
+    tar -xf %md5%.tar.gz
+    del %md5%.tar.gz
+    if !errorlevel! EQU 0 (
+        echo Getting source code of third party : successful
+    )
+) else (
+    git submodule update --init --recursive
+    set BCE_FILE=%cache_dir%\bce-python-sdk-new\BosClient.py
+    echo Uploading source code of third_party: checking bce ...
+    if not exist %cache_dir%\bce-python-sdk-new (
+        echo There is no bce in this PC, will install bce.
+        cd /d %cache_dir%
+        echo Download package from https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz
+        python -c "import wget;wget.download('https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz')"
+        python -c "import shutil;shutil.unpack_archive('bos_new.tar.gz', extract_dir='./bce-python-sdk-new',format='gztar')"
+    )
+    python -m pip install pycryptodome
+    python -m pip install bce-python-sdk==0.8.74
+    if !errorlevel! EQU 0 (
+        cd /d %work_dir%
+        echo Uploading source code of third party: compressing ...
+        tar -zcf %md5%.tar.gz ./third_party ./.git/modules
+        if !errorlevel! EQU 0 (
+            echo Uploading source code of third party: uploading ...
+            python !BCE_FILE! %md5%.tar.gz paddle-windows/third_party_code/%sub_dir% 1>nul
+            if !errorlevel! EQU 0 (
+                echo Upload source code of third party %md5% to bos paddle-windows/third_party_code/%sub_dir% successfully.
+            ) else (
+                echo Failed upload source code of third party to bos, reason: upload failed.
+            )
+        ) else (
+            echo Failed upload source code of third party to bos, reason: compress failed.
+        )
+        del %md5%.tar.gz
+    ) else (
+        echo Failed upload source code of third party to bos, reason: install bce failed.
+    )
+)
+
 set THIRD_PARTY_HOME=%cache_dir:\=/%/third_party/%sub_dir%
 set THIRD_PARTY_PATH=%THIRD_PARTY_HOME%/%md5%
 
@@ -450,7 +495,9 @@ if not exist %THIRD_PARTY_PATH% (
     echo Found reusable third_party cache in %THIRD_PARTY_PATH%, will reuse it.
 )
 
+
 :cmake_impl
+cd /d %work_dir%\%BUILD_DIR%
 echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^
 -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DON_INFER=%ON_INFER% ^
 -DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^
@@ -582,8 +629,9 @@ if "%UPLOAD_TP_FILE%"=="ON" (
         echo Download package from https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz
         python -c "import wget;wget.download('https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz')"
         python -c "import shutil;shutil.unpack_archive('bos_new.tar.gz', extract_dir='./bce-python-sdk-new',format='gztar')"
-        python -m pip install bce-python-sdk==0.8.74
     )
+    python -m pip install pycryptodome
+    python -m pip install bce-python-sdk==0.8.74
     if !errorlevel! EQU 0 (
         cd /d %THIRD_PARTY_HOME%
         echo Uploading third_party: compressing ...