Skip to content

Commit bd35ca3

Browse files
swgu98swgu98
authored
[CI] beijing cfs (#73697)
* cfs test=document_fix * cfs test=document_fix * cfs env test=document_fix * cfs env test=document_fix
1 parent 409c59d commit bd35ca3

File tree

5 files changed

+35
-18
lines changed

5 files changed

+35
-18
lines changed

‎.github/workflows/Coverage.yml‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ env:
2121
PADDLE_ROOT: /paddle
2222
GIT_PR_ID: ${{ github.event.pull_request.number }}
2323
CI_name: coverage
24+
CFS_DIR: /home/data/cfs
2425
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
2526

2627
defaults:
@@ -129,6 +130,7 @@ jobs:
129130
-e GITHUB_TOKEN \
130131
-e GITHUB_API_TOKEN \
131132
-e UT_RUN_TYPE_SETTING \
133+
-e CFS_DIR \
132134
-e no_proxy \
133135
-w /paddle --network host ${docker_image}
134136
@@ -161,8 +163,8 @@ jobs:
161163
if: steps.check-bypass.outputs.can-skip != 'true'
162164
run: |
163165
docker exec -t ${{ env.container_name }} /bin/bash -c '
164-
mkdir -p /home/data/cfs/.cache/coverage
165-
mkdir -p /home/data/cfs/.ccache/coverage
166+
mkdir -p ${CFS_DIR}/.cache/coverage
167+
mkdir -p ${CFS_DIR}/.ccache/coverage
166168
source ${{ github.workspace }}/../../../proxy
167169
bash ${ci_scripts}/cmake-predownload.sh
168170
bash $ci_scripts/coverage_build.sh bdist_wheel
@@ -227,13 +229,14 @@ jobs:
227229
tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry
228230
fi
229231
cd /paddle/dist
232+
mkdir -p ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}
230233
echo "Uploading coverage build size"
231234
python ${{ env.bos_file }} coverage_build_size paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
232235
echo "Uploading coverage wheel"
233236
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
234237
cd /
235238
echo "Uploading Paddle.tar.gz"
236-
python ${{ env.bos_file }} Paddle.tar.gz paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
239+
cp Paddle.tar.gz ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}
237240
rm Paddle.tar.gz
238241
'
239242
@@ -321,7 +324,8 @@ jobs:
321324
-e ON_INFER \
322325
-e COVERAGE_FILE \
323326
-e GITHUB_TOKEN \
324-
-e GITHUB_API_TOKEN \
327+
-e GITHUB_API_TOKEN \
328+
-e CFS_DIR \
325329
-e no_proxy \
326330
-w /paddle --network host ${docker_image}
327331
@@ -330,8 +334,8 @@ jobs:
330334
docker exec -t ${{ env.container_name }} /bin/bash -c '
331335
rm -rf * .[^.]*
332336
set -e
333-
echo "Downloading Paddle.tar.gz"
334-
wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
337+
echo "Downloading Paddle.tar.gz from cfs"
338+
cp ${CFS_DIR}/coverage_bos/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz .
335339
echo "Extracting Paddle.tar.gz"
336340
tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
337341
rm Paddle.tar.gz

‎.github/workflows/_CE-CINN-Framework.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ env:
1919
ci_scripts: /paddle/ci
2020
BRANCH: ${{ github.event.pull_request.base.ref }}
2121
CI_name: ce-cinn-framework
22+
CFS_DIR: /home/data/cfs
2223
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
2324

2425
defaults:
@@ -60,6 +61,7 @@ jobs:
6061
-e CI_name \
6162
-e CACHE_DIR \
6263
-e GITHUB_API_TOKEN \
64+
-e CFS_DIR \
6365
-w /paddle --network host ${docker_image}
6466
6567
- name: Download Paddle and PaddleTest
@@ -68,8 +70,8 @@ jobs:
6870
run: |
6971
docker exec -t ${{ env.container_name }} /bin/bash -c '
7072
rm -rf * .[^.]*
71-
echo "Downloading build.tar.gz"
72-
wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/build/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
73+
echo "Downloading build.tar.gz from cfs"
74+
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
7375
echo "Extracting build.tar.gz"
7476
git config --global --add safe.directory ${work_dir}
7577
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1

‎.github/workflows/_CE-Framework.yml‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ env:
1919
BRANCH: ${{ github.event.pull_request.base.ref }}
2020
TASK: paddle-CI-${{ github.event.pull_request.number }}-CE-Framework
2121
CI_name: ce-framework
22+
CFS_DIR: /home/data/cfs
2223
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
2324

2425
defaults:
@@ -92,6 +93,7 @@ jobs:
9293
-e CI_name \
9394
-e CACHE_DIR \
9495
-e FLAGS_enable_eager_mode \
96+
-e CFS_DIR \
9597
-w /paddle --network host ${docker_image}
9698
9799
- name: Download Paddle and PaddleTest
@@ -100,8 +102,8 @@ jobs:
100102
run: |
101103
docker exec -t ${{ env.container_name }} /bin/bash -c '
102104
rm -rf * .[^.]*
103-
echo "Downloading build.tar.gz"
104-
wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/build/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
105+
echo "Downloading build.tar.gz from cfs"
106+
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
105107
echo "Extracting build.tar.gz"
106108
git config --global --add safe.directory ${work_dir}
107109
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
@@ -189,6 +191,7 @@ jobs:
189191
-e CI_name \
190192
-e CACHE_DIR \
191193
-e FLAGS_enable_eager_mode \
194+
-e CFS_DIR \
192195
-w /paddle --network host ${docker_image}
193196
194197
- name: Download Paddle and PaddleTest
@@ -197,8 +200,8 @@ jobs:
197200
run: |
198201
docker exec -t ${{ env.container_name }} /bin/bash -c '
199202
rm -rf * .[^.]*
200-
echo "Downloading build.tar.gz"
201-
wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/build/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
203+
echo "Downloading build.tar.gz from cfs"
204+
cp ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
202205
echo "Extracting build.tar.gz"
203206
git config --global --add safe.directory ${work_dir}
204207
tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
@@ -229,7 +232,7 @@ jobs:
229232
ln -s $(which python3.10) run_env/python
230233
ln -s $(which pip3.10) run_env/pip
231234
export PATH=$(pwd)/run_env:${PATH}
232-
pip config set global.cache-dir "/home/data/cfs/.cache/pip"
235+
pip config set global.cache-dir "${CFS_DIR}/.cache/pip"
233236
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
234237
pip install sympy
235238
pip install /paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl

‎.github/workflows/_Inference.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ env:
1414
work_dir: /paddle
1515
PADDLE_ROOT: /paddle
1616
BRANCH: ${{ github.event.pull_request.base.ref }}
17+
CFS_DIR: /home/data/cfs
1718
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
1819

1920
defaults:
@@ -104,6 +105,7 @@ jobs:
104105
-e ci_scripts \
105106
-e CUDA_ARCH_NAME \
106107
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
108+
-e CFS_DIR \
107109
-e no_proxy \
108110
-w /paddle --network host ${docker_image} /bin/bash
109111
@@ -159,7 +161,9 @@ jobs:
159161
source /action-runner/unproxy
160162
cd /
161163
tar --use-compress-program="pigz -1" -cpPf build.tar.gz paddle
162-
python ${{ env.bos_file }} build.tar.gz paddle-github-action/PR/inference/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
164+
echo "Uploading build.tar.gz to cfs"
165+
mkdir -p ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID}
166+
cp build.tar.gz ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID} && echo success
163167
rm build.tar.gz
164168
'
165169
@@ -247,15 +251,16 @@ jobs:
247251
-e CUDA_ARCH_NAME \
248252
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
249253
-e GITHUB_API_TOKEN \
254+
-e CFS_DIR \
250255
-e no_proxy \
251256
-w /paddle --network host ${docker_image} /bin/bash
252257
253258
- name: Download paddle.tar.gz
254259
run: |
255260
docker exec -t ${{ env.container_name }} /bin/bash -c '
256261
set -e
257-
echo "Downloading build.tar.gz"
258-
wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/inference/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
262+
echo "Downloading build.tar.gz from cfs"
263+
cp ${CFS_DIR}/inference_bos/${PR_ID}/${COMMIT_ID}/build.tar.gz .
259264
echo "Extracting build.tar.gz"
260265
tar --use-compress-program="pigz -1" -xpf build.tar.gz --strip-components=1
261266
rm build.tar.gz

‎.github/workflows/_Linux-build.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ env:
2424
ci_scripts: /paddle/ci
2525
BRANCH: ${{ github.event.pull_request.base.ref }}
2626
CI_name: build
27+
CFS_DIR: /home/data/cfs
2728
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
2829

2930
defaults:
@@ -114,6 +115,7 @@ jobs:
114115
-e CCACHE_DIR \
115116
-e CCACHE_SLOPPINESS \
116117
-e CCACHE_STATSLOG \
118+
-e CFS_DIR \
117119
-e no_proxy \
118120
-e CI_name \
119121
-w /paddle --network host ${docker_image}
@@ -195,8 +197,9 @@ jobs:
195197
tar --use-compress-program="pzstd -1" -cf build.tar.gz paddle
196198
echo "Uploading build.tar.gz to bos"
197199
python ${{ env.bos_file }} build.tar.gz paddle-github-action/PR/build/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
198-
mkdir -p /home/data/cfs/build/${PR_ID}/${COMMIT_ID}
199-
mv build.tar.gz /home/data/cfs/build/${PR_ID}/${COMMIT_ID}
200+
echo "Uploading build.tar.gz to cfs"
201+
mkdir -p ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID}
202+
mv build.tar.gz ${CFS_DIR}/build_bos/${PR_ID}/${COMMIT_ID} && echo success
200203
mv /paddle/build/pr_whl/${{ env.paddle_whl }} .
201204
echo "Uploading paddle_whl to bos"
202205
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/build/${{ env.PR_ID }}/${{ env.COMMIT_ID }}

0 commit comments

Comments
 (0)