|
| 1 | +name: Distribute-stable-Formers |
| 2 | + |
| 3 | +on: |
| 4 | + workflow_call: |
| 5 | + inputs: |
| 6 | + docker_distribute_image: |
| 7 | + type: string |
| 8 | + required: true |
| 9 | + clone-can-skip: |
| 10 | + type: string |
| 11 | + required: false |
| 12 | + default: "false" |
| 13 | + |
| 14 | +env: |
| 15 | + PR_ID: ${{ github.event.pull_request.number }} |
| 16 | + COMMIT_ID: ${{ github.event.pull_request.head.sha }} |
| 17 | + work_dir: /paddle |
| 18 | + PADDLE_ROOT: /paddle |
| 19 | + TASK: paddle-CI-${{ github.event.pull_request.number }}-distribute-formers |
| 20 | + ci_scripts: /paddle/ci |
| 21 | + BRANCH: ${{ github.event.pull_request.base.ref }} |
| 22 | + CI_name: distribute |
| 23 | + no_proxy: bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn,paddlepaddle.org.cn |
| 24 | + docker_image: ${{ inputs.docker_distribute_image }} |
| 25 | + |
| 26 | +defaults: |
| 27 | + run: |
| 28 | + shell: bash |
| 29 | + |
| 30 | +jobs: |
| 31 | + formers-test: |
| 32 | + name: formers-Test |
| 33 | + if: ${{ inputs.clone-can-skip != 'true' }} |
| 34 | + runs-on: |
| 35 | + group: Distribute |
| 36 | + steps: |
| 37 | + - name: Check docker image and run container |
| 38 | + env: |
| 39 | + FLAGS_fraction_of_gpu_memory_to_use: 0.15 |
| 40 | + CTEST_OUTPUT_ON_FAILURE: 1 |
| 41 | + CTEST_PARALLEL_LEVEL: 4 |
| 42 | + WITH_GPU: "ON" |
| 43 | + WITH_AVX: "ON" |
| 44 | + WITH_DISTRIBUTE: "ON" |
| 45 | + WITH_TESTING: "ON" |
| 46 | + WITH_COVERAGE: "OFF" |
| 47 | + CMAKE_BUILD_TYPE: Release |
| 48 | + PADDLE_FRACTION_GPU_MEMORY_TO_USE: 0.15 |
| 49 | + PRECISION_TEST: "OFF" |
| 50 | + WITH_UNITY_BUILD: "ON" |
| 51 | + AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }} |
| 52 | + AGILE_REVISION: ${{ github.event.pull_request.head.sha }} |
| 53 | + WITH_INCREMENTAL_COVERAGE: "OFF" |
| 54 | + WITH_ONNXRUNTIME: "OFF" |
| 55 | + COVERALLS_UPLOAD: "ON" |
| 56 | + PADDLE_VERSION: 0.0.0 |
| 57 | + GIT_PR_ID: ${{ github.event.pull_request.number }} |
| 58 | + PY_VERSION: "3.10" |
| 59 | + CUDA_ARCH_NAME: Auto |
| 60 | + WITH_CUDNN_FRONTEND: "ON" |
| 61 | + FLAGS_enable_cudnn_frontend: 1 |
| 62 | + CACHE_DIR: /root/.cache/build |
| 63 | + CCACHE_DIR: /root/.ccache/formers |
| 64 | + CFS_DIR: /home/data/cfs |
| 65 | + paddle_whl: /workspace/dist/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl |
| 66 | + formers_docker: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:cuda126-dev-latest |
| 67 | + run: | |
| 68 | + export CUDA_SO="$(\ls -d /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls -d /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" |
| 69 | + export DEVICES="$(\ls -d /dev/nvidia* | xargs -I{} echo "-v {}:{}") $(\ls /dev/nvidia-caps/* | xargs -I{} echo "-v {}:{}")" |
| 70 | + export SMI="-v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi" |
| 71 | + container_name=${TASK}-test-$(date +%Y%m%d-%H%M%S) |
| 72 | + echo "container_name=${container_name}" >> ${{ github.env }} |
| 73 | + docker run -d -t --name ${container_name} ${CUDA_SO} ${DEVICES} ${SMI} --runtime=nvidia --shm-size=32G --privileged \ |
| 74 | + -v "/home/data/cfs:/home/data/cfs" \ |
| 75 | + -v "/home/data/cfs/.cache/:/root/.cache" \ |
| 76 | + -v "/home/data/cfs/.ccache:/root/.ccache" \ |
| 77 | + -v "/ssd1/models:/home/models" \ |
| 78 | + -v "/ssd1/root:/root" \ |
| 79 | + -v "/dev/shm:/dev/shm" \ |
| 80 | + -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \ |
| 81 | + -v ${{ github.workspace }}:/workspace \ |
| 82 | + -e BRANCH \ |
| 83 | + -e PR_ID \ |
| 84 | + -e COMMIT_ID \ |
| 85 | + -e work_dir \ |
| 86 | + -e PADDLE_ROOT \ |
| 87 | + -e ci_scripts \ |
| 88 | + -e CI_name \ |
| 89 | + -e PF_HOME=/home/models \ |
| 90 | + -e FLAGS_fraction_of_gpu_memory_to_use \ |
| 91 | + -e CTEST_OUTPUT_ON_FAILURE \ |
| 92 | + -e CTEST_PARALLEL_LEVEL \ |
| 93 | + -e WITH_GPU \ |
| 94 | + -e WITH_AVX \ |
| 95 | + -e WITH_DISTRIBUTE \ |
| 96 | + -e WITH_TESTING \ |
| 97 | + -e WITH_COVERAGE \ |
| 98 | + -e CMAKE_BUILD_TYPE \ |
| 99 | + -e PADDLE_FRACTION_GPU_MEMORY_TO_USE \ |
| 100 | + -e PRECISION_TEST \ |
| 101 | + -e WITH_UNITY_BUILD \ |
| 102 | + -e AGILE_COMPILE_BRANCH \ |
| 103 | + -e AGILE_REVISION \ |
| 104 | + -e WITH_INCREMENTAL_COVERAGE \ |
| 105 | + -e WITH_ONNXRUNTIME \ |
| 106 | + -e COVERALLS_UPLOAD \ |
| 107 | + -e PADDLE_VERSION \ |
| 108 | + -e GIT_PR_ID \ |
| 109 | + -e PY_VERSION \ |
| 110 | + -e CUDA_ARCH_NAME \ |
| 111 | + -e WITH_CUDNN_FRONTEND \ |
| 112 | + -e FLAGS_enable_cudnn_frontend \ |
| 113 | + -e CACHE_DIR \ |
| 114 | + -e CCACHE_DIR \ |
| 115 | + -e CFS_DIR \ |
| 116 | + -e paddle_whl \ |
| 117 | + -e no_proxy \ |
| 118 | + -w /workspace --network host ${formers_docker} |
| 119 | + - name: Download paddle.tar.gz and merge target branch |
| 120 | + run: | |
| 121 | + docker exec -t ${{ env.container_name }} /bin/bash -c ' |
| 122 | + rm -rf * .[^.]* |
| 123 | + echo "Downloading Paddle.tar.gz" |
| 124 | + wget -q --tries=5 --no-proxy https://paddle-github-action.bj.bcebos.com/PR/gpups/${{ env.PR_ID }}/${{ env.COMMIT_ID }}/Paddle.tar.gz --no-check-certificate |
| 125 | + echo "Extracting Paddle.tar.gz" |
| 126 | + tar --use-compress-program="pzstd" -xf Paddle.tar.gz --strip-components=1 |
| 127 | + rm Paddle.tar.gz |
| 128 | + git config --global --add safe.directory /workspace |
| 129 | + git checkout test |
| 130 | + ' |
| 131 | + - name: Test |
| 132 | + run: | |
| 133 | + docker exec -t ${{ env.container_name }} /bin/bash -c ' |
| 134 | + source ${{ github.workspace }}/../../../proxy |
| 135 | + source ${{ github.workspace }}/../../../AISTUDIO_ACCESS_TOKEN |
| 136 | + set -ex |
| 137 | + bash /workspace/ci/formers_test.sh |
| 138 | + ' |
| 139 | + - name: Upload and display logs |
| 140 | + if: always() |
| 141 | + env: |
| 142 | + home_path: ${{ github.workspace }}/.. |
| 143 | + bos_file: ${{ github.workspace }}/../bos_retry/BosClient.py |
| 144 | + run: | |
| 145 | + docker exec -t ${{ env.container_name }} /bin/bash -c ' |
| 146 | + export AK=paddle |
| 147 | + export SK=paddle |
| 148 | + if [ ! -f "${{ env.bos_file }}" ]; then |
| 149 | + wget -q --no-proxy -O ${{ env.home_path }}/bos_retry.tar.gz https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate |
| 150 | + mkdir ${{ env.home_path }}/bos_retry |
| 151 | + tar xf ${{ env.home_path }}/bos_retry.tar.gz -C ${{ env.home_path }}/bos_retry |
| 152 | + fi |
| 153 | + if [ -n "$PR_ID" ] && [ "$PR_ID" != "0" ]; then |
| 154 | + bos_prefix="${PR_ID}/${COMMIT_ID}" |
| 155 | + else |
| 156 | + bos_prefix="schedule/$(date +%Y%m%d)" |
| 157 | + fi |
| 158 | + # api test logs |
| 159 | + cd /workspace/PaddleFormers/unittest_logs |
| 160 | + for FILE in /workspace/PaddleFormers/unittest_logs/*; do |
| 161 | + file=$(basename "$FILE") |
| 162 | + python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs |
| 163 | + echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/unittest-gpu/${bos_prefix}/logs/$file" |
| 164 | + done |
| 165 | + # models test logs |
| 166 | + cd /workspace/PaddleFormers/model_unittest_logs |
| 167 | + for FILE in /workspace/PaddleFormers/model_unittest_logs/*; do |
| 168 | + file=$(basename "$FILE") |
| 169 | + python ${{ env.bos_file }} $file paddle-github-action/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs |
| 170 | + echo "$file: https://paddle-github-action.bj.bcebos.com/PR/PaddleFormers/model-unittest-gpu/${bos_prefix}/logs/$file" |
| 171 | + done |
| 172 | + ' |
| 173 | + - name: Terminate and delete the container |
| 174 | + if: always() |
| 175 | + run: | |
| 176 | + set +e |
| 177 | + docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*' |
| 178 | + docker rm -f ${{ env.container_name }} |
0 commit comments