Nightly Tests #44
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| torchrepo: | |
| description: 'torchrepo' | |
| required: true | |
| default: 'https://github.com/pytorch/pytorch.git' | |
| torchbranch: | |
| description: 'torchbranch' | |
| required: true | |
| default: 'v2.0.1' | |
| torchcommit: | |
| description: 'torchcommit' | |
| required: true | |
| default: 'e9ebda29d87ce0916ab08c06ab26fd3766a870e5' | |
| ipexrepo: | |
| description: 'ipexrepo' | |
| required: true | |
| default: 'https://github.com/intel/intel-extension-for-pytorch.git' | |
| ipexbranch: | |
| description: 'ipexbranch' | |
| required: true | |
| default: 'xpu-master' | |
| ipexcommit: | |
| description: 'ipexcommit' | |
| required: true | |
| default: '4af80f77740ed939be78eba28ae36951823f335c' | |
| oneapi: | |
| description: 'oneAPI basekit version' | |
| required: true | |
| default: '2023.2.0' | |
| schedule: | |
| - cron: "0 14 * * *" | |
| jobs: | |
| Nightly-Tests: | |
| runs-on: [self-hosted, PVC] | |
| steps: | |
| - name: Create conda environment | |
| run: | | |
| source ${HOME}/miniconda3/bin/activate triton-nightly-test | |
| conda install -y astunparse numpy ninja pyyaml setuptools cmake cffi typing_extensions future six requests dataclasses mkl mkl-include | |
| conda install -y -c conda-forge libstdcxx-ng | |
| python3 -m pip install pytest-xdist | |
| python3 -m pip install --upgrade pre-commit | |
| - name: Check Pre-commit | |
| run: | | |
| source ${HOME}/miniconda3/bin/activate triton-nightly-test | |
| cd ${HOME}/triton-nightly | |
| rm -rf triton | |
| git clone https://github.com/openai/triton triton | |
| cd triton | |
| triton_commit=`git rev-parse HEAD` | |
| echo "triton_commit: ${triton_commit}" | tee sw_info.log | |
| git submodule sync | |
| git submodule update --init --recursive --jobs 0 | |
| cd third_party/intel_xpu_backend | |
| git checkout main && git pull | |
| python3 -m pre_commit run --all-files | |
| - name: Install Dependency | |
| run: | | |
| python --version | |
| source ${HOME}/miniconda3/bin/activate triton-nightly-test | |
| python --version | |
| pip install setuptools cython numpy wheel scikit-build scipy | |
| pip install psutil cpuid | |
| cd ${HOME}/triton-nightly | |
| cp triton/third_party/intel_xpu_backend/.github/scripts/env_prepare.sh . | |
| cp triton/third_party/intel_xpu_backend/.github/scripts/env_triton.sh ${HOME}/ | |
| cp -r triton/third_party/intel_xpu_backend/.github/patches/ . | |
| bash env_prepare.sh triton-nightly \ | |
| ${{ github.event.inputs.torchrepo }} \ | |
| ${{ github.event.inputs.torchbranch }} \ | |
| ${{ github.event.inputs.torchcommit }} \ | |
| ${{ github.event.inputs.ipexrepo }} \ | |
| ${{ github.event.inputs.ipexbranch }} \ | |
| ${{ github.event.inputs.ipexcommit }} \ | |
| ${{ github.event.inputs.oneapi }} | |
| source ${HOME}/env_triton.sh ${{ github.event.inputs.oneapi }} | |
| python -c "import torch;import intel_extension_for_pytorch" | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo -e "[ERROR] Public-torch or IPEX BUILD FAIL" | |
| exit 1 | |
| fi | |
| - name: Build Triton | |
| shell: bash | |
| run: | | |
| source ${HOME}/miniconda3/bin/activate triton-nightly-test | |
| pip install pybind11 | |
| pip uninstall -y triton | |
| cd ${HOME}/triton-nightly/triton/python | |
| python setup.py clean | |
| TRITON_CODEGEN_INTEL_XPU_BACKEND=1 python setup.py bdist_wheel | |
| pip install dist/*.whl | |
| cd ${HOME}/triton-nightly | |
| python -c "import triton" | |
| if [ ${PIPESTATUS[0]} -ne 0 ]; then | |
| echo -e "[ERROR] Triton BUILD FAIL" | |
| exit 1 | |
| fi | |
| - name: Unit Test for triton on PVC | |
| run: | | |
| echo -e "[ INFO ] Run UT test on Node $(hostname)" | |
| source ${HOME}/miniconda3/bin/activate triton-preci | |
| source ${HOME}/env_triton.sh ${{ github.event.inputs.oneapi }} | |
| pip install pytest | |
| rm -rf ~/.triton/cache | |
| export TRITON_LIBDEVICE_PATH=${HOME}/triton-nightly/triton/python/triton/third_party/xpu/lib | |
| cd ${HOME}/triton-nightly/triton/python/test/unit/language | |
| bash ${HOME}/triton-nightly/triton/third_party/intel_xpu_backend/.github/scripts/case_update.sh triton-nightly | |
| ZE_AFFINITY_MASK=1.0 pytest -n 32 -v -ra --tb=line . --device=xpu 2>&1 | tee ./ut_raw.log || true | |
| res=$(cat ut_raw.log | sed -n '7p' | awk '{print $NF}') | |
| if [ "$res" == "error" ]; then | |
| echo -e "[ERROR] IPEX PVC Triton UT FAIL" | |
| exit 1 | |
| fi | |
| - name: UT Log Preview | |
| run: | | |
| cd ${HOME}/triton-nightly/triton/python/test/unit/language | |
| echo "============ UT raw log head ============" | |
| head -n 20 ut_raw.log | |
| echo "============ UT raw log tail ============" | |
| tail -n 20 ut_raw.log | |
| - name: UT Status Overview | |
| run: | | |
| cd ${HOME}/triton-nightly/triton/python/test/unit/language | |
| echo -e "============ UT Status Overview ============" | tee ./ut_summary.log | |
| grep "^FAILED" ut_raw.log | awk '{print $2}' > ./ut_failed.log | |
| grep "^SKIPPED" ut_raw.log | awk '{print $2}' | grep -o '[0-9]\+' > ./ut_skipped.log | |
| grep "PASSED" ut_raw.log | awk '{print $5}' > ./ut_passed.log | |
| num_failed=$(cat ./ut_failed.log | wc -l) | |
| num_skipped=$(echo $(echo -n `cat ./ut_skipped.log | awk '{print $1}'`| tr ' ' '+')|bc) | |
| num_passed=$(cat ./ut_passed.log | wc -l) | |
| num_total=$(grep "items" ut_raw.log | awk '{print $3}' | grep -o '[0-9]\+') | |
| if [ -z $num_total ]; then num_total=0; fi | |
| echo "num_total: $num_total" | tee -a ./ut_summary.log | |
| echo "num_skipped: $num_skipped" | tee -a ./ut_summary.log | |
| echo "num_failed: $num_failed" | tee -a ./ut_summary.log | |
| echo "num_passed: $num_passed" | tee -a ./ut_summary.log | |
| - name: New Solved Cases | |
| run: | | |
| cd ${HOME}/triton-nightly/triton/python/test/unit/language | |
| awk 'NR==FNR{a[$0]}NR>FNR{ if(!($1 in a)) print $0}' ./ut_failed.log ${HOME}/triton-nightly/triton/third_party/intel_xpu_backend/.github/tests/triton_expected_failure_tests.log | tee -a ./new_solved_cases.log | |
| echo -e "============ New Solved Cases ============" | |
| cat ./new_solved_cases.log | |
| num_new_solved_cases=$(cat ./new_solved_cases.log | wc -l) | |
| echo "num_new_solved_cases: $num_new_solved_cases" | tee -a ./ut_summary.log | |
| - name: New Failed Cases | |
| run: | | |
| cd ${HOME}/triton-nightly/triton/python/test/unit/language | |
| awk 'NR==FNR{a[$0]}NR>FNR{ if(!($1 in a)) print $0}' ${HOME}/triton-nightly/triton/third_party/intel_xpu_backend/.github/tests/triton_expected_failure_tests.log ./ut_failed.log | tee -a ./new_failed_cases.log | |
| echo -e "============ New Failed Cases ============" | |
| cat ./new_failed_cases.log | |
| num_new_failed_cases=$(cat ./new_failed_cases.log | wc -l) | |
| echo "num_new_failed_cases: $num_new_failed_cases" | tee -a ./ut_summary.log | |
| if [ "$num_new_failed_cases" != "0" ]; then | |
| echo -e "Triton has new failed cases" | |
| exit 1 | |
| fi | |
| - name: Total Pass Rate Calculate (no matmul/dot) | |
| run: | | |
| cd ${HOME}/triton-nightly/triton/python/test/unit/language | |
| echo -e "============ Total Pass Rate (no matmul/dot) ============" | tee -a ./ut_summary.log | |
| num_total=$(grep "items" ut_raw.log | awk '{print $3}' | grep -o '[0-9]\+') | |
| num_passed=$(cat ./ut_passed.log | wc -l) | |
| num_expected=$(grep "(expected)" ut_raw.log | wc -l) | |
| num_dot=$(grep "dot" ut_raw.log | wc -l) | |
| num_matmul=$(grep "matmul" ut_raw.log | wc -l) | |
| num_dot_success=$(grep "dot" ./ut_passed.log | wc -l) | |
| num_matmul_success=$(grep "matmul" ./ut_passed.log | wc -l) | |
| let num_expected_skip_and_sucess=num_passed-num_dot_success-num_matmul_success+num_expected | |
| let num_total_wo_mat_dot=num_total-num_dot-num_matmul | |
| total_pass_rate=`awk 'BEGIN{printf "%.2f%%\n",('$num_expected_skip_and_sucess'/'$num_total_wo_mat_dot')*100}'` | |
| echo "num_matmul: $num_matmul" | tee -a ./ut_summary.log | |
| echo "num_dot: $num_dot" | tee -a ./ut_summary.log | |
| echo "num_expected: $num_expected" | tee -a ./ut_summary.log | |
| echo "num_expected_skip_and_sucess: $num_expected_skip_and_sucess" | tee -a ./ut_summary.log | |
| echo "num_total_wo_mat_dot: $num_total_wo_mat_dot" | tee -a ./ut_summary.log | |
| echo "total_pass_rate: $total_pass_rate" | tee -a ./ut_summary.log | |
| - name: Float32 Cases Pass Rate Calculate (no matmul/dot) | |
| run: | | |
| cd ${HOME}/triton-nightly/triton/python/test/unit/language | |
| echo -e "============ Float32 Cases Pass Rate (no matmul/dot) ============" | tee -a ./ut_summary.log | |
| num_float32=$(grep "float32" ut_raw.log | grep -v 'matmul\|dot' | wc -l) | |
| num_no_datatype=$(grep -v 'float\|int\|bool\|long\|f8\|f16\|i32\|i64\|u64' ut_raw.log | grep -v 'matmul\|dot' | wc -l) | |
| num_float32_success=$(grep -v 'matmul\|dot' ./ut_passed.log | grep "float32" | wc -l) | |
| num_no_datatype_success=$(grep -v 'matmul\|dot' ./ut_passed.log | grep -v 'float\|int\|bool\|long\|f8\|f16\|i32\|i64\|u64' | wc -l) | |
| let num_float32_cases_total=num_float32+num_no_datatype | |
| let num_float32_cases_expected_skip_and_success_total=num_float32_success+num_no_datatype_success | |
| float32_cases_pass_rate=`awk 'BEGIN{printf "%.2f%%\n",('$num_float32_cases_expected_skip_and_success_total'/'$num_float32_cases_total')*100}'` | |
| echo "num_float32: $num_float32" | tee -a ./ut_summary.log | |
| echo "num_no_datatype: $num_no_datatype" | tee -a ./ut_summary.log | |
| echo "num_float32_success: $num_float32_success" | tee -a ./ut_summary.log | |
| echo "num_no_datatype_success: $num_no_datatype_success" | tee -a ./ut_summary.log | |
| echo "num_float32_cases_total: $num_float32_cases_total" | tee -a ./ut_summary.log | |
| echo "num_float32_cases_expected_skip_and_success_total: $num_float32_cases_expected_skip_and_success_total" | tee -a ./ut_summary.log | |
| echo "float32_cases_pass_rate: $float32_cases_pass_rate" | tee -a ./ut_summary.log | |
| - name: UT Status Summary (no matmul/dot) | |
| run: | | |
| cd ${HOME}/triton-nightly/triton/python/test/unit/language | |
| cp ${HOME}/triton-nightly/triton/*.log . | |
| cat ut_summary.log | |
| - name: Upload Triton PVC nightly Data | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: Triton-nightly-data | |
| path: /home/sdp/triton-nightly/triton/python/test/unit/language/*.log |