Unit Tests #45
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Unit Tests | |
| on: | |
| schedule: | |
| - cron: '0 19 * * *' | |
| repository_dispatch: | |
| workflow_dispatch: | |
| inputs: | |
| repo: | |
| description: 'GitHub repo {owner}/{repo}' | |
| required: false | |
| default: '' | |
| ref: | |
| description: 'Branch, Tag or Commit SHA' | |
| required: false | |
| default: '' | |
| test_names: | |
| description: 'Input Test(s) to Run (default all)' | |
| required: false | |
| default: '' | |
| env: | |
| CUDA_DEVICE_ORDER: PCI_BUS_ID | |
| AMD_SERVER: 10.0.13.31 | |
| INTEL_SERVER: 10.0.23.35 | |
| CPU_TEST_FILES: "test_qbits.py" | |
| concurrency: | |
| group: ${{ github.ref }}-workflow | |
| cancel-in-progress: true | |
| jobs: | |
| check-vm: | |
| runs-on: self-hosted | |
| container: | |
| image: modelcloud/gptqmodel:alpine-ci-v1 | |
| outputs: | |
| ip: ${{ steps.get_ip.outputs.ip }} | |
| tag: ${{ steps.get_ip.outputs.tag }} | |
| steps: | |
| - name: Select server | |
| id: get_ip | |
| shell: bash | |
| run: | | |
| if [[ "${{ runner.name }}" == *"intel"* ]]; then | |
| echo "current ci is intel" | |
| response=0 | |
| else | |
| echo "test intel vm status" | |
| response=$(curl --silent --fail --max-time 5 http://$INTEL_SERVER/gpu/runner/status/intel) || response=error | |
| if [ "$response" == "error" ]; then | |
| echo "test amd vm status" | |
| response=$(curl --silent --fail --max-time 5 http://${AMD_SERVER}/gpu/runner/status/intel) || response=error | |
| fi | |
| fi | |
| echo "response: $response" | |
| if [ "$response" == "0" ]; then | |
| tag="intel" | |
| elif [ "$response" == "-1" ]; then | |
| tag="amd" | |
| else | |
| echo "Error: Unexpected result - $response" | |
| exit 1 | |
| fi | |
| echo "Runner tag: $tag" | |
| response=$(curl -s --head --fail --max-time 5 http://${INTEL_SERVER}/gpu/status) || response=error | |
| if echo "$response" | grep "200 OK" > /dev/null; then | |
| echo "Intel server is online. set ip to $ip" | |
| ip=${INTEL_SERVER} | |
| else | |
| response=$(curl -s --head --max-time 5 http://${AMD_SERVER}/gpu/status) || response=error | |
| if echo "$response" | grep "200 OK" > /dev/null; then | |
| ip=${AMD_SERVER} | |
| echo "AMD server is online. set ip to $ip" | |
| else | |
| echo "AMD server is offline." | |
| exit 1 | |
| fi | |
| fi | |
| echo "ip=$ip" >> "$GITHUB_OUTPUT" | |
| echo "tag=$tag" >> "$GITHUB_OUTPUT" | |
| echo "tag: $tag, ip: $ip" | |
| build-intel: | |
| runs-on: [self-hosted, intel] | |
| needs: check-vm | |
| if: needs.check-vm.outputs.tag == 'intel' | |
| container: | |
| image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:github-ci-v2 | |
| steps: | |
| - name: Checkout Codes | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.repo }} | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: Install requirements | |
| shell: bash | |
| run: pip install -r requirements.txt -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }} | |
| - name: Compile | |
| shell: bash | |
| run: python setup.py bdist_wheel | |
| - name: Upload to artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dist | |
| path: dist | |
| build-amd: | |
| runs-on: [self-hosted, amd] | |
| needs: check-vm | |
| if: needs.check-vm.outputs.tag == 'amd' | |
| container: | |
| image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:github-ci-v2 | |
| steps: | |
| - name: Checkout Codes | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.repo }} | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: Install requirements | |
| shell: bash | |
| run: pip install -r requirements.txt -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }} | |
| - name: Compile | |
| shell: bash | |
| run: python setup.py bdist_wheel | |
| - name: Upload to artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dist | |
| path: dist | |
| list-test-files: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| cpu-files: ${{ steps.files.outputs.cpu-files }} | |
| gpu-files: ${{ steps.files.outputs.gpu-files }} | |
| steps: | |
| - name: Checkout Codes | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.repo }} | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: List files | |
| id: files | |
| shell: bash | |
| run: | | |
| script=" | |
| import json | |
| import os | |
| cpu_file_list = [f.strip().removesuffix('.py') for f in '${CPU_TEST_FILES}'.split(',') if f.strip()] | |
| test_files_list = [f.strip().removesuffix('.py') for f in '${{ github.event.inputs.test_names }}'.split(',') if f.strip()] | |
| cpu_test_files = [f for f in cpu_file_list if not test_files_list or f in test_files_list] | |
| all_tests = [f.removesuffix('.py') for f in os.listdir('tests/') if f.startswith('test_') and f.endswith('.py')] | |
| gpu_test_files = [f for f in all_tests if f not in cpu_file_list and (not test_files_list or f in test_files_list)] | |
| print(f'{json.dumps(cpu_test_files)}|{json.dumps(gpu_test_files)}') | |
| " | |
| test_files=$(python3 -c "$script") | |
| IFS='|' read -r cpu_test_files gpu_test_files <<< "$test_files" | |
| echo "cpu-files=$cpu_test_files" >> "$GITHUB_OUTPUT" | |
| echo "gpu-files=$gpu_test_files" >> "$GITHUB_OUTPUT" | |
| echo "Test files: $test_files" | |
| echo "CPU Test files: $cpu_test_files" | |
| echo "GPU Test files: $gpu_test_files" | |
| test_gpu: | |
| needs: | |
| - build-intel | |
| - build-amd | |
| - list-test-files | |
| - check-vm | |
| runs-on: self-hosted | |
| if: always() && !cancelled() && (needs.build-intel.result == 'success' || needs.build-amd.result == 'success') | |
| timeout-minutes: 30 | |
| container: | |
| image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:github-ci-v2 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 6 | |
| matrix: | |
| test_script: ${{ fromJSON(needs.list-test-files.outputs.gpu-files) }} | |
| steps: | |
| - name: Checkout Codes | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.repo }} | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: Show folder | |
| run: | | |
| ls -alh . || true | |
| ls -alh dist || true | |
| rm -rf dist/* || true | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: dist | |
| path: dist | |
| - name: Show dist folder | |
| run: ls -alh dist || true | |
| - name: Install wheel | |
| shell: bash | |
| run: | | |
| pip install bitblas parameterized uvicorn -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }} | |
| pip install dist/*.whl | |
| - name: Check platform | |
| shell: bash | |
| run: | | |
| if [[ "${{ runner.name }}" == *"intel"* ]]; then | |
| ip=${INTEL_SERVER} | |
| else | |
| ip=${AMD_SERVER} | |
| fi | |
| echo "GPU_IP=$ip" >> $GITHUB_ENV | |
| - name: Find suitable GPU | |
| shell: bash | |
| run: | | |
| gpu_id=-1 | |
| while [ "$gpu_id" -lt 0 ]; do | |
| gpu_id=$(curl -s "http://${{ env.GPU_IP }}/gpu/get?id=${{ github.run_id }}") | |
| if [ "$gpu_id" -lt 0 ]; then | |
| echo "http://${{ env.GPU_IP }}/gpu/get?id=${{ github.run_id }} returned $gpu_id" | |
| echo "No available GPU, waiting 5 seconds..." | |
| sleep 5 | |
| else | |
| echo "Allocated GPU ID: $gpu_id" | |
| fi | |
| done | |
| echo "CUDA_VISIBLE_DEVICES=$gpu_id" >> $GITHUB_ENV | |
| echo "CUDA_VISIBLE_DEVICES set to $gpu_id" | |
| - name: Run tests | |
| if: ${{ !github.event.inputs.test_names || contains(github.event.inputs.test_names, matrix.test_script) }} | |
| run: pytest --durations=0 tests/${{ matrix.test_script }}.py | |
| - name: Release GPU | |
| if: always() | |
| shell: bash | |
| run: curl -X GET "http://${{ env.GPU_IP }}/gpu/release?id=${{ github.run_id }}&gpu=$CUDA_VISIBLE_DEVICES" | |
| test_cpu: | |
| needs: | |
| - build-intel | |
| - build-amd | |
| - list-test-files | |
| - check-vm | |
| runs-on: self-hosted | |
| if: always() && (needs.build-intel.result == 'success' || needs.build-amd.result == 'success') | |
| container: | |
| image: ${{ needs.check-vm.outputs.ip }}:5000/modelcloud/gptqmodel:github-ci-v2 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_script: ${{ fromJSON(needs.list-test-files.outputs.cpu-files) }} | |
| steps: | |
| - name: Checkout Codes | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.repo }} | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: dist | |
| path: dist | |
| - name: Install wheel | |
| shell: bash | |
| run: | | |
| pip install dist/*.whl -i http://${{ needs.check-vm.outputs.ip }}/simple/ --trusted-host ${{ needs.check-vm.outputs.ip }} | |
| - name: Run tests | |
| if: ${{ !github.event.inputs.test_names || contains(github.event.inputs.test_names, matrix.test_script) }} | |
| run: pytest --durations=0 tests/${{ matrix.test_script }}.py |