add time out assert data collection and presentation #14322
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🧪 test | |
| on: | |
| push: | |
| paths-ignore: | |
| - '**.md' | |
| branches: | |
| - long_lived/** | |
| - main | |
| - release/** | |
| tags: | |
| - '**' | |
| pull_request: | |
| paths-ignore: | |
| - '**.md' | |
| branches: | |
| - '**' | |
| workflow_dispatch: null | |
| concurrency: | |
| group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}--${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/heads/long_lived/')) && github.sha || '' }} | |
| cancel-in-progress: true | |
| jobs: | |
| configure: | |
| name: Configure matrix | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Python environment | |
| uses: Chia-Network/actions/setup-python@main | |
| with: | |
| python-version: '3.10' | |
| - name: Generate matrix configuration | |
| id: configure | |
| run: | | |
| python tests/build-job-matrix.py --per directory --verbose --only plot_sync/ > matrix.json | |
| cat matrix.json | |
| echo configuration=$(cat matrix.json) >> "$GITHUB_OUTPUT" | |
| echo matrix_mode=${{ ( github.event_name == 'workflow_dispatch' ) && 'all' || ( github.repository_owner == 'Chia-Network' && github.repository != 'Chia-Network/chia-blockchain' ) && 'limited' || ( github.repository_owner == 'Chia-Network' && github.repository == 'Chia-Network/chia-blockchain' && github.ref == 'refs/heads/main' ) && 'main' || ( github.repository_owner == 'Chia-Network' && github.repository == 'Chia-Network/chia-blockchain' && startsWith(github.ref, 'refs/heads/release/') ) && 'all' || ( github.repository_owner == 'Chia-Network' && github.repository == 'Chia-Network/chia-blockchain' && startsWith(github.base_ref, 'release/') ) && 'all' || 'main' }} >> "$GITHUB_OUTPUT" | |
| outputs: | |
| configuration: ${{ steps.configure.outputs.configuration }} | |
| matrix_mode: ${{ steps.configure.outputs.matrix_mode }} | |
| macos: | |
| uses: ./.github/workflows/test-single.yml | |
| needs: configure | |
| with: | |
| emoji: 🍎 | |
| matrix: macos | |
| name: macOS | |
| file_name: macos | |
| concurrency_name: macos | |
| configuration: ${{ needs.configure.outputs.configuration }} | |
| matrix_mode: ${{ needs.configure.outputs.matrix_mode }} | |
| runs-on: macos-latest | |
| ubuntu: | |
| uses: ./.github/workflows/test-single.yml | |
| needs: configure | |
| with: | |
| emoji: 🐧 | |
| matrix: ubuntu | |
| name: Ubuntu | |
| file_name: ubuntu | |
| concurrency_name: ubuntu | |
| configuration: ${{ needs.configure.outputs.configuration }} | |
| matrix_mode: ${{ needs.configure.outputs.matrix_mode }} | |
| runs-on: ubuntu-latest | |
| windows: | |
| uses: ./.github/workflows/test-single.yml | |
| needs: configure | |
| with: | |
| emoji: 🪟 | |
| matrix: windows | |
| name: Windows | |
| file_name: windows | |
| concurrency_name: windows | |
| configuration: ${{ needs.configure.outputs.configuration }} | |
| matrix_mode: ${{ needs.configure.outputs.matrix_mode }} | |
| runs-on: windows-latest | |
| coverage: | |
| name: ${{ matrix.os.emoji }} Coverage - ${{ matrix.python.name }} | |
| runs-on: ${{ matrix.os.runs-on }} | |
| needs: | |
| - macos | |
| - ubuntu | |
| - windows | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - emoji: 🐧 | |
| matrix: ubuntu | |
| name: Ubuntu | |
| runs-on: ubuntu-latest | |
| python: | |
| - name: '3.10' | |
| action: '3.10' | |
| apt: '3.10' | |
| install_sh: '3.10' | |
| matrix: '3.10' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set Env | |
| uses: Chia-Network/actions/setjobenv@main | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download Results | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: junit-data | |
| path: junit-data | |
| - name: Format JUnit data and prepare results | |
| run: | | |
| sudo snap install yq | |
| ls junit-data/*.xml | xargs --max-procs=10 --replace={} yq eval '.testsuites.testsuite.testcase |= sort_by(.+@classname, .+@name)' --inplace {} | |
| sudo apt-get install junitparser | |
| mkdir junit-results | |
| junitparser merge junit-data/*.xml junit-results/junit.xml | |
| ls junit-results/*.xml | xargs --max-procs=10 --replace={} yq eval '.testsuites.testsuite |= sort_by(.+@name) | .testsuites.testsuite[].testcase |= sort_by(.+@classname, .+@name)' --inplace {} | |
| - name: Publish formatted JUnit data | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: junit-data | |
| path: junit-data/* | |
| if-no-files-found: error | |
| - name: Publish JUnit results | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: junit-results | |
| path: junit-results/* | |
| if-no-files-found: error | |
| - name: Download Coverage | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: coverage-data | |
| path: coverage-data | |
| - name: Set up ${{ matrix.python.name }} | |
| uses: Chia-Network/actions/setup-python@main | |
| with: | |
| python-version: ${{ matrix.python.action }} | |
| - uses: ./.github/actions/install | |
| with: | |
| python-version: ${{ matrix.python.action }} | |
| development: true | |
| - uses: chia-network/actions/activate-venv@main | |
| - name: Add time out assert results to workflow summary | |
| if: always() | |
| run: | | |
| python -m tests.process_junit --type time_out_assert --xml junit-results/junit.xml --markdown --link-prefix ${{ github.event.repository.html_url }}/blob/${{ github.sha }}/ --link-line-separator \#L >> "$GITHUB_STEP_SUMMARY" | |
| - name: Coverage Processing | |
| run: | | |
| coverage combine --rcfile=.coveragerc --data-file=coverage-reports/.coverage coverage-data/ | |
| coverage xml --rcfile=.coveragerc --data-file=coverage-reports/.coverage -o coverage-reports/coverage.xml | |
| coverage html --rcfile=.coveragerc --data-file=coverage-reports/.coverage --directory coverage-reports/html/ | |
| - uses: coverallsapp/github-action@v2 | |
| if: always() | |
| env: | |
| COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} | |
| - name: Coverage report (chia/) | |
| if: always() | |
| run: | | |
| set -o pipefail | |
| coverage report --rcfile=.coveragerc --data-file=coverage-reports/.coverage --include='chia/**/*' --show-missing | tee coverage-reports/coverage-chia-stdout | |
| - name: Coverage report (tests/) | |
| if: always() | |
| run: | | |
| set -o pipefail | |
| coverage report --rcfile=.coveragerc --data-file=coverage-reports/.coverage --include='tests/**/*' --show-missing | tee coverage-reports/coverage-tests-stdout | |
| - name: Coverage report (diff) | |
| if: always() | |
| env: | |
| compare-branch: ${{ github.base_ref == '' && github.event.before || format('origin/{0}', github.base_ref) }} | |
| run: | | |
| diff-cover --compare-branch=${{ env.compare-branch }} --fail-under=100 --html-report=coverage-reports/diff-cover.html --markdown-report=coverage-reports/diff-cover.md coverage-reports/coverage.xml | tee coverage-reports/diff-cover-stdout | |
| COV_STATUS="${PIPESTATUS[0]}" | |
| echo "COV_STATUS=$COV_STATUS" >> "$GITHUB_ENV" | |
| if [[ $COV_STATUS != '0' ]]; then | |
| exit 1 | |
| fi | |
| - name: Remove previous coverage report comment and label from PR | |
| if: github.base_ref != '' && always() | |
| shell: bash | |
| env: | |
| COV_STATUS: ${{ env.COV_STATUS }} | |
| GH_TOKEN: ${{ github.token }} | |
| ORG_REPO: ${{ github.repository }} | |
| run: | | |
| PR_NUM=$(jq -r '.number' "$GITHUB_EVENT_PATH") | |
| COMMENTS=$(gh api -X GET /repos/"${ORG_REPO}"/issues/"${PR_NUM}"/comments) | |
| COMMENT_ID=$(echo "$COMMENTS" | jq '.[] | select(.user.login == "github-actions[bot]" and (.body | tostring | contains("<!-- COVERAGE_COMMENT_'${PR_NUM}' -->"))) | .id') | |
| COVERAGE_LABEL=$(gh pr view "$PR_NUM" --json labels --jq ' .labels[].name | select(. == "coverage-diff")') | |
| if [[ -n "$COMMENT_ID" ]]; then | |
| gh api -X DELETE /repos/"${ORG_REPO}"/issues/comments/"${COMMENT_ID}" | |
| fi | |
| if [[ "$COV_STATUS" != '0' ]]; then | |
| if [[ "$COVERAGE_LABEL" == "coverage-diff" ]]; then | |
| gh pr edit "$PR_NUM" --remove-label "coverage-diff" | |
| fi | |
| fi | |
| - name: Add diff coverage report comment to PR | |
| if: github.base_ref != '' && always() | |
| env: | |
| BRANCH_NAME: ${{ github.sha }} | |
| COV_STATUS: ${{ env.COV_STATUS }} | |
| GH_TOKEN: ${{ github.token }} | |
| ORG_REPO: ${{ github.repository }} | |
| run: | | |
| if [[ $COV_STATUS != '0' ]]; then | |
| BASE_URL="https://github.com/${ORG_REPO}/blob/${BRANCH_NAME}" | |
| PR_NUM=$(jq -r '.number' "$GITHUB_EVENT_PATH") | |
| COVERAGE_LABEL=$(gh pr view "$PR_NUM" --json labels --jq ' .labels[].name | select(. == "coverage-diff")') | |
| CONTENT=$(<coverage-reports/diff-cover-stdout) | |
| OUTPUT="<!-- COVERAGE_COMMENT_${PR_NUM} -->\n| File | Coverage | Missing Lines |\n|------|----------|---------------|\n" | |
| TOTAL_LINES="Unknown" | |
| MISSING_LINES="Unknown" | |
| TOTAL_COVERAGE="Unknown" | |
| IFS=$'\n' | |
| for LINE in $CONTENT; do | |
| if [[ $LINE =~ ^(.*\.py)\ \(([0-9\.]+%)\)\:\ Missing\ lines\ (.*)$ ]]; then | |
| FILE="${BASH_REMATCH[1]}" | |
| COVERAGE="${BASH_REMATCH[2]}" | |
| MISSING="${BASH_REMATCH[3]}" | |
| MISSING_TRANSFORMED="lines " | |
| IFS=',' read -ra MISSING_PARTS <<< "$MISSING" | |
| for PART in "${MISSING_PARTS[@]}"; do | |
| if [[ $PART =~ ^([0-9]+)\-([0-9]+)$ ]]; then | |
| MISSING_TRANSFORMED+="[${BASH_REMATCH[1]}-${BASH_REMATCH[2]}](${BASE_URL}/${FILE}#L${BASH_REMATCH[1]}-L${BASH_REMATCH[2]}), " | |
| else | |
| MISSING_TRANSFORMED+="[${PART}](${BASE_URL}/${FILE}#L${PART}), " | |
| fi | |
| done | |
| MISSING_TRANSFORMED=${MISSING_TRANSFORMED%, } | |
| OUTPUT+="| [\`${FILE}\`](${BASE_URL}/${FILE}) | ${COVERAGE} | ${MISSING_TRANSFORMED} |\n" | |
| elif [[ $LINE =~ ^Total:\ \ \ (.*)\ lines$ ]]; then | |
| TOTAL_LINES="${BASH_REMATCH[1]} lines" | |
| elif [[ $LINE =~ ^Missing:\ (.*)\ lines$ ]]; then | |
| MISSING_LINES="${BASH_REMATCH[1]} lines" | |
| elif [[ $LINE =~ ^Coverage:\ (.*)%$ ]]; then | |
| TOTAL_COVERAGE="${BASH_REMATCH[1]}%" | |
| fi | |
| done | |
| OUTPUT+="\n| Total | Missing | Coverage |\n|-------|---------|----------|\n| ${TOTAL_LINES} | ${MISSING_LINES} | ${TOTAL_COVERAGE} |\n" | |
| echo -e "$OUTPUT" > temp_comment.txt | |
| gh pr comment "$PR_NUM" -F temp_comment.txt | |
| if [[ $COVERAGE_LABEL != "coverage-diff" ]]; then | |
| gh pr edit "$PR_NUM" --add-label "coverage-diff" | |
| fi | |
| fi | |
| - name: Add diff coverage report to workflow summary | |
| if: always() | |
| run: | | |
| cat coverage-reports/diff-cover.md >> "$GITHUB_STEP_SUMMARY" | |
| - name: Publish coverage reports | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage-reports | |
| path: coverage-reports/* | |
| if-no-files-found: error |