CodeQL Unit Testing #3140
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CodeQL Unit Testing | |
| on: | |
| merge_group: | |
| types: [checks_requested] | |
| push: | |
| branches: | |
| - main | |
| - next | |
| - "rc/**" | |
| pull_request: | |
| branches: | |
| - main | |
| - next | |
| - "rc/**" | |
| jobs: | |
| prepare-unit-test-matrix: | |
| name: Prepare CodeQL unit test matrix | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| matrix: ${{ steps.export-unit-test-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Export unit test matrix | |
| id: export-unit-test-matrix | |
| run: | | |
| echo "Merging Result:" | |
| python scripts/create_language_matrix.py | |
| echo "matrix=$( | |
| python scripts/create_language_matrix.py | \ | |
| jq --compact-output 'map([.+{os: "ubuntu-latest-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}')" >> $GITHUB_OUTPUT | |
| run-test-suites: | |
| name: Run unit tests | |
| needs: prepare-unit-test-matrix | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.prepare-unit-test-matrix.outputs.matrix) }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| - name: Install Python dependencies | |
| run: pip install -r scripts/requirements.txt | |
| - name: Cache CodeQL | |
| id: cache-codeql | |
| uses: actions/cache@v3 | |
| with: | |
| # A list of files, directories, and wildcard patterns to cache and restore | |
| path: ${{github.workspace}}/codeql_home | |
| # An explicit key for restoring and saving the cache | |
| key: codeql-home-${{matrix.os}}-${{matrix.codeql_cli}}-${{matrix.codeql_standard_library}} | |
| - name: Install CodeQL | |
| if: steps.cache-codeql.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/install-codeql | |
| with: | |
| codeql-cli-version: ${{ matrix.codeql_cli }} | |
| codeql-stdlib-version: ${{ matrix.codeql_standard_library }} | |
| codeql-home: ${{ github.workspace }}/codeql_home | |
| add-to-path: false | |
| - name: Install CodeQL packs | |
| uses: ./.github/actions/install-codeql-packs | |
| with: | |
| cli_path: ${{ github.workspace }}/codeql_home/codeql | |
| - name: Pre-Compile Queries | |
| id: pre-compile-queries | |
| run: | | |
| ${{ github.workspace }}/codeql_home/codeql/codeql query compile --threads 0 ${{ matrix.language }} | |
| - name: Run test suites | |
| id: run-test-suites | |
| env: | |
| RUNNER_OS: ${{ runner.os }} | |
| RUNNER_TEMP: ${{ runner.temp }} | |
| CODEQL_CLI: ${{ matrix.codeql_cli }} | |
| CODEQL_STDLIB: ${{ matrix.codeql_standard_library }} | |
| CODEQL_STDLIB_IDENT: ${{matrix.codeql_standard_library_ident}} | |
| CODEQL_HOME: ${{ github.workspace }}/codeql_home | |
| shell: python | |
| run: | | |
| import os | |
| import sys | |
| import subprocess | |
| import re | |
| import json | |
| from pathlib import Path | |
| def print_error(fmt, *args): | |
| print(f"::error::{fmt}", *args) | |
| def print_error_and_fail(fmt, *args): | |
| print_error(fmt, args) | |
| sys.exit(1) | |
| cli_version = os.environ['CODEQL_CLI'] | |
| stdlib_ref = os.environ['CODEQL_STDLIB'] | |
| stdlib_ref_ident = os.environ['CODEQL_STDLIB_IDENT'] | |
| workspace = os.environ['GITHUB_WORKSPACE'] | |
| runner_os = os.environ['RUNNER_OS'] | |
| runner_temp = os.environ['RUNNER_TEMP'] | |
| codeql_home = os.environ['CODEQL_HOME'] | |
| codeql_bin = os.path.join(codeql_home, 'codeql', 'codeql') | |
| language_root = Path(workspace, '${{ matrix.language }}') | |
| test_roots = list(map(str, language_root.glob('*/test'))) | |
| for test_root in test_roots: | |
| print(f"Executing tests found (recursively) in the directory '{test_root}'") | |
| files_to_close = [] | |
| try: | |
| # XL runners have 8 cores, so split the tests into 8 "slices", and run one per thread | |
| num_slices = 8 | |
| procs = [] | |
| for slice in range(1, num_slices+1): | |
| test_report_path = os.path.join(runner_temp, "${{ matrix.language }}", f"test_report_{runner_os}_{cli_version}_{stdlib_ref_ident}_slice_{slice}_of_{num_slices}.json") | |
| os.makedirs(os.path.dirname(test_report_path), exist_ok=True) | |
| test_report_file = open(test_report_path, 'w') | |
| files_to_close.append(test_report_file) | |
| procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", *test_roots], stdout=test_report_file, stderr=subprocess.PIPE)) | |
| for p in procs: | |
| _, err = p.communicate() | |
| if p.returncode != 0: | |
| if p.returncode == 122: | |
| # Failed because a test case failed, so just print the regular output. | |
| # This will allow us to proceed to validate-test-results, which will fail if | |
| # any test cases failed | |
| print(f"{err.decode()}") | |
| else: | |
| # Some more serious problem occurred, so print and fail fast | |
| print_error_and_fail(f"Failed to run tests with return code {p.returncode}\n{err.decode()}") | |
| finally: | |
| for file in files_to_close: | |
| file.close() | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.language }}-test-results-${{ runner.os }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library_ident }} | |
| path: | | |
| ${{ runner.temp }}/${{ matrix.language }}/test_report_${{ runner.os }}_${{ matrix.codeql_cli }}_${{ matrix.codeql_standard_library_ident }}_slice_*.json | |
| if-no-files-found: error | |
| validate-test-results: | |
| name: Validate test results | |
| if: ${{ always() }} | |
| needs: run-test-suites | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Check if run-test-suites job failed to complete, if so fail | |
| if: ${{ needs.run-test-suites.result == 'failure' }} | |
| uses: actions/github-script@v3 | |
| with: | |
| script: | | |
| core.setFailed('Test run job failed') | |
| - name: Collect test results | |
| uses: actions/download-artifact@v4 | |
| - name: Validate test results | |
| run: | | |
| for json_report in *-test-results-*/test_report_* | |
| do | |
| jq --raw-output '"PASS \(map(select(.pass == true)) | length)/\(length)'" $json_report\"" "$json_report" | |
| done | |
| FAILING_TESTS=$(jq --raw-output '.[] | select(.pass == false)' *-test-results-*/test_report_*.json) | |
| if [[ ! -z "$FAILING_TESTS" ]]; then | |
| echo "ERROR: The following tests failed:" | |
| echo $FAILING_TESTS | jq . | |
| exit 1 | |
| fi |