From e3d1469d2db60403c96c47c3183746d105877ba5 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 26 Aug 2024 12:33:02 +0200 Subject: [PATCH] Improve coverage reporting - Add `cov.pth` file and `COVERAGE_PROCESS_START` to track coverage of subprocesses and e2e tests - e2e tests set `COVERAGE_PROCESS_START` and run `command_pre` hooks - Configure mapping of file paths for coverage - Add step to combine coverage results in GHA - Use plain `coverage` instead of `pytest-cov`. The pytest plugin interfers with parallel coverage collection. - Fix `test_dependencies` to actually run tests Signed-off-by: Christian Heimes --- .github/workflows/test.yaml | 59 +++++++++++++++++++++++++++++++++++++ .gitignore | 1 + e2e/common.sh | 8 +++-- pyproject.toml | 28 ++++++++++++++++++ requirements-test.txt | 5 ++-- tests/test_dependencies.py | 2 ++ tox.ini | 27 +++++++++++++---- 7 files changed, 120 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a6a313e4..e2b4f95e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,6 +45,13 @@ jobs: working-directory: ./e2e/pyo3_test/ run: tox -e py + - name: Upload coverage + uses: actions/upload-artifact@v4 + with: + name: coverage-unit-py${{ matrix.python-version }}-rs${{ matrix.rust-version }} + path: .coverage.* + if-no-files-found: ignore + e2e: name: e2e runs-on: ${{ matrix.os }} @@ -107,3 +114,55 @@ jobs: with: name: ${{ matrix.test-script }}-py${{ matrix.python-version }}-rs${{ matrix.rust-version }}-${{ matrix.os }} path: e2e-output + + - name: Upload coverage + uses: actions/upload-artifact@v4 + with: + name: coverage-e2e-${{ matrix.test-script }}-py${{ matrix.python-version }}-rs${{ matrix.rust-version }}-${{ matrix.os }} + path: .coverage.* + if-no-files-found: ignore + + coverage: + name: Coverage report + runs-on: ubuntu-latest + needs: + - unit + - e2e + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + cache: pip + cache-dependency-path: | + **/pyproject.toml + **/tox.ini + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install coverage[toml] + + - name: Download coverage data + uses: actions/download-artifact@v4 + with: + pattern: coverage-* + merge-multiple: true + + - name: Coverage report + run: | + coverage combine + coverage html + coverage report --format=markdown >> $GITHUB_STEP_SUMMARY + coverage report --fail-under=60 + + - name: Upload report + uses: actions/upload-artifact@v4 + with: + path: htmlcov + name: htmlcov diff --git a/.gitignore b/.gitignore index 1986ca1a..53f2bc81 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ **/*.egg-info/ /src/fromager/version.py /.coverage +.coverage.* /artifacts /build-logs/ /.pypirc diff --git a/e2e/common.sh b/e2e/common.sh index f4ffb8d9..1a4446b8 100644 --- a/e2e/common.sh +++ b/e2e/common.sh @@ -8,6 +8,9 @@ set -o pipefail SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" OUTDIR="$(dirname "$SCRIPTDIR")/e2e-output" +# coverage reporting +export COVERAGE_PROCESS_START="$( dirname "$SCRIPTDIR" )/pyproject.toml" + # Recreate output directory rm -rf "$OUTDIR" mkdir "$OUTDIR" @@ -19,7 +22,8 @@ OUTDIR=$(cd "$OUTDIR" && pwd) mkdir -p "$OUTDIR/build-logs" # Recreate the virtualenv with fromager installed -tox -e e2e -n -r +# command_pre hook creates cov.pth +tox -e e2e -r source .tox/e2e/bin/activate # Set a variable to constrain packages used in the tests @@ -50,4 +54,4 @@ start_local_wheel_server() { IP=$(ipconfig getifaddr en0) fi export WHEEL_SERVER_URL="http://${IP}:9999/simple" -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index 8c735c2f..484bc2db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,34 @@ download_source = "fromager.sources:default_download_source" build_sdist = "fromager.sources:default_build_sdist" build_wheel = "fromager.wheels:default_build_wheel" +[tool.coverage.run] +branch = true +parallel = true +relative_files = true +source = [ + "fromager", + "tests/", +] + +[tool.coverage.paths] +source = [ + "src/fromager", + ".tox/**/site-packages/fromager", +] +tests =[ + "tests/", +] + +[tool.coverage.report] +show_missing = true +skip_covered = true +exclude_lines = [ + "pragma: no cover", + "@abc.abstractmethod", + "@typing.overload", + "if typing.TYPE_CHECKING", +] + [tool.setuptools_scm] version_file = "src/fromager/version.py" diff --git a/requirements-test.txt b/requirements-test.txt index 4de42dba..a0d472fa 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,5 @@ -coverage!=4.4,>=4.0 +coverage[toml]!=4.4,>=4.0 pytest -pytest-cov requests-mock setuptools_scm>=8 -setuptools>=64 \ No newline at end of file +setuptools>=64 diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 7b615bcd..706aeb3a 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -62,6 +62,8 @@ def _with_cleanup(*args, **kwds): ): shutil.rmtree(d) + return _with_cleanup + @_clean_build_artifacts def test_get_build_system_dependencies(tmp_context: context.WorkContext): diff --git a/tox.ini b/tox.ini index 198e6394..145d06bb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,17 +1,30 @@ [tox] minversion = 3.2.0 -envlist=py,linter,mypy +envlist=py,linter,mypy,coverage-report [testenv] extras = test +set_env = + COVERAGE_PROCESS_START={toxinidir}/pyproject.toml +commands_pre = + py,py3{11,12},e2e: {envpython} -c 'import pathlib; pathlib.Path("{env_site_packages_dir}/cov.pth").write_text("import coverage; coverage.process_startup()")' commands = - python -m pytest \ - --cov=fromager \ - --cov-report term-missing \ + python -m coverage run -m pytest \ --log-level DEBUG \ -vv \ {posargs:tests} +[testenv:coverage-report] +description = Report coverage over all test runs. +basepython = py312 +depends = py,py3{11,12},e2e +deps = coverage[toml] +skip_install = true +parallel_show_output = true +commands = + coverage combine + coverage report + [testenv:linter] base_python=python3.11 deps= @@ -41,7 +54,11 @@ commands = fromager {posargs} [testenv:e2e] -deps = . +deps = + . + coverage[toml] +# empty commands +commands = [testenv:mypy] description = Python type checking with mypy