Improve coverage reporting

- Add `cov.pth` file and `COVERAGE_PROCESS_START` to track coverage of
  subprocesses and e2e tests
- e2e tests set `COVERAGE_PROCESS_START` and run `command_pre` hooks
- Configure mapping of file paths for coverage
- Add step to combine coverage results in GHA
- Use plain `coverage` instead of `pytest-cov`. The pytest plugin
  interfers with parallel coverage collection.
- Fix `test_dependencies` to actually run tests

Signed-off-by: Christian Heimes <cheimes@redhat.com>

.github/workflows/test.yaml

@@ -45,6 +45,13 @@ jobs:
         working-directory: ./e2e/pyo3_test/
         run: tox -e py
 
+      - name: Upload coverage
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-unit-py${{ matrix.python-version }}-rs${{ matrix.rust-version }}
+          path: .coverage.*
+          if-no-files-found: ignore
+
   e2e:
     name: e2e
     runs-on: ${{ matrix.os }}
@@ -107,3 +114,55 @@ jobs:
         with:
           name: ${{ matrix.test-script }}-py${{ matrix.python-version }}-rs${{ matrix.rust-version }}-${{ matrix.os }}
           path: e2e-output
+
+      - name: Upload coverage
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-e2e-${{ matrix.test-script }}-py${{ matrix.python-version }}-rs${{ matrix.rust-version }}-${{ matrix.os }}
+          path: .coverage.*
+          if-no-files-found: ignore
+
+  coverage:
+    name: Coverage report
+    runs-on: ubuntu-latest
+    needs:
+      - unit
+      - e2e
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+          cache: pip
+          cache-dependency-path: |
+            **/pyproject.toml
+            **/tox.ini
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install coverage[toml]
+
+      - name: Download coverage data
+        uses: actions/download-artifact@v4
+        with:
+          pattern: coverage-*
+          merge-multiple: true
+
+      - name: Coverage report
+        run: |
+          coverage combine
+          coverage html
+          coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
+          coverage report --fail-under=60
+
+      - name: Upload report
+        uses: actions/upload-artifact@v4
+        with:
+          path: htmlcov
+          name: htmlcov

.gitignore

@@ -9,6 +9,7 @@ __pycache__/
 **/*.egg-info/
 /src/fromager/version.py
 /.coverage
+.coverage.*
 /artifacts
 /build-logs/
 /.pypirc

e2e/common.sh

@@ -8,6 +8,9 @@ set -o pipefail
 SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 OUTDIR="$(dirname "$SCRIPTDIR")/e2e-output"
 
+# coverage reporting
+export COVERAGE_PROCESS_START="$( dirname "$SCRIPTDIR" )/pyproject.toml"
+
 # Recreate output directory
 rm -rf "$OUTDIR"
 mkdir "$OUTDIR"
@@ -19,7 +22,8 @@ OUTDIR=$(cd "$OUTDIR" && pwd)
 mkdir -p "$OUTDIR/build-logs"
 
 # Recreate the virtualenv with fromager installed
-tox -e e2e -n -r
+# command_pre hook creates cov.pth
+tox -e e2e -r
 source .tox/e2e/bin/activate
 
 # Set a variable to constrain packages used in the tests
@@ -50,4 +54,4 @@ start_local_wheel_server() {
         IP=$(ipconfig getifaddr en0)
     fi
     export WHEEL_SERVER_URL="http://${IP}:9999/simple"
-}
+}

pyproject.toml

@@ -52,6 +52,34 @@ download_source = "fromager.sources:default_download_source"
 build_sdist = "fromager.sources:default_build_sdist"
 build_wheel = "fromager.wheels:default_build_wheel"
 
+[tool.coverage.run]
+branch = true
+parallel = true
+relative_files = true
+source = [
+    "fromager",
+    "tests/",
+]
+
+[tool.coverage.paths]
+source = [
+   "src/fromager",
+   ".tox/**/site-packages/fromager",
+]
+tests =[
+   "tests/",
+]
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = true
+exclude_lines = [
+    "pragma: no cover",
+    "@abc.abstractmethod",
+    "@typing.overload",
+    "if typing.TYPE_CHECKING",
+]
+
 [tool.setuptools_scm]
 version_file = "src/fromager/version.py"
 

requirements-test.txt

@@ -1,6 +1,5 @@
-coverage!=4.4,>=4.0
+coverage[toml]!=4.4,>=4.0
 pytest
-pytest-cov
 requests-mock
 setuptools_scm>=8
-setuptools>=64
+setuptools>=64

tests/test_dependencies.py

@@ -62,6 +62,8 @@ def _with_cleanup(*args, **kwds):
             ):
                 shutil.rmtree(d)
 
+    return _with_cleanup
+
 
 @_clean_build_artifacts
 def test_get_build_system_dependencies(tmp_context: context.WorkContext):

tox.ini

@@ -1,17 +1,30 @@
 [tox]
 minversion = 3.2.0
-envlist=py,linter,mypy
+envlist=py,linter,mypy,coverage-report
 
 [testenv]
 extras = test
+set_env =
+    COVERAGE_PROCESS_START={toxinidir}/pyproject.toml
+commands_pre =
+    py,py3{11,12},e2e: {envpython} -c 'import pathlib; pathlib.Path("{env_site_packages_dir}/cov.pth").write_text("import coverage; coverage.process_startup()")'
 commands =
-    python -m pytest \
-      --cov=fromager \
-      --cov-report term-missing \
+    python -m coverage run -m pytest \
       --log-level DEBUG \
       -vv \
       {posargs:tests}
 
+[testenv:coverage-report]
+description = Report coverage over all test runs.
+basepython = py312
+depends = py,py3{11,12},e2e
+deps = coverage[toml]
+skip_install = true
+parallel_show_output = true
+commands =
+    coverage combine
+    coverage report
+
 [testenv:linter]
 base_python=python3.11
 deps=
@@ -41,7 +54,11 @@ commands =
     fromager {posargs}
 
 [testenv:e2e]
-deps = .
+deps =
+    .
+    coverage[toml]
+# empty commands
+commands =
 
 [testenv:mypy]
 description = Python type checking with mypy