From 561e64d03a7cf38fe403c1c6a7c5a148c8614afd Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Mon, 20 Feb 2023 23:12:55 +0000 Subject: [PATCH 1/5] Add coverage testing --- .github/workflows/ci-tests.yml | 11 ++++++++++- docs/src/whatsnew/latest.rst | 6 ++++++ lib/iris/tests/runner/_runner.py | 18 ++++++------------ noxfile.py | 12 +++++++++--- pyproject.toml | 17 +++++++++++++++++ requirements/ci/py310.yml | 1 + 6 files changed, 49 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index cee98dc33d..19c73c10b8 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -36,8 +36,12 @@ jobs: matrix: os: ["ubuntu-latest"] python-version: ["3.10"] - session: ["tests", "doctest", "gallery", "linkcheck"] + session: ["doctest", "gallery", "linkcheck"] include: + - os: "ubuntu-latest" + python-version: "3.10" + session: "tests" + coverage: True - os: "ubuntu-latest" python-version: "3.9" session: "tests" @@ -132,5 +136,10 @@ jobs: - name: "iris ${{ matrix.session }}" env: PY_VER: ${{ matrix.python-version }} + COVERAGE: ${{ matrix.coverage }} run: | nox --session ${{ matrix.session }} -- --verbose + + - name: Upload coverage report + uses: codecov/codecov-action@v3 + if: ${{ matrix.coverage }} \ No newline at end of file diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index b23687af7a..8696adeaaf 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -109,6 +109,12 @@ This document explains the changes made to Iris for this release #. `@rcomer`_ removed some old infrastructure that printed test timings. (:pull:`5101`) +#. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) added coverage testing. This can + be enabled by setting the environment variable ``COVERAGE=True`` when running + the tests with nox, i.e. ``nox --session tests``. (:pull:`4765`) + +#. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) removed the ``--coding-tests`` + option from Iris' test runner. (:pull:`4765`) .. comment Whatsnew author names (@github name) in alphabetical order. Note that, diff --git a/lib/iris/tests/runner/_runner.py b/lib/iris/tests/runner/_runner.py index bfb2cc2402..7f9439d4b6 100644 --- a/lib/iris/tests/runner/_runner.py +++ b/lib/iris/tests/runner/_runner.py @@ -35,18 +35,13 @@ class TestRunner: ("system-tests", "s", "Run the limited subset of system tests."), ("gallery-tests", "e", "Run the gallery code tests."), ("default-tests", "d", "Run the default tests."), - ( - "coding-tests", - "c", - "Run the coding standards tests. (These are a " - "subset of the default tests.)", - ), ( "num-processors=", "p", "The number of processors used for running " "the tests.", ), ("create-missing", "m", "Create missing test result files."), + ("coverage", "c", "Enable coverage testing"), ] boolean_options = [ "no-data", @@ -54,8 +49,8 @@ class TestRunner: "stop", "gallery-tests", "default-tests", - "coding-tests", "create-missing", + "coverage", ] def initialize_options(self): @@ -64,9 +59,9 @@ def initialize_options(self): self.system_tests = False self.gallery_tests = False self.default_tests = False - self.coding_tests = False self.num_processors = None self.create_missing = False + self.coverage = False def finalize_options(self): # These environment variables will be propagated to all the @@ -84,8 +79,6 @@ def finalize_options(self): tests.append("system") if self.default_tests: tests.append("default") - if self.coding_tests: - tests.append("coding") if self.gallery_tests: tests.append("gallery") if not tests: @@ -109,8 +102,6 @@ def run(self): tests.append("lib/iris/tests/system_test.py") if self.default_tests: tests.append("lib/iris/tests") - if self.coding_tests: - tests.append("lib/iris/tests/test_coding_standards.py") if self.gallery_tests: import iris.config @@ -136,6 +127,9 @@ def run(self): if self.stop: args.append("-x") + if self.coverage: + args.extend(["--cov=lib/iris", "--cov-report=xml"]) + result = True for test in tests: args[0] = test diff --git a/noxfile.py b/noxfile.py index 8aabf862fb..85806ab558 100755 --- a/noxfile.py +++ b/noxfile.py @@ -28,6 +28,7 @@ #: Cirrus-CI environment variable hook. PY_VER = os.environ.get("PY_VER", _PY_VERSIONS_ALL) +COVERAGE = os.environ.get("COVERAGE", False) #: Default cartopy cache directory. CARTOPY_CACHE_DIR = os.environ.get("HOME") / Path(".local/share/cartopy") @@ -176,6 +177,9 @@ def tests(session: nox.sessions.Session): """ Perform iris system, integration and unit tests. + Coverage testing is enabled if the COVERAGE environment variable is set to + True. + Parameters ---------- session: object @@ -185,13 +189,15 @@ def tests(session: nox.sessions.Session): prepare_venv(session) session.install("--no-deps", "--editable", ".") session.env.update(ENV) - session.run( + run_args = [ "python", "-m", "iris.tests.runner", "--default-tests", - "--system-tests", - ) + ] + if COVERAGE: + run_args.append("--coverage") + session.run(*run_args) @nox.session(python=_PY_VERSION_DOCSBUILD, venv_backend="conda") diff --git a/pyproject.toml b/pyproject.toml index bdb8a431e5..fdbeff7809 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,3 +46,20 @@ verbose = "False" [tool.pytest.ini_options] addopts = "-ra" testpaths = "lib/iris" + +[tool.coverage.run] +branch = true +source = [ + "lib/iris", +] +omit = [ + "lib/iris/tests/*", + "lib/iris/etc/*", +] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if __name__ == .__main__.:" +] \ No newline at end of file diff --git a/requirements/ci/py310.yml b/requirements/ci/py310.yml index 72d353c5c8..ae66d66a77 100644 --- a/requirements/ci/py310.yml +++ b/requirements/ci/py310.yml @@ -39,6 +39,7 @@ dependencies: - pre-commit - psutil - pytest + - pytest-cov - pytest-xdist - requests From a518b0f1e4dab747794265a394c9cd9024ed223f Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Tue, 21 Feb 2023 11:17:10 +0000 Subject: [PATCH 2/5] Update lockfile --- requirements/ci/nox.lock/py310-linux-64.lock | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/requirements/ci/nox.lock/py310-linux-64.lock b/requirements/ci/nox.lock/py310-linux-64.lock index 9dd269eff6..0d97158f45 100644 --- a/requirements/ci/nox.lock/py310-linux-64.lock +++ b/requirements/ci/nox.lock/py310-linux-64.lock @@ -1,6 +1,6 @@ # Generated by conda-lock. # platform: linux-64 -# input_hash: b3eba68adec85dc6750f8775b6e48189678a75c3baa3dc3f6ed9b9351137fe50 +# input_hash: f8af5f4aafcb766f463a1a897d3dab9e04f05f1494bced5931d78175ca0c66df @EXPLICIT https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81 https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2022.12.7-ha878542_0.conda#ff9f73d45c4a07d6f424495288a26080 @@ -150,7 +150,7 @@ https://conda.anaconda.org/conda-forge/linux-64/kiwisolver-1.4.4-py310hbf28c38_1 https://conda.anaconda.org/conda-forge/linux-64/lcms2-2.14-hfd0df8a_1.conda#c2566c2ea5f153ddd6bf4acaf7547d97 https://conda.anaconda.org/conda-forge/linux-64/libclang13-15.0.7-default_h3e3d535_1.conda#a3a0f7a6f0885f5e1e0ec691566afb77 https://conda.anaconda.org/conda-forge/linux-64/libcups-2.3.3-h36d4200_3.conda#c9f4416a34bc91e0eb029f912c68f81f -https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.88.0-hdc1c0ab_0.conda#c44acb3847ff118c068b662aff858afd +https://conda.anaconda.org/conda-forge/linux-64/libcurl-7.88.1-hdc1c0ab_0.conda#81eaeb3b35163c8e90e57532bc93754d https://conda.anaconda.org/conda-forge/linux-64/libpq-15.2-hb675445_0.conda#4654b17eccaba55b8581d6b9c77f53cc https://conda.anaconda.org/conda-forge/linux-64/libsystemd0-252-h2a991cd_0.tar.bz2#3c5ae9f61f663b3d5e1bf7f7da0c85f5 https://conda.anaconda.org/conda-forge/linux-64/libwebp-1.2.4-h1daa5a0_1.conda#77003f63d1763c1e6569a02c1742c9f4 @@ -198,7 +198,8 @@ https://conda.anaconda.org/conda-forge/linux-64/cairo-1.16.0-ha61ee94_1014.tar.b https://conda.anaconda.org/conda-forge/linux-64/cffi-1.15.1-py310h255011f_3.conda#800596144bb613cd7ac58b80900ce835 https://conda.anaconda.org/conda-forge/linux-64/cftime-1.6.2-py310hde88566_1.tar.bz2#94ce7a76b0c912279f6958e0b6b21d2b https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.0.7-py310hdf3cbec_0.conda#7bf9d8c765b6b04882c719509652c6d6 -https://conda.anaconda.org/conda-forge/linux-64/curl-7.88.0-hdc1c0ab_0.conda#5d9ac94ee84305ada32c3d287d0ec602 +https://conda.anaconda.org/conda-forge/linux-64/coverage-7.1.0-py310h1fa729e_0.conda#da7c45dbe780f5e162011a3af44e5009 +https://conda.anaconda.org/conda-forge/linux-64/curl-7.88.1-hdc1c0ab_0.conda#1968e4fef727858ac04746560e820928 https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.38.0-py310h5764c6d_1.tar.bz2#12ebe92a8a578bc903bd844744f4d040 https://conda.anaconda.org/conda-forge/linux-64/glib-2.74.1-h6239696_1.tar.bz2#f3220a9e9d3abcbfca43419a219df7e4 https://conda.anaconda.org/conda-forge/linux-64/hdf5-1.12.2-mpi_mpich_h5d83325_1.conda#811c4d55cf17b42336ffa314239717b0 @@ -234,6 +235,7 @@ https://conda.anaconda.org/conda-forge/linux-64/pandas-1.5.3-py310h9b08913_0.con https://conda.anaconda.org/conda-forge/noarch/platformdirs-3.0.0-pyhd8ed1ab_0.conda#c34694044915d7f291ef257029f2e2af https://conda.anaconda.org/conda-forge/linux-64/pyproj-3.4.1-py310h15e2413_1.conda#5be35366687def87437d210fd673100c https://conda.anaconda.org/conda-forge/linux-64/pyqt5-sip-12.11.0-py310heca2aa9_3.conda#3b1946b676534472ce65181dda0b9554 +https://conda.anaconda.org/conda-forge/noarch/pytest-cov-4.0.0-pyhd8ed1ab_0.tar.bz2#c9e3f8bfdb9bfc34aa1836a6ed4b25d7 https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.2.0-pyhd8ed1ab_0.conda#70ab87b96126f35d1e68de2ad9fb6423 https://conda.anaconda.org/conda-forge/noarch/setuptools-scm-7.1.0-pyhd8ed1ab_0.conda#6613dbb3b25cc648a107f33ca9f80fc1 https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-napoleon-0.7-py_0.tar.bz2#0bc25ff6f2e34af63ded59692df5f749 @@ -243,7 +245,7 @@ https://conda.anaconda.org/conda-forge/noarch/identify-2.5.18-pyhd8ed1ab_0.conda https://conda.anaconda.org/conda-forge/noarch/nc-time-axis-1.4.1-pyhd8ed1ab_0.tar.bz2#281b58948bf60a2582de9e548bcc5369 https://conda.anaconda.org/conda-forge/linux-64/netcdf-fortran-4.6.0-mpi_mpich_h1e13492_2.conda#d4ed7704f0fa589e4d7656780fa87557 https://conda.anaconda.org/conda-forge/linux-64/netcdf4-1.6.2-nompi_py310h55e1e36_100.tar.bz2#4dd7aa28fb7d9a6de061c9579a30e7dd -https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.12-hd33c08f_1.conda#667dc93c913f0156e1237032e3a22046 +https://conda.anaconda.org/conda-forge/linux-64/pango-1.50.13-hd33c08f_0.conda#e3b13445b8ee9d6a3d53a714f89ccd76 https://conda.anaconda.org/conda-forge/linux-64/parallelio-2.5.10-mpi_mpich_h862c5c2_100.conda#56e43c5226670aa0943fae9a2628a934 https://conda.anaconda.org/conda-forge/noarch/pyopenssl-23.0.0-pyhd8ed1ab_0.conda#d41957700e83bbb925928764cb7f8878 https://conda.anaconda.org/conda-forge/noarch/virtualenv-20.19.0-pyhd8ed1ab_0.conda#afaa9bf6992f67a82d75fad47a93ec84 From 8d4384ee5e580a20f575e35b512a9520651d00e3 Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Tue, 21 Feb 2023 11:39:27 +0000 Subject: [PATCH 3/5] Include __repr__ in coverage --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fdbeff7809..b44187191b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,5 @@ omit = [ [tool.coverage.report] exclude_lines = [ "pragma: no cover", - "def __repr__", "if __name__ == .__main__.:" -] \ No newline at end of file +] From bea9a6837d4adabd60ba6ae10a8257c7daa41517 Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Wed, 22 Feb 2023 17:19:52 +0000 Subject: [PATCH 4/5] use posargs for coverage flag instead of env var --- .github/workflows/ci-tests.yml | 7 +++---- noxfile.py | 6 ++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 19c73c10b8..81f5132ccf 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -41,7 +41,7 @@ jobs: - os: "ubuntu-latest" python-version: "3.10" session: "tests" - coverage: True + coverage: "--coverage" - os: "ubuntu-latest" python-version: "3.9" session: "tests" @@ -136,10 +136,9 @@ jobs: - name: "iris ${{ matrix.session }}" env: PY_VER: ${{ matrix.python-version }} - COVERAGE: ${{ matrix.coverage }} run: | - nox --session ${{ matrix.session }} -- --verbose + nox --session ${{ matrix.session }} -- --verbose ${{ matrix.coverage }} - name: Upload coverage report uses: codecov/codecov-action@v3 - if: ${{ matrix.coverage }} \ No newline at end of file + if: ${{ matrix.coverage }} diff --git a/noxfile.py b/noxfile.py index 85806ab558..c7b0a0e05b 100755 --- a/noxfile.py +++ b/noxfile.py @@ -28,7 +28,6 @@ #: Cirrus-CI environment variable hook. PY_VER = os.environ.get("PY_VER", _PY_VERSIONS_ALL) -COVERAGE = os.environ.get("COVERAGE", False) #: Default cartopy cache directory. CARTOPY_CACHE_DIR = os.environ.get("HOME") / Path(".local/share/cartopy") @@ -177,8 +176,7 @@ def tests(session: nox.sessions.Session): """ Perform iris system, integration and unit tests. - Coverage testing is enabled if the COVERAGE environment variable is set to - True. + Coverage testing is enabled if the "--coverage" or "-c" flag is used. Parameters ---------- @@ -195,7 +193,7 @@ def tests(session: nox.sessions.Session): "iris.tests.runner", "--default-tests", ] - if COVERAGE: + if "-c" in session.posargs or "--coverage" in session.posargs: run_args.append("--coverage") session.run(*run_args) From 65449ed6b44e50db783cf347cbbef9c9bd2e0669 Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Wed, 22 Feb 2023 17:31:35 +0000 Subject: [PATCH 5/5] update whatsnew to mentions posargs --- docs/src/whatsnew/latest.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 8696adeaaf..c5c3b2d173 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -109,9 +109,9 @@ This document explains the changes made to Iris for this release #. `@rcomer`_ removed some old infrastructure that printed test timings. (:pull:`5101`) -#. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) added coverage testing. This can - be enabled by setting the environment variable ``COVERAGE=True`` when running - the tests with nox, i.e. ``nox --session tests``. (:pull:`4765`) +#. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) added coverage testing. This + can be enabled by using the "--coverage" flag when running the tests with + nox i.e. ``nox --session tests -- --coverage``. (:pull:`4765`) #. `@lbdreyer`_ and `@trexfeathers`_ (reviewer) removed the ``--coding-tests`` option from Iris' test runner. (:pull:`4765`)