ctest does not run Python tests properly

[KrisThielemans]

We have

SIRF/src/xSTIR/pSTIR/tests/CMakeLists.txt

Lines 21 to 26 in 44ed358

	add_test(NAME PET_TESTS_PYTHON
	COMMAND ${Python_EXECUTABLE} -m pytest --cov=sirf.STIR --cov-config=.coveragerc-STIR src/xSTIR/pSTIR
	WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
	#add_test(NAME PET_DEMOS_PYTHON
	# COMMAND ${PYTHON_EXECUTABLE} run_all.py --non-interactive
	# WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}/examples/Python/PET")

Running it on my desktop gives

$ ctest --output-on-failure -VVV -R PET_TESTS_PY
Test project /home/kris/devel/buildCondaGT/builds/SIRF/build
    Start 9: PET_TESTS_PYTHON
1/1 Test #9: PET_TESTS_PYTHON .................   Passed   29.74 sec

100% tests passed, 0 tests failed out of 1

Running it manually in sources/STIR gives

F$ python -m pytest --verbosity=1 src/xSTIR/pSTIR/
======================================================================= test session starts ========================================================================
platform linux -- Python 3.12.4, pytest-8.2.2, pluggy-1.5.0 -- /home/kris/miniconda3/envs/cilsirfbuildGT/bin/python
cachedir: .pytest_cache
rootdir: /home/kris/devel/buildCondaGT/sources/SIRF
plugins: cov-5.0.0
collected 42 items

src/xSTIR/pSTIR/tests/test_ObjectiveFunction.py::TestSTIRObjectiveFunction::test_Hessian PASSED                                                              [  2%]
src/xSTIR/pSTIR/tests/test_ObjectiveFunction.py::TestSTIRObjectiveFunction::test_Poisson_loglikelihood_call PASSED                                           [  4%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_add_datacontainer PASSED                                                               [  7%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_add_scalar PASSED                                                                      [  9%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_divide_datacontainer PASSED                                                            [ 11%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_divide_scalar PASSED                                                                   [ 14%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_division_by_datacontainer_zero PASSED                                                  [ 16%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_division_by_scalar_zero PASSED                                                         [ 19%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_multiply_datacontainer PASSED                                                          [ 21%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_multiply_scalar PASSED                                                                 [ 23%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_sapyb_mixed PASSED                                                                     [ 26%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_sapyb_scalars PASSED                                                                   [ 28%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_sapyb_vectors PASSED                                                                   [ 30%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_subtract_datacontainer PASSED                                                          [ 33%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRImageDataAlgebra::test_subtract_scalar PASSED                                                                 [ 35%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_add_datacontainer PASSED                                                     [ 38%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_add_scalar PASSED                                                            [ 40%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_divide_datacontainer PASSED                                                  [ 42%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_divide_scalar PASSED                                                         [ 45%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_division_by_datacontainer_zero PASSED                                        [ 47%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_division_by_scalar_zero PASSED                                               [ 50%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_multiply_datacontainer PASSED                                                [ 52%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_multiply_scalar PASSED                                                       [ 54%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_sapyb_mixed PASSED                                                           [ 57%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_sapyb_scalars PASSED                                                         [ 59%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_sapyb_vectors PASSED                                                         [ 61%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_subtract_datacontainer PASSED                                                [ 64%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraFile::test_subtract_scalar PASSED                                                       [ 66%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_add_datacontainer PASSED                                                   [ 69%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_add_scalar PASSED                                                          [ 71%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_divide_datacontainer PASSED                                                [ 73%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_divide_scalar PASSED                                                       [ 76%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_division_by_datacontainer_zero PASSED                                      [ 78%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_division_by_scalar_zero PASSED                                             [ 80%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_multiply_datacontainer PASSED                                              [ 83%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_multiply_scalar PASSED                                                     [ 85%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_sapyb_mixed PASSED                                                         [ 88%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_sapyb_scalars PASSED                                                       [ 90%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_sapyb_vectors PASSED                                                       [ 92%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_subtract_datacontainer PASSED                                              [ 95%]
src/xSTIR/pSTIR/tests/test_algebra.py::TestSTIRAcquisitionDataAlgebraMemory::test_subtract_scalar PASSED                                                     [ 97%]
src/xSTIR/pSTIR/tests/test_six_adjoint.py::test_main PASSED                                                                                                  [100%]

========================================================================= warnings summary =========================================================================
src/xSTIR/pSTIR/tests/test_six_adjoint.py::test_main
  /home/kris/miniconda3/envs/cilsirfbuildGT/lib/python3.12/site-packages/_pytest/python.py:166: PytestReturnNotNoneWarning: Expected None, but src/xSTIR/pSTIR/tests/test_six_adjoint.py::test_main returned (0, 1), which will be an error in a future version of pytest.  Did you mean to use `assert` instead of `return`?
    warnings.warn(

-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html

But there are a lot of tests which are not yet unittest/pytests. Note that test_six_adjoint.py is picked up, presumably because it contains an AssertionError (Edit: because its filename matches test_*py)

Trying to run
https://github.com/SyneRBI/SIRF/blob/75364cb9367da94dabca8f79cbb976fcd5eb1d3a/src/xSTIR/pSTIR/tests/test_all.py fails as its glob pattern includes the unittests.

$ python test_all.py

--- running test_ObjectiveFunction.py
Traceback (most recent call last):
  File "/home/kris/devel/buildCondaGT/sources/SIRF/src/xSTIR/pSTIR/tests/test_all.py", line 42, in <module>
    f, n = eval(main)
           ^^^^^^^^^^
  File "<string>", line 1, in <module>
AttributeError: module 'test_ObjectiveFunction' has no attribute 'test_main'

Executing one by one flags up 2 problems:

$ python tests_two.py -v
+++ test 0 passed
+++ test 1 failed: expected 718.2, got 723.8 (tolerance 1.44)
...
+++ test 32 failed: expected 718.2, got 723.8 (tolerance 1.44)

$ python tests_five.py  -v
/home/kris/devel/buildCondaGT/sources/SIRF/src/xSTIR/pSTIR/tests/tests_five.py:33: DeprecatedWarning: petmr_data_path is deprecated as of 2.0.0 and will be removed in 4.0. use examples_data_path() instead
  data_path = petmr_data_path('pet')

[KrisThielemans]

Note that the "other" tests use runner

@casperdcl is there an easy way to let pytest pick up the runner-style tests? I suppose we could follow add an AssertionError on the number of failed tests.

In the main time, I'm going to create a work-around PR.

[KrisThielemans]

It's more complicated than I hoped, of course.

The reason that the "other" tests are not found is that pytest by default looks for files called test_*py and those files are called tests_*py.

Executing pytest src/xSTIR/pSTIR/tests/ does discover all the files, but we get 2 errors

===================================================================================================== ERRORS ======================================================================================================
__________________________________________________________________________________ ERROR at setup of test_data_container_algebra __________________________________________________________________________________
file /home/kris/devel/install/python/sirf/Utilities.py, line 591
  def test_data_container_algebra(test, x, eps=1e-5):
E       fixture 'test' not found
>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory
>       use 'pytest --fixtures [testpath]' for help on them.

/home/kris/devel/install/python/sirf/Utilities.py:591
_________________________________________________________________________________________ ERROR at setup of test_Hessian __________________________________________________________________________________________
file /home/kris/devel/buildCondaGT/sources/SIRF/src/xSTIR/pSTIR/tests/tests_qp_lc_rdp.py, line 24
  def test_Hessian(test, prior, x, eps=1e-3):
E       fixture 'test' not found
>       available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, cov, doctest_namespace, monkeypatch, no_cover, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory
>       use 'pytest --fixtures [testpath]' for help on them.

/home/kris/devel/buildCondaGT/sources/SIRF/src/xSTIR/pSTIR/tests/tests_qp_lc_rdp.py:24

This is because the naming of functions.