TEST: Run all tests through pytest (#2090)

* run legacy tests with pytest * add back pyargs to pytest call * more pyargs * try changing order of PYTHONPATH * try changing to relative imports instead of adding PYTHONPATH * missing delete * try executing non-mpi tests first to see if they also fail * try deleting __init__.py in test folder * try removing relative import * move patching test also to pytest, restore MPI on windows * fix unexecuted parametrization * remove unused import * better naming * add name of estimator being patched * account for platforms using non-standard process codes * correct fixture names * use yield instead of finalize * try removing pyargs to see what happens * add separate tests for global and module-specific * remove unused argument * remove global patching test * fixes for windows * try changing fixtures back to manual finalizers * add comment about the yield fixtures * Update .ci/scripts/test_global_patch.py Co-authored-by: Ian Faust <icfaust@gmail.com> --------- Co-authored-by: Ian Faust <icfaust@gmail.com>
uxlfoundation · Oct 14, 2024 · 614e088 · 614e088
1 parent 8883b39
commit 614e088
Show file tree

Hide file tree

Showing 6 changed files with 160 additions and 99 deletions.
diff --git a/.ci/scripts/test_global_patch.py b/.ci/scripts/test_global_patch.py
@@ -15,55 +15,136 @@
 # limitations under the License.
 # ===============================================================================
 
+import os
 import subprocess
 import sys
 
-# test patching from command line
-err_code = subprocess.call(
-    [sys.executable, "-m", "sklearnex.glob", "patch_sklearn", "-a", "svc"]
-)
-assert not err_code
-from sklearn.svm import SVC, SVR
+import pytest
 
-assert SVC.__module__.startswith("daal4py") or SVC.__module__.startswith("sklearnex")
-assert not SVR.__module__.startswith("daal4py") and not SVR.__module__.startswith(
-    "sklearnex"
-)
+# This is a workaround for older versions of Python on Windows
+# which didn't have it as part of the built-in 'os' module.
+EX_OK = os.EX_OK if hasattr(os, "EX_OK") else 0
 
+# Note: from the structure of this file, one might think of adding a test
 
-from sklearnex import patch_sklearn, unpatch_sklearn
+# along the lines of 'test_patching_all_from_command_line'. There is however
+# an issue in that, after the first time a scikit-learn module is imported,
+# further calls to 'patch_sklearn' with different arguments will have no effect
+# since sklearn is already imported. Reloading it through 'importlib.reload'
+# or deleting it from 'sys.modules' doesn't appear to have the intended effect
+# either. This also makes this first command-line fixture and tests that use
+# it not entirely idempotent, given that they need to import sklearn modules.
 
-# test unpatching from command line
-err_code = subprocess.call([sys.executable, "-m", "sklearnex.glob", "unpatch_sklearn"])
-assert not err_code
-unpatch_sklearn()
-from sklearn.svm import SVC, SVR
+# Note 2: don't try to change these into 'yield' fixtures, because otherwise
+# some test runners on windows which use multi-processing will throw errors
+# about the fixtures not being serializable.
 
-assert not SVC.__module__.startswith("daal4py") and not SVC.__module__.startswith(
-    "sklearnex"
-)
-assert not SVR.__module__.startswith("daal4py") and not SVR.__module__.startswith(
-    "sklearnex"
-)
 
+@pytest.fixture
+def patch_svc_from_command_line(request):
+    err_code = subprocess.call(
+        [sys.executable, "-m", "sklearnex.glob", "patch_sklearn", "-a", "svc"]
+    )
+    assert err_code == EX_OK
 
-# test patching from function
-patch_sklearn(name=["svc"], global_patch=True)
-from sklearn.svm import SVC, SVR
+    def finalizer():
+        err_code = subprocess.call(
+            [sys.executable, "-m", "sklearnex.glob", "unpatch_sklearn", "-a", "svc"]
+        )
+        assert err_code == EX_OK
 
-assert SVC.__module__.startswith("daal4py") or SVC.__module__.startswith("sklearnex")
-assert not SVR.__module__.startswith("daal4py") and not SVR.__module__.startswith(
-    "sklearnex"
-)
+    request.addfinalizer(finalizer)
+    return
 
 
-# test unpatching from function
-unpatch_sklearn(global_unpatch=True)
-from sklearn.svm import SVC, SVR
+def test_patching_svc_from_command_line(patch_svc_from_command_line):
+    from sklearn.svm import SVC, SVR
 
-assert not SVC.__module__.startswith("daal4py") and not SVC.__module__.startswith(
-    "sklearnex"
-)
-assert not SVR.__module__.startswith("daal4py") and not SVR.__module__.startswith(
-    "sklearnex"
-)
+    assert SVC.__module__.startswith("daal4py") or SVC.__module__.startswith("sklearnex")
+    assert not SVR.__module__.startswith("daal4py") and not SVR.__module__.startswith(
+        "sklearnex"
+    )
+
+
+def test_unpatching_svc_from_command_line(patch_svc_from_command_line):
+    err_code = subprocess.call(
+        [sys.executable, "-m", "sklearnex.glob", "unpatch_sklearn"]
+    )
+    assert err_code == EX_OK
+    from sklearnex import unpatch_sklearn
+
+    unpatch_sklearn()
+    from sklearn.svm import SVC, SVR
+
+    assert not SVC.__module__.startswith("daal4py") and not SVC.__module__.startswith(
+        "sklearnex"
+    )
+    assert not SVR.__module__.startswith("daal4py") and not SVR.__module__.startswith(
+        "sklearnex"
+    )
+
+
+@pytest.fixture
+def patch_svc_from_function(request):
+    from sklearnex import patch_sklearn, unpatch_sklearn
+
+    patch_sklearn(name=["svc"], global_patch=True)
+
+    def finalizer():
+        unpatch_sklearn(name=["svc"], global_unpatch=True)
+
+    request.addfinalizer(finalizer)
+    return
+
+
+def test_patching_svc_from_function(patch_svc_from_function):
+    from sklearn.svm import SVC, SVR
+
+    assert SVC.__module__.startswith("daal4py") or SVC.__module__.startswith("sklearnex")
+    assert not SVR.__module__.startswith("daal4py") and not SVR.__module__.startswith(
+        "sklearnex"
+    )
+
+
+def test_unpatching_svc_from_function(patch_svc_from_function):
+    from sklearnex import unpatch_sklearn
+
+    unpatch_sklearn(global_unpatch=True)
+    from sklearn.svm import SVC, SVR
+
+    assert not SVC.__module__.startswith("daal4py")
+    assert not SVC.__module__.startswith("sklearnex")
+    assert not SVR.__module__.startswith("daal4py")
+    assert not SVR.__module__.startswith("sklearnex")
+
+
+@pytest.fixture
+def patch_all_from_function(request):
+    from sklearnex import patch_sklearn, unpatch_sklearn
+
+    patch_sklearn(global_patch=True)
+
+    def finalizer():
+        unpatch_sklearn(global_unpatch=True)
+
+    request.addfinalizer(finalizer)
+    return
+
+
+def test_patching_svc_from_function(patch_all_from_function):
+    from sklearn.svm import SVC, SVR
+
+    assert SVC.__module__.startswith("daal4py") or SVC.__module__.startswith("sklearnex")
+    assert SVR.__module__.startswith("daal4py") or SVR.__module__.startswith("sklearnex")
+
+
+def test_unpatching_all_from_function(patch_all_from_function):
+    from sklearnex import unpatch_sklearn
+
+    unpatch_sklearn(global_unpatch=True)
+    from sklearn.svm import SVC, SVR
+
+    assert not SVC.__module__.startswith("daal4py")
+    assert not SVC.__module__.startswith("sklearnex")
+    assert not SVR.__module__.startswith("daal4py")
+    assert not SVR.__module__.startswith("sklearnex")
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
@@ -76,9 +76,9 @@ test:
         - daal4py/sklearn
     commands:
         - python -c "import daal4py"
-        - mpirun -n 4 python -m unittest discover -v -s tests -p test*spmd*.py # [not win]
-        - mpiexec -localonly -n 4 python -m unittest discover -v -s tests -p test*spmd*.py # [win]
-        - python -m unittest discover -v -s tests -p test*.py
+        - mpirun -n 4 pytest --verbose -s tests/test*spmd*.py # [not win]
+        - mpiexec -localonly -n 4 pytest --verbose -s tests/test*spmd*.py # [win]
+        - pytest --pyargs --verbose -s tests
         - pytest --pyargs daal4py/sklearn/
         - python tests/run_examples.py
 

diff --git a/conda-recipe/run_test.bat b/conda-recipe/run_test.bat
@@ -39,10 +39,10 @@ IF DEFINED TBBROOT (
     call "%TBBROOT%\env\vars.bat" || set exitcode=1
 )
 
-%PYTHON% -m unittest discover -v -s %1\tests -p test*.py || set exitcode=1
+%PYTHON% -m pytest --verbose -s %1\tests || set exitcode=1
 
 pytest --verbose --pyargs %1\daal4py\sklearn || set exitcode=1
 pytest --verbose --pyargs sklearnex || set exitcode=1
 pytest --verbose --pyargs %1\onedal --deselect="onedal/common/tests/test_policy.py" || set exitcode=1
-python %1\.ci\scripts\test_global_patch.py || set exitcode=1
+pytest --verbose %1\.ci\scripts\test_global_patch.py || set exitcode=1
 EXIT /B %exitcode%
diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh
@@ -37,18 +37,19 @@ return_code=0
 python -c "import daal4py"
 return_code=$(($return_code + $?))
 
+echo "Pytest run of legacy unittest ..."
+echo ${daal4py_dir}
+pytest --verbose -s ${daal4py_dir}/tests
+return_code=$(($return_code + $?))
+
 echo "NO_DIST=$NO_DIST"
 if [[ ! $NO_DIST ]]; then
-    echo "MPI unittest discover testing ..."
+    echo "MPI pytest run of legacy unittest ..."
     mpirun --version
-    mpirun -n 4 python -m unittest discover -v -s ${daal4py_dir}/tests -p test*spmd*.py
+    mpirun -n 4 pytest --verbose -s ${daal4py_dir}/tests/test*spmd*.py
     return_code=$(($return_code + $?))
 fi
 
-echo "Unittest discover testing ..."
-python -m unittest discover -v -s ${daal4py_dir}/tests -p test*.py
-return_code=$(($return_code + $?))
-
 echo "Pytest of daal4py running ..."
 pytest --verbose --pyargs ${daal4py_dir}/daal4py/sklearn
 return_code=$(($return_code + $?))
@@ -62,7 +63,7 @@ pytest --verbose --pyargs ${daal4py_dir}/onedal
 return_code=$(($return_code + $?))
 
 echo "Global patching test running ..."
-python ${daal4py_dir}/.ci/scripts/test_global_patch.py
+pytest --verbose -s ${daal4py_dir}/.ci/scripts/test_global_patch.py
 return_code=$(($return_code + $?))
 
 exit $return_code
diff --git a/tests/__init__.py b/tests/__init__.py
diff --git a/tests/test_examples_sklearnex.py b/tests/test_examples_sklearnex.py
@@ -17,59 +17,38 @@
 import os
 import subprocess
 import sys
-import unittest
 
-from daal4py.sklearn._utils import get_daal_version
+import pytest
 
 test_path = os.path.abspath(os.path.dirname(__file__))
 unittest_data_path = os.path.join(test_path, "unittest_data")
 examples_path = os.path.join(os.path.dirname(test_path), "examples", "sklearnex")
 
-print("Testing examples_sklearnex")
-# First item is major version - 2021,
-# second is minor+patch - 0110,
-# third item is status - B
-sklearnex_version = get_daal_version()
-print("oneDAL version:", sklearnex_version)
-
-
-class TestsklearnexExamples(unittest.TestCase):
-    """Class for testing sklernex examples"""
-
-    # Get a list of all Python files in the examples directory
-    pass
-
-
-def test_generator(file):
-    def testit(self):
-        # Run the script and capture its exit code
-        process = subprocess.run(
-            [sys.executable, os.path.join(examples_path, file)],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            check=False,
-        )  # nosec
-        exit_code = process.returncode
-
-        # Assert that the exit code is 0
-        self.assertEqual(
-            exit_code,
-            0,
-            msg=f"Example has failed, the example's output:\n{process.stdout.decode()}\n{process.stderr.decode()}",
+# This is a workaround for older versions of Python on Windows
+# which didn't have it as part of the built-in 'os' module.
+EX_OK = os.EX_OK if hasattr(os, "EX_OK") else 0
+
+
+@pytest.mark.parametrize(
+    "file",
+    [
+        f
+        for f in os.listdir(examples_path)
+        if f.endswith(".py") and "spmd" not in f and "dpnp" not in f and "dpctl" not in f
+    ],
+)
+def test_sklearn_example(file):
+    # Run the script and capture its exit code
+    process = subprocess.run(
+        [sys.executable, os.path.join(examples_path, file)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=False,
+    )  # nosec
+    exit_code = process.returncode
+
+    if exit_code != EX_OK:
+        pytest.fail(
+            pytrace=False,
+            reason=f"Example has failed, the example's output:\n{process.stdout.decode()}\n{process.stderr.decode()}",
         )
-
-    setattr(TestsklearnexExamples, "test_" + os.path.splitext(file)[0], testit)
-    print("Generating tests for " + os.path.splitext(file)[0])
-
-
-files = [
-    f
-    for f in os.listdir(examples_path)
-    if f.endswith(".py") and "spmd" not in f and "dpnp" not in f and "dpctl" not in f
-]
-
-for file in files:
-    test_generator(file)
-
-if __name__ == "__main__":
-    unittest.main()