tests: cleanup and ci hardening (#2397)

* tests: refactor and cleanup

* refactor: more consistent

* tests: vendor six

* tests: more xfails, nicer system

* tests: simplify to info

* tests: suggestions from @YannickJadoul and @bstaletic

* tests: restore some pypy tests that now pass

* tests: rename info to env

* tests: strict False/True

* tests: drop explicit strict=True again

* tests: reduce minimum PyTest to 3.1

.github/workflows/ci.yml

@@ -17,11 +17,11 @@ jobs:
         runs-on: [ubuntu-latest, windows-latest, macos-latest]
         arch: [x64]
         max-cxx-std: [17]
+        dev: [false]
         python:
         - 2.7
         - 3.5
         - 3.8
-        - 3.9-dev
         - pypy2
         - pypy3
 
@@ -30,46 +30,68 @@ jobs:
             python: 3.6
             arch: x64
             max-cxx-std: 17
+            dev: false
           - runs-on: macos-latest
             python: 3.7
             arch: x64
             max-cxx-std: 17
+            dev: false
           - runs-on: windows-2016
             python: 3.7
             arch: x86
             max-cxx-std: 14
+            dev: false
           - runs-on: windows-latest
             python: 3.6
             arch: x64
             max-cxx-std: 17
+            dev: false
           - runs-on: windows-latest
             python: 3.7
             arch: x64
             max-cxx-std: 17
+            dev: false
+
+          - runs-on: ubuntu-latest
+            python: 3.9-dev
+            arch: x64
+            max-cxx-std: 17
+            dev: true
+          - runs-on: macos-latest
+            python: 3.9-dev
+            arch: x64
+            max-cxx-std: 17
+            dev: true
 
         exclude:
             # Currently 32bit only, and we build 64bit
           - runs-on: windows-latest
             python: pypy2
             arch: x64
             max-cxx-std: 17
+            dev: false
           - runs-on: windows-latest
             python: pypy3
             arch: x64
             max-cxx-std: 17
+            dev: false
 
             # Currently broken on embed_test
           - runs-on: windows-latest
             python: 3.8
             arch: x64
             max-cxx-std: 17
+            dev: false
           - runs-on: windows-latest
             python: 3.9-dev
             arch: x64
             max-cxx-std: 17
+            dev: false
+
 
     name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • ${{ matrix.arch }}"
     runs-on: ${{ matrix.runs-on }}
+    continue-on-error: ${{ matrix.dev }}
 
     steps:
     - uses: actions/checkout@v2
@@ -289,7 +311,8 @@ jobs:
     - name: Install requirements
       run: |
         apt-get update
-        apt-get install -y git make cmake g++ libeigen3-dev python3-dev python3-pip python3-pytest
+        apt-get install -y git make cmake g++ libeigen3-dev python3-dev python3-pip
+        pip3 install "pytest==3.1.*"
 
     - name: Configure for install
       run: >

tests/CMakeLists.txt

@@ -266,8 +266,8 @@ if(NOT PYBIND11_PYTEST_FOUND)
   if(pytest_not_found)
     message(FATAL_ERROR "Running the tests requires pytest. Please install it manually"
                         " (try: ${PYTHON_EXECUTABLE} -m pip install pytest)")
-  elseif(pytest_version VERSION_LESS 3.0)
-    message(FATAL_ERROR "Running the tests requires pytest >= 3.0. Found: ${pytest_version}"
+  elseif(pytest_version VERSION_LESS 3.1)
+    message(FATAL_ERROR "Running the tests requires pytest >= 3.1. Found: ${pytest_version}"
                         "Please update it (try: ${PYTHON_EXECUTABLE} -m pip install -U pytest)")
   endif()
   set(PYBIND11_PYTEST_FOUND

tests/conftest.py

@@ -5,24 +5,21 @@
 Adds docstring and exceptions message sanitizers: ignore Python 2 vs 3 differences.
 """
 
-import pytest
-import textwrap
-import difflib
-import re
-import sys
 import contextlib
-import platform
+import difflib
 import gc
+import re
+import textwrap
+
+import pytest
+
+# Early diagnostic for failed imports
+import pybind11_tests  # noqa: F401
 
 _unicode_marker = re.compile(r'u(\'[^\']*\')')
 _long_marker = re.compile(r'([0-9])L')
 _hexadecimal = re.compile(r'0x[0-9a-fA-F]+')
 
-# test_async.py requires support for async and await
-collect_ignore = []
-if sys.version_info[:2] < (3, 5):
-    collect_ignore.append("test_async.py")
-
 
 def _strip_and_dedent(s):
     """For triple-quote strings"""
@@ -192,63 +189,5 @@ def gc_collect():
 
 
 def pytest_configure():
-    """Add import suppression and test requirements to `pytest` namespace"""
-    try:
-        import numpy as np
-    except ImportError:
-        np = None
-    try:
-        import scipy
-    except ImportError:
-        scipy = None
-    try:
-        from pybind11_tests.eigen import have_eigen
-    except ImportError:
-        have_eigen = False
-
-    # Provide simple `six`-like aliases.
-    pytest.PY2 = (sys.version_info.major == 2)
-    pytest.CPYTHON = (platform.python_implementation() == "CPython")
-    pytest.PYPY = (platform.python_implementation() == "PyPy")
-
-    skipif = pytest.mark.skipif
     pytest.suppress = suppress
-    pytest.requires_numpy = skipif(not np, reason="numpy is not installed")
-    pytest.requires_scipy = skipif(not np, reason="scipy is not installed")
-    pytest.requires_eigen_and_numpy = skipif(not have_eigen or not np,
-                                             reason="eigen and/or numpy are not installed")
-    pytest.requires_eigen_and_scipy = skipif(
-        not have_eigen or not scipy, reason="eigen and/or scipy are not installed")
-    pytest.unsupported_on_pypy = skipif(pytest.PYPY, reason="unsupported on PyPy")
-    pytest.bug_in_pypy = pytest.mark.xfail(pytest.PYPY, reason="bug in PyPy")
-    pytest.unsupported_on_pypy3 = skipif(pytest.PYPY and not pytest.PY2,
-                                         reason="unsupported on PyPy3")
-    pytest.unsupported_on_pypy_lt_6 = skipif(pytest.PYPY and sys.pypy_version_info[0] < 6,
-                                             reason="unsupported on PyPy<6")
-    pytest.unsupported_on_py2 = skipif(pytest.PY2,
-                                       reason="unsupported on Python 2.x")
     pytest.gc_collect = gc_collect
-
-
-def _test_import_pybind11():
-    """Early diagnostic for test module initialization errors
-
-    When there is an error during initialization, the first import will report the
-    real error while all subsequent imports will report nonsense. This import test
-    is done early (in the pytest configuration file, before any tests) in order to
-    avoid the noise of having all tests fail with identical error messages.
-
-    Any possible exception is caught here and reported manually *without* the stack
-    trace. This further reduces noise since the trace would only show pytest internals
-    which are not useful for debugging pybind11 module issues.
-    """
-    # noinspection PyBroadException
-    try:
-        import pybind11_tests  # noqa: F401 imported but unused
-    except Exception as e:
-        print("Failed to import pybind11_tests from pytest:")
-        print("  {}: {}".format(type(e).__name__, e))
-        sys.exit(1)
-
-
-_test_import_pybind11()

tests/env.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+import platform
+import sys
+
+LINUX = sys.platform.startswith("linux")
+MACOS = sys.platform.startswith("darwin")
+WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin")
+
+CPYTHON = platform.python_implementation() == "CPython"
+PYPY = platform.python_implementation() == "PyPy"
+
+PY2 = sys.version_info.major == 2

tests/pybind11_tests.cpp

@@ -88,6 +88,4 @@ PYBIND11_MODULE(pybind11_tests, m) {
 
     for (const auto &initializer : initializers())
         initializer(m);
-
-    if (!py::hasattr(m, "have_eigen")) m.attr("have_eigen") = false;
 }

tests/pytest.ini

@@ -1,11 +1,14 @@
 [pytest]
-minversion = 3.0
+minversion = 3.1
 norecursedirs = test_cmake_build test_embed
+xfail_strict = True
 addopts =
     # show summary of skipped tests
     -rs
     # capture only Python print and C++ py::print, but not C output (low-level Python errors)
     --capture=sys
+    # enable all warnings
+    -Wa
 filterwarnings =
     # make warnings into errors but ignore certain third-party extension issues
     error

tests/test_async.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
-import asyncio
 import pytest
-from pybind11_tests import async_module as m
+
+asyncio = pytest.importorskip("asyncio")
+m = pytest.importorskip("pybind11_tests.async_module")
 
 
 @pytest.fixture

tests/test_buffers.py

@@ -4,13 +4,12 @@
 
 import pytest
 
+import env  # noqa: F401
+
 from pybind11_tests import buffers as m
 from pybind11_tests import ConstructorStats
 
-pytestmark = pytest.requires_numpy
-
-with pytest.suppress(ImportError):
-    import numpy as np
+np = pytest.importorskip("numpy")
 
 
 def test_from_python():
@@ -36,9 +35,7 @@ def test_from_python():
     assert cstats.move_assignments == 0
 
 
-# PyPy: Memory leak in the "np.array(m, copy=False)" call
-# https://bitbucket.org/pypy/pypy/issues/2444
-@pytest.unsupported_on_pypy
+# https://foss.heptapod.net/pypy/pypy/-/issues/2444
 def test_to_python():
     mat = m.Matrix(5, 4)
     assert memoryview(mat).shape == (5, 4)
@@ -73,7 +70,6 @@ def test_to_python():
     assert cstats.move_assignments == 0
 
 
-@pytest.unsupported_on_pypy
 def test_inherited_protocol():
     """SquareMatrix is derived from Matrix and inherits the buffer protocol"""
 
@@ -82,7 +78,6 @@ def test_inherited_protocol():
     assert np.asarray(matrix).shape == (5, 5)
 
 
-@pytest.unsupported_on_pypy
 def test_pointer_to_member_fn():
     for cls in [m.Buffer, m.ConstBuffer, m.DerivedBuffer]:
         buf = cls()
@@ -91,26 +86,24 @@ def test_pointer_to_member_fn():
         assert value == 0x12345678
 
 
-@pytest.unsupported_on_pypy
 def test_readonly_buffer():
     buf = m.BufferReadOnly(0x64)
     view = memoryview(buf)
-    assert view[0] == b'd' if pytest.PY2 else 0x64
+    assert view[0] == b'd' if env.PY2 else 0x64
     assert view.readonly
 
 
-@pytest.unsupported_on_pypy
 def test_selective_readonly_buffer():
     buf = m.BufferReadOnlySelect()
 
-    memoryview(buf)[0] = b'd' if pytest.PY2 else 0x64
+    memoryview(buf)[0] = b'd' if env.PY2 else 0x64
     assert buf.value == 0x64
 
     io.BytesIO(b'A').readinto(buf)
     assert buf.value == ord(b'A')
 
     buf.readonly = True
     with pytest.raises(TypeError):
-        memoryview(buf)[0] = b'\0' if pytest.PY2 else 0
+        memoryview(buf)[0] = b'\0' if env.PY2 else 0
     with pytest.raises(TypeError):
         io.BytesIO(b'1').readinto(buf)

tests/test_builtin_casters.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 import pytest
 
+import env  # noqa: F401
+
 from pybind11_tests import builtin_casters as m
 from pybind11_tests import UserType, IncType
 
@@ -117,10 +119,7 @@ def test_bytes_to_string():
     # Issue #816
 
     def to_bytes(s):
-        if pytest.PY2:
-            b = s
-        else:
-            b = s.encode("utf8")
+        b = s if env.PY2 else s.encode("utf8")
         assert isinstance(b, bytes)
         return b
 
@@ -197,7 +196,7 @@ def test_integer_casting():
     assert m.i64_str(-1) == "-1"
     assert m.i32_str(2000000000) == "2000000000"
     assert m.u32_str(2000000000) == "2000000000"
-    if pytest.PY2:
+    if env.PY2:
         assert m.i32_str(long(-1)) == "-1"  # noqa: F821 undefined name 'long'
         assert m.i64_str(long(-1)) == "-1"  # noqa: F821 undefined name 'long'
         assert m.i64_str(long(-999999999999)) == "-999999999999"  # noqa: F821 undefined name
@@ -219,7 +218,7 @@ def test_integer_casting():
         m.i32_str(3000000000)
     assert "incompatible function arguments" in str(excinfo.value)
 
-    if pytest.PY2:
+    if env.PY2:
         with pytest.raises(TypeError) as excinfo:
             m.u32_str(long(-1))  # noqa: F821 undefined name 'long'
         assert "incompatible function arguments" in str(excinfo.value)
@@ -360,9 +359,9 @@ class B(object):
     assert convert(A(False)) is False
 
 
-@pytest.requires_numpy
 def test_numpy_bool():
-    import numpy as np
+    np = pytest.importorskip("numpy")
+
     convert, noconvert = m.bool_passthrough, m.bool_passthrough_noconvert
 
     def cant_convert(v):

tests/test_call_policies.py

@@ -1,9 +1,13 @@
 # -*- coding: utf-8 -*-
 import pytest
+
+import env  # noqa: F401
+
 from pybind11_tests import call_policies as m
 from pybind11_tests import ConstructorStats
 
 
+@pytest.mark.xfail("env.PYPY", reason="sometimes comes out 1 off on PyPy", strict=False)
 def test_keep_alive_argument(capture):
     n_inst = ConstructorStats.detail_reg_inst()
     with capture:
@@ -70,8 +74,8 @@ def test_keep_alive_return_value(capture):
     """
 
 
-# https://bitbucket.org/pypy/pypy/issues/2447
-@pytest.unsupported_on_pypy
+# https://foss.heptapod.net/pypy/pypy/-/issues/2447
+@pytest.mark.xfail("env.PYPY", reason="_PyObject_GetDictPtr is unimplemented")
 def test_alive_gc(capture):
     n_inst = ConstructorStats.detail_reg_inst()
     p = m.ParentGC()