gh-91321: Fix compatibility with C++ older than C++11 (#93784) (#93802)

* Fix the compatibility of the Python C API with C++ older than C++11.
* _Py_NULL is only defined as nullptr on C++11 and newer.

(cherry picked from commit 4caf5c2)

* test_cppext now builds the C++ extension with setuptools.
* Add @test.support.requires_venv_with_pip.

(cherry picked from commit ca0cc9c)

Author: vstinner

Diff for: Include/pyport.h

@@ -36,10 +36,12 @@ extern "C++" {
         inline type _Py_CAST_impl(int ptr) {
             return reinterpret_cast<type>(ptr);
         }
+#if __cplusplus >= 201103
         template <typename type>
         inline type _Py_CAST_impl(std::nullptr_t) {
             return static_cast<type>(nullptr);
         }
+#endif
 
         template <typename type, typename expr_type>
             inline type _Py_CAST_impl(expr_type *expr) {
@@ -70,8 +72,9 @@ extern "C++" {
 #endif
 
 // Static inline functions should use _Py_NULL rather than using directly NULL
-// to prevent C++ compiler warnings. In C++, _Py_NULL uses nullptr.
-#ifdef __cplusplus
+// to prevent C++ compiler warnings. On C++11 and newer, _Py_NULL is defined as
+// nullptr.
+#if defined(__cplusplus) && __cplusplus >= 201103
 #  define _Py_NULL nullptr
 #else
 #  define _Py_NULL NULL

Diff for: Lib/test/_testcppext.cpp

@@ -6,6 +6,12 @@
 
 #include "Python.h"
 
+#if __cplusplus >= 201103
+#  define NAME _testcpp11ext
+#else
+#  define NAME _testcpp03ext
+#endif
+
 PyDoc_STRVAR(_testcppext_add_doc,
 "add(x, y)\n"
 "\n"
@@ -16,7 +22,7 @@ _testcppext_add(PyObject *Py_UNUSED(module), PyObject *args)
 {
     long i, j;
     if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) {
-        return nullptr;
+        return _Py_NULL;
     }
     long res = i + j;
     return PyLong_FromLong(res);
@@ -47,8 +53,8 @@ static PyObject *
 test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
 {
     PyObject *obj = Py_BuildValue("(ii)", 1, 2);
-    if (obj == nullptr) {
-        return nullptr;
+    if (obj == _Py_NULL) {
+        return _Py_NULL;
     }
     Py_ssize_t refcnt = Py_REFCNT(obj);
     assert(refcnt >= 1);
@@ -77,9 +83,11 @@ test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
     // gh-93442: Pass 0 as NULL for PyObject*
     Py_XINCREF(0);
     Py_XDECREF(0);
-    // ensure that nullptr works too
+#if _cplusplus >= 201103
+    // Test nullptr passed as PyObject*
     Py_XINCREF(nullptr);
     Py_XDECREF(nullptr);
+#endif
 
     Py_DECREF(obj);
     Py_RETURN_NONE;
@@ -90,16 +98,16 @@ static PyObject *
 test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
 {
     PyObject *str = PyUnicode_FromString("abc");
-    if (str == nullptr) {
-        return nullptr;
+    if (str == _Py_NULL) {
+        return _Py_NULL;
     }
 
     assert(PyUnicode_Check(str));
     assert(PyUnicode_GET_LENGTH(str) == 3);
 
     // gh-92800: test PyUnicode_READ()
     const void* data = PyUnicode_DATA(str);
-    assert(data != nullptr);
+    assert(data != _Py_NULL);
     int kind = PyUnicode_KIND(str);
     assert(kind == PyUnicode_1BYTE_KIND);
     assert(PyUnicode_READ(kind, data, 0) == 'a');
@@ -118,9 +126,9 @@ test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
 
 static PyMethodDef _testcppext_methods[] = {
     {"add", _testcppext_add, METH_VARARGS, _testcppext_add_doc},
-    {"test_api_casts", test_api_casts, METH_NOARGS, nullptr},
-    {"test_unicode", test_unicode, METH_NOARGS, nullptr},
-    {nullptr, nullptr, 0, nullptr}  /* sentinel */
+    {"test_api_casts", test_api_casts, METH_NOARGS, _Py_NULL},
+    {"test_unicode", test_unicode, METH_NOARGS, _Py_NULL},
+    {_Py_NULL, _Py_NULL, 0, _Py_NULL}  /* sentinel */
 };
 
 
@@ -135,26 +143,32 @@ _testcppext_exec(PyObject *module)
 
 static PyModuleDef_Slot _testcppext_slots[] = {
     {Py_mod_exec, reinterpret_cast<void*>(_testcppext_exec)},
-    {0, nullptr}
+    {0, _Py_NULL}
 };
 
 
 PyDoc_STRVAR(_testcppext_doc, "C++ test extension.");
 
+#define _STR(NAME) #NAME
+#define STR(NAME) _STR(NAME)
+
 static struct PyModuleDef _testcppext_module = {
     PyModuleDef_HEAD_INIT,  // m_base
-    "_testcppext",  // m_name
+    STR(NAME),  // m_name
     _testcppext_doc,  // m_doc
     0,  // m_size
     _testcppext_methods,  // m_methods
     _testcppext_slots,  // m_slots
-    nullptr,  // m_traverse
-    nullptr,  // m_clear
-    nullptr,  // m_free
+    _Py_NULL,  // m_traverse
+    _Py_NULL,  // m_clear
+    _Py_NULL,  // m_free
 };
 
+#define _FUNC_NAME(NAME) PyInit_ ## NAME
+#define FUNC_NAME(NAME) _FUNC_NAME(NAME)
+
 PyMODINIT_FUNC
-PyInit__testcppext(void)
+FUNC_NAME(NAME)(void)
 {
     return PyModuleDef_Init(&_testcppext_module);
 }

Diff for: Lib/test/setup_testcppext.py

@@ -0,0 +1,51 @@
+# gh-91321: Build a basic C++ test extension to check that the Python C API is
+# compatible with C++ and does not emit C++ compiler warnings.
+import sys
+from test import support
+
+from setuptools import setup, Extension
+
+
+MS_WINDOWS = (sys.platform == 'win32')
+
+
+SOURCE = support.findfile('_testcppext.cpp')
+if not MS_WINDOWS:
+    # C++ compiler flags for GCC and clang
+    CPPFLAGS = [
+        # gh-91321: The purpose of _testcppext extension is to check that building
+        # a C++ extension using the Python C API does not emit C++ compiler
+        # warnings
+        '-Werror',
+        # Warn on old-style cast (C cast) like: (PyObject*)op
+        '-Wold-style-cast',
+        # Warn when using NULL rather than _Py_NULL in static inline functions
+        '-Wzero-as-null-pointer-constant',
+    ]
+else:
+    # Don't pass any compiler flag to MSVC
+    CPPFLAGS = []
+
+
+def main():
+    cppflags = list(CPPFLAGS)
+    if '-std=c++03' in sys.argv:
+        sys.argv.remove('-std=c++03')
+        std = 'c++03'
+        name = '_testcpp03ext'
+    else:
+        # Python currently targets C++11
+        std = 'c++11'
+        name = '_testcpp11ext'
+
+    cppflags = [*CPPFLAGS, f'-std={std}']
+    cpp_ext = Extension(
+        name,
+        sources=[SOURCE],
+        language='c++',
+        extra_compile_args=cppflags)
+    setup(name=name, ext_modules=[cpp_ext])
+
+
+if __name__ == "__main__":
+    main()

Diff for: Lib/test/support/__init__.py

@@ -2196,3 +2196,20 @@ def clear_ignored_deprecations(*tokens: object) -> None:
     if warnings.filters != new_filters:
         warnings.filters[:] = new_filters
         warnings._filters_mutated()
+
+
+# Skip a test if venv with pip is known to not work.
+def requires_venv_with_pip():
+    # ensurepip requires zlib to open ZIP archives (.whl binary wheel packages)
+    try:
+        import zlib
+    except ImportError:
+        return unittest.skipIf(True, "venv: ensurepip requires zlib")
+
+    # bpo-26610: pip/pep425tags.py requires ctypes.
+    # gh-92820: setuptools/windows_support.py uses ctypes (setuptools 58.1).
+    try:
+        import ctypes
+    except ImportError:
+        ctypes = None
+    return unittest.skipUnless(ctypes, 'venv: pip requires ctypes')

Diff for: Lib/test/test_cppext.py

@@ -1,91 +1,73 @@
 # gh-91321: Build a basic C++ test extension to check that the Python C API is
 # compatible with C++ and does not emit C++ compiler warnings.
-import contextlib
-import os
+import os.path
 import sys
 import unittest
-import warnings
+import subprocess
 from test import support
 from test.support import os_helper
 
-with warnings.catch_warnings():
-    warnings.simplefilter('ignore', DeprecationWarning)
-    from distutils.core import setup, Extension
-    import distutils.sysconfig
-
 
 MS_WINDOWS = (sys.platform == 'win32')
 
 
-SOURCE = support.findfile('_testcppext.cpp')
-if not MS_WINDOWS:
-    # C++ compiler flags for GCC and clang
-    CPPFLAGS = [
-        # Python currently targets C++11
-        '-std=c++11',
-        # gh-91321: The purpose of _testcppext extension is to check that building
-        # a C++ extension using the Python C API does not emit C++ compiler
-        # warnings
-        '-Werror',
-        # Warn on old-style cast (C cast) like: (PyObject*)op
-        '-Wold-style-cast',
-        # Warn when using NULL rather than _Py_NULL in static inline functions
-        '-Wzero-as-null-pointer-constant',
-    ]
-else:
-    # Don't pass any compiler flag to MSVC
-    CPPFLAGS = []
+SETUP_TESTCPPEXT = support.findfile('setup_testcppext.py')
 
 
 @support.requires_subprocess()
 class TestCPPExt(unittest.TestCase):
-    def build(self):
-        cpp_ext = Extension(
-            '_testcppext',
-            sources=[SOURCE],
-            language='c++',
-            extra_compile_args=CPPFLAGS)
-        capture_stdout = (not support.verbose)
+    def test_build_cpp11(self):
+        self.check_build(False)
 
-        try:
-            try:
-                if capture_stdout:
-                    stdout = support.captured_stdout()
-                else:
-                    print()
-                    stdout = contextlib.nullcontext()
-                with (stdout,
-                      support.swap_attr(sys, 'argv', ['setup.py', 'build_ext', '--verbose'])):
-                    setup(name="_testcppext", ext_modules=[cpp_ext])
-                    return
-            except:
-                if capture_stdout:
-                    # Show output on error
-                    print()
-                    print(stdout.getvalue())
-                raise
-        except SystemExit:
-            self.fail("Build failed")
+    def test_build_cpp03(self):
+        self.check_build(True)
 
     # With MSVC, the linker fails with: cannot open file 'python311.lib'
     # https://github.com/python/cpython/pull/32175#issuecomment-1111175897
     @unittest.skipIf(MS_WINDOWS, 'test fails on Windows')
-    def test_build(self):
-        # save/restore os.environ
-        def restore_env(old_env):
-            os.environ.clear()
-            os.environ.update(old_env)
-        self.addCleanup(restore_env, dict(os.environ))
-
-        def restore_sysconfig_vars(old_config_vars):
-            distutils.sysconfig._config_vars.clear()
-            distutils.sysconfig._config_vars.update(old_config_vars)
-        self.addCleanup(restore_sysconfig_vars,
-                        dict(distutils.sysconfig._config_vars))
-
+    # the test uses venv+pip: skip if it's not available
+    @support.requires_venv_with_pip()
+    def check_build(self, std_cpp03):
         # Build in a temporary directory
         with os_helper.temp_cwd():
-            self.build()
+            self._check_build(std_cpp03)
+
+    def _check_build(self, std_cpp03):
+        venv_dir = 'env'
+        verbose = support.verbose
+
+        # Create virtual environment to get setuptools
+        cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir]
+        if verbose:
+            print()
+            print('Run:', ' '.join(cmd))
+        subprocess.run(cmd, check=True)
+
+        # Get the Python executable of the venv
+        python_exe = 'python'
+        if sys.executable.endswith('.exe'):
+            python_exe += '.exe'
+        if MS_WINDOWS:
+            python = os.path.join(venv_dir, 'Scripts', python_exe)
+        else:
+            python = os.path.join(venv_dir, 'bin', python_exe)
+
+        # Build the C++ extension
+        cmd = [python, '-X', 'dev',
+               SETUP_TESTCPPEXT, 'build_ext', '--verbose']
+        if std_cpp03:
+            cmd.append('-std=c++03')
+        if verbose:
+            print('Run:', ' '.join(cmd))
+            subprocess.run(cmd, check=True)
+        else:
+            proc = subprocess.run(cmd,
+                                  stdout=subprocess.PIPE,
+                                  stderr=subprocess.STDOUT,
+                                  text=True)
+            if proc.returncode:
+                print(proc.stdout, end='')
+                self.fail(f"Build failed with exit code {proc.returncode}")
 
 
 if __name__ == "__main__":

Diff for: Lib/test/test_venv.py

@@ -17,7 +17,8 @@
 import tempfile
 from test.support import (captured_stdout, captured_stderr, requires_zlib,
                           skip_if_broken_multiprocessing_synchronize, verbose,
-                          requires_subprocess, is_emscripten, is_wasi)
+                          requires_subprocess, is_emscripten, is_wasi,
+                          requires_venv_with_pip)
 from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree)
 import unittest
 import venv
@@ -619,9 +620,7 @@ def do_test_with_pip(self, system_site_packages):
         if not system_site_packages:
             self.assert_pip_not_installed()
 
-    # Issue #26610: pip/pep425tags.py requires ctypes
-    @unittest.skipUnless(ctypes, 'pip requires ctypes')
-    @requires_zlib()
+    @requires_venv_with_pip()
     def test_with_pip(self):
         self.do_test_with_pip(False)
         self.do_test_with_pip(True)

Diff for: Misc/NEWS.d/next/C API/2022-06-13-21-37-31.gh-issue-91321.DgJFvS.rst

@@ -0,0 +1,2 @@
+Fix the compatibility of the Python C API with C++ older than C++11. Patch by
+Victor Stinner.