Snapshot of pybind/pybind11#4601 (squashed).

include/pybind11/cast.h

@@ -1207,7 +1207,11 @@ make_caster<T> load_type(const handle &handle) {
 PYBIND11_NAMESPACE_END(detail)
 
 // pytype -> C++ type
-template <typename T, detail::enable_if_t<!detail::is_pyobject<T>::value, int> = 0>
+template <typename T,
+          detail::enable_if_t<!detail::is_pyobject<T>::value
+                                  && !detail::is_same_ignoring_cvref<T, PyObject *>::value,
+                              int>
+          = 0>
 T cast(const handle &handle) {
     using namespace detail;
     constexpr bool is_enum_cast = type_uses_type_caster_enum_type<intrinsic_t<T>>::value;
@@ -1231,6 +1235,12 @@ T cast(const handle &handle) {
     return T(reinterpret_borrow<object>(handle));
 }
 
+template <typename T,
+          detail::enable_if_t<detail::is_same_ignoring_cvref<T, PyObject *>::value, int> = 0>
+T cast(const handle &handle) {
+    return handle.ptr();
+}
+
 // C++ type -> py::object
 template <typename T, detail::enable_if_t<!detail::is_pyobject<T>::value, int> = 0>
 object cast(T &&value,

include/pybind11/detail/common.h

@@ -751,6 +751,10 @@ template <class T>
 using remove_cvref_t = typename remove_cvref<T>::type;
 #endif
 
+/// Example usage: is_same_ignoring_cvref<T, PyObject *>
+template <typename T, typename U>
+using is_same_ignoring_cvref = std::is_same<detail::remove_cvref_t<T>, U>;
+
 /// Index sequences
 #if defined(PYBIND11_CPP14)
 using std::index_sequence;

include/pybind11/type_caster_pyobject_ptr.h

@@ -0,0 +1,57 @@
+// Copyright (c) 2023 The pybind Community.
+
+#pragma once
+
+#include "detail/common.h"
+#include "detail/descr.h"
+#include "cast.h"
+#include "pytypes.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+PYBIND11_NAMESPACE_BEGIN(detail)
+
+template <>
+class type_caster<PyObject> {
+public:
+    static constexpr auto name = const_name("PyObject *");
+
+    // This overload is purely to guard against accidents.
+    template <typename T,
+              detail::enable_if_t<!is_same_ignoring_cvref<T, PyObject *>::value, int> = 0>
+    static handle cast(T &&, return_value_policy, handle /*parent*/) {
+        static_assert(is_same_ignoring_cvref<T, PyObject *>::value,
+                      "Invalid C++ type T for to-Python conversion (type_caster<PyObject>).");
+        return nullptr; // Unreachable.
+    }
+
+    static handle cast(PyObject *src, return_value_policy policy, handle /*parent*/) {
+        if (src == nullptr) {
+            throw error_already_set();
+        }
+        if (PyErr_Occurred()) {
+            raise_from(PyExc_SystemError, "src != nullptr but PyErr_Occurred()");
+            throw error_already_set();
+        }
+        if (policy == return_value_policy::take_ownership) {
+            return src;
+        }
+        Py_INCREF(src);
+        return src;
+    }
+
+    bool load(handle src, bool) {
+        value = reinterpret_borrow<object>(src);
+        return true;
+    }
+
+    template <typename T>
+    using cast_op_type = PyObject *;
+
+    explicit operator PyObject *() { return value.ptr(); }
+
+private:
+    object value;
+};
+
+PYBIND11_NAMESPACE_END(detail)
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

tests/CMakeLists.txt

@@ -180,6 +180,7 @@ set(PYBIND11_TEST_FILES
     test_thread
     test_type_caster_odr_guard_1
     test_type_caster_odr_guard_2
+    test_type_caster_pyobject_ptr
     test_union
     test_virtual_functions)
 

tests/test_type_caster_pyobject_ptr.cpp

@@ -0,0 +1,52 @@
+#include <pybind11/functional.h>
+#include <pybind11/type_caster_pyobject_ptr.h>
+
+#include "pybind11_tests.h"
+
+TEST_SUBMODULE(type_caster_pyobject_ptr, m) {
+    m.def("cast_from_pyobject_ptr", []() {
+        PyObject *ptr = PyLong_FromLongLong(6758L);
+        py::object retval = py::cast(ptr, py::return_value_policy::take_ownership);
+        return retval;
+    });
+    m.def("cast_to_pyobject_ptr", [](py::handle obj) {
+        auto *ptr = py::cast<PyObject *>(obj);
+        return bool(PyTuple_CheckExact(ptr));
+    });
+
+    m.def(
+        "return_pyobject_ptr",
+        []() { return PyLong_FromLongLong(2314L); },
+        py::return_value_policy::take_ownership);
+    m.def("pass_pyobject_ptr", [](PyObject *obj) { return bool(PyTuple_CheckExact(obj)); });
+
+    m.def("call_callback_with_object_return",
+          [](const std::function<py::object(int mode)> &cb, int mode) { return cb(mode); });
+    m.def("call_callback_with_handle_return",
+          [](const std::function<py::handle(int mode)> &cb, int mode) { return cb(mode); });
+    //
+    m.def("call_callback_with_pyobject_ptr_return",
+          [](const std::function<PyObject *(int mode)> &cb, int mode) { return cb(mode); });
+    m.def("call_callback_with_pyobject_ptr_arg",
+          [](const std::function<bool(PyObject *)> &cb, py::handle obj) { return cb(obj.ptr()); });
+
+    m.def("cast_to_pyobject_ptr_nullptr", [](bool set_error) {
+        if (set_error) {
+            PyErr_SetString(PyExc_RuntimeError, "Reflective of healthy error handling.");
+        }
+        PyObject *ptr = nullptr;
+        py::cast(ptr);
+    });
+
+    m.def("cast_to_pyobject_ptr_non_nullptr_with_error_set", []() {
+        PyErr_SetString(PyExc_RuntimeError, "Reflective of unhealthy error handling.");
+        py::cast(Py_None);
+    });
+
+#ifdef PYBIND11_NO_COMPILE_SECTION // Change to ifndef for manual testing.
+    {
+        PyObject *ptr = nullptr;
+        (void) py::cast(*ptr);
+    }
+#endif
+}

tests/test_type_caster_pyobject_ptr.py

@@ -0,0 +1,71 @@
+import pytest
+
+from pybind11_tests import type_caster_pyobject_ptr as m
+
+
+def test_cast_from_pyobject_ptr():
+    assert m.cast_from_pyobject_ptr() == 6758
+
+
+def test_cast_to_pyobject_ptr():
+    assert m.cast_to_pyobject_ptr(())
+    assert not m.cast_to_pyobject_ptr({})
+
+
+def test_return_pyobject_ptr():
+    assert m.return_pyobject_ptr() == 2314
+
+
+def test_pass_pyobject_ptr():
+    assert m.pass_pyobject_ptr(())
+    assert not m.pass_pyobject_ptr({})
+
+
+@pytest.mark.parametrize(
+    "call_callback",
+    [
+        m.call_callback_with_object_return,
+        m.call_callback_with_handle_return,
+        m.call_callback_with_pyobject_ptr_return,
+    ],
+)
+def test_call_callback_with_object_return(call_callback):
+    def cb(mode):
+        if mode == 0:
+            return 10
+        if mode == 1:
+            return "One"
+        raise NotImplementedError(f"Unknown mode: {mode}")
+
+    assert call_callback(cb, 0) == 10
+    assert call_callback(cb, 1) == "One"
+    with pytest.raises(NotImplementedError, match="Unknown mode: 2"):
+        call_callback(cb, 2)
+
+
+def test_call_callback_with_pyobject_ptr_arg():
+    def cb(obj):
+        return isinstance(obj, tuple)
+
+    assert m.call_callback_with_pyobject_ptr_arg(cb, ())
+    assert not m.call_callback_with_pyobject_ptr_arg(cb, {})
+
+
+@pytest.mark.parametrize("set_error", [True, False])
+def test_cast_to_python_nullptr(set_error):
+    expected = {
+        True: r"^Reflective of healthy error handling\.$",
+        False: (
+            r"^Internal error: pybind11::error_already_set called "
+            r"while Python error indicator not set\.$"
+        ),
+    }[set_error]
+    with pytest.raises(RuntimeError, match=expected):
+        m.cast_to_pyobject_ptr_nullptr(set_error)
+
+
+def test_cast_to_python_non_nullptr_with_error_set():
+    with pytest.raises(SystemError) as excinfo:
+        m.cast_to_pyobject_ptr_non_nullptr_with_error_set()
+    assert str(excinfo.value) == "src != nullptr but PyErr_Occurred()"
+    assert str(excinfo.value.__cause__) == "Reflective of unhealthy error handling."