diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index 1df4d3e14b..d1d852b8a3 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -1,93 +1,140 @@ Custom type casters =================== -In very rare cases, applications may require custom type casters that cannot be -expressed using the abstractions provided by pybind11, thus requiring raw -Python C API calls. This is fairly advanced usage and should only be pursued by -experts who are familiar with the intricacies of Python reference counting. +Some applications may prefer custom type casters that convert between existing +Python types and C++ types, similar to the ``list`` ↔ ``std::vector`` +and ``dict`` ↔ ``std::map`` conversions which are built into pybind11. +Implementing custom type casters is fairly advanced usage and requires +familiarity with the intricacies of the Python C API. The following snippets demonstrate how this works for a very simple ``inty`` -type that that should be convertible from Python types that provide a -``__int__(self)`` method. +type that we want to be convertible to C++ from any Python type that provides +an ``__int__`` method, and is converted to a Python ``int`` when returned from +C++ to Python. -.. code-block:: cpp - - struct inty { long long_value; }; +.. + PLEASE KEEP THE CODE BLOCKS IN SYNC WITH + tests/test_docs_advanced_cast_custom.cpp + Ideally, change the test, run pre-commit (incl. clang-format), + then copy the changed code back here. - void print(inty s) { - std::cout << s.long_value << std::endl; - } +.. code-block:: cpp -The following Python snippet demonstrates the intended usage from the Python side: + namespace user_space { -.. code-block:: python + struct inty { + long long_value; + }; - class A: - def __int__(self): - return 123 + std::string to_string(const inty &s) { return std::to_string(s.long_value); } + inty return_42() { return inty{42}; } - from example import print + } // namespace user_space - print(A()) +The necessary conversion routines are defined by a caster class, which +is then "plugged into" pybind11 using one of two alternative mechanisms. -To register the necessary conversion routines, it is necessary to add an -instantiation of the ``pybind11::detail::type_caster`` template. -Although this is an implementation detail, adding an instantiation of this -type is explicitly allowed. +Starting with the example caster class: .. code-block:: cpp - namespace pybind11 { namespace detail { - template <> struct type_caster { - public: - /** - * This macro establishes the name 'inty' in - * function signatures and declares a local variable - * 'value' of type inty - */ - PYBIND11_TYPE_CASTER(inty, const_name("inty")); - - /** - * Conversion part 1 (Python->C++): convert a PyObject into a inty - * instance or return false upon failure. The second argument - * indicates whether implicit conversions should be applied. - */ - bool load(handle src, bool) { - /* Extract PyObject from handle */ - PyObject *source = src.ptr(); - /* Try converting into a Python integer value */ - PyObject *tmp = PyNumber_Long(source); - if (!tmp) - return false; - /* Now try to convert into a C++ int */ - value.long_value = PyLong_AsLong(tmp); - Py_DECREF(tmp); - /* Ensure return code was OK (to avoid out-of-range errors etc) */ - return !(value.long_value == -1 && !PyErr_Occurred()); + namespace user_space { + + struct inty_type_caster { + public: + // This macro establishes the name 'inty' in function signatures and declares a local variable + // 'value' of type inty. + PYBIND11_TYPE_CASTER(inty, pybind11::detail::const_name("inty")); + + // Python -> C++: convert a PyObject into an inty instance or return false upon failure. The + // second argument indicates whether implicit conversions should be allowed. + bool load(pybind11::handle src, bool) { + // Extract PyObject from handle. + PyObject *source = src.ptr(); + // Try converting into a Python integer value. + PyObject *tmp = PyNumber_Long(source); + if (!tmp) { + return false; } - - /** - * Conversion part 2 (C++ -> Python): convert an inty instance into - * a Python object. The second and third arguments are used to - * indicate the return value policy and parent object (for - * ``return_value_policy::reference_internal``) and are generally - * ignored by implicit casters. - */ - static handle cast(inty src, return_value_policy /* policy */, handle /* parent */) { - return PyLong_FromLong(src.long_value); + // Now try to convert into a C++ int. + value.long_value = PyLong_AsLong(tmp); + Py_DECREF(tmp); + // Ensure PyLong_AsLong succeeded (to catch out-of-range errors etc). + if (PyErr_Occurred()) { + PyErr_Clear(); + return false; } - }; - }} // namespace pybind11::detail + return true; + } + + // C++ -> Python: convert an inty instance into a Python object. The second and third arguments + // are used to indicate the return value policy and parent object (for + // return_value_policy::reference_internal) and are often ignored by custom casters. + static pybind11::handle + cast(inty src, pybind11::return_value_policy /* policy */, pybind11::handle /* parent */) { + return PyLong_FromLong(src.long_value); + } + }; + + } // namespace user_space .. note:: - A ``type_caster`` defined with ``PYBIND11_TYPE_CASTER(T, ...)`` requires + A caster class using ``PYBIND11_TYPE_CASTER(T, ...)`` requires that ``T`` is default-constructible (``value`` is first default constructed - and then ``load()`` assigns to it). + and then ``load()`` assigns to it). It is possible but more involved to define + a caster class for types that are not default-constructible. -.. warning:: +The caster class defined above can be plugged into pybind11 in two ways: + +* Starting with pybind11 v2.10, a new — and the recommended — alternative is to *declare* a + function named ``pybind11_select_caster``: + + .. code-block:: cpp + + namespace user_space { + + inty_type_caster pybind11_select_caster(inty *); - When using custom type casters, it's important to declare them consistently - in every compilation unit of the Python extension module. Otherwise, - undefined behavior can ensue. + } // namespace user_space + + The argument is a *pointer* to the C++ type, the return type is the caster type. + This function has no implementation! Its only purpose is to associate the C++ type + with its caster class. pybind11 exploits C++ Argument Dependent Lookup + (`ADL `_) + to discover the association. + + Note that ``pybind11_select_caster`` can alternatively be declared as a ``friend`` + function of the C++ type, if that is practical and preferred: + + .. code-block:: cpp + + struct inty_type_caster; + + struct inty { + ... + friend inty_type_caster pybind11_select_caster(inty *); + }; + +* An older alternative is to specialize the ``pybind11::detail::type_caster`` template. + Although the ``detail`` namespace is involved, adding a ``type_caster`` specialization + is explicitly allowed: + + .. code-block:: cpp + + namespace pybind11 { + namespace detail { + template <> + struct type_caster : user_space::inty_type_caster {}; + } // namespace detail + } // namespace pybind11 + + .. note:: + ``type_caster` specializations may be full (as in this simple example) or partial. + +.. warning:: + With either alternative, for a given type ``T``, the ``pybind11_select_caster`` + declaration or ``type_caster`` specialization must be consistent across all compilation + units of a Python extension module, to satisfy the C++ One Definition Rule + (`ODR `_). diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 376a679544..af1a7fc34e 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -33,8 +33,35 @@ PYBIND11_NAMESPACE_BEGIN(detail) template class type_caster : public type_caster_base {}; + +template +type_caster pybind11_select_caster(IntrinsicType *); + +// MSVC 2015 generates an internal compiler error for the common code (in the #else branch below). +// MSVC 2017 in C++14 mode produces incorrect code, leading to a tests/test_stl runtime failure. +// Luckily, the workaround for MSVC 2015 also resolves the MSVC 2017 C++14 runtime failure. +#if defined(_MSC_VER) && (_MSC_VER < 1910 || (_MSC_VER < 1920 && !defined(PYBIND11_CPP17))) + +template +struct make_caster_impl; + +template +struct make_caster_impl::value>> + : type_caster {}; + +template +struct make_caster_impl::value>> + : decltype(pybind11_select_caster(static_cast(nullptr))) {}; + +template +using make_caster = make_caster_impl>; + +#else + template -using make_caster = type_caster>; +using make_caster = decltype(pybind11_select_caster(static_cast *>(nullptr))); + +#endif // Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T template @@ -1007,8 +1034,8 @@ struct return_value_policy_override< }; // Basic python -> C++ casting; throws if casting fails -template -type_caster &load_type(type_caster &conv, const handle &handle) { +template +Caster &load_type(Caster &conv, const handle &handle) { if (!conv.load(handle, true)) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) throw cast_error("Unable to cast Python instance to C++ type (#define " @@ -1021,11 +1048,17 @@ type_caster &load_type(type_caster &conv, const handle &ha } return conv; } + +template +type_caster &load_type(type_caster &conv, const handle &handle) { + return load_type>(conv, handle); +} + // Wrapper around the above that also constructs and returns a type_caster template make_caster load_type(const handle &handle) { make_caster conv; - load_type(conv, handle); + load_type(conv, handle); return conv; } @@ -1152,7 +1185,7 @@ using override_caster_t = conditional_t enable_if_t::value, T> cast_ref(object &&o, make_caster &caster) { - return cast_op(load_type(caster, o)); + return cast_op(load_type(caster, o)); } template enable_if_t::value, T> cast_ref(object &&, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bce8cceb1e..66ee50de68 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -128,6 +128,7 @@ set(PYBIND11_TEST_FILES test_custom_type_casters test_custom_type_setup test_docstring_options + test_docs_advanced_cast_custom test_eigen test_enum test_eval @@ -147,6 +148,7 @@ set(PYBIND11_TEST_FILES test_operator_overloading test_pickling test_pytypes + test_select_caster test_sequences_and_iterators test_smart_ptr test_stl diff --git a/tests/test_docs_advanced_cast_custom.cpp b/tests/test_docs_advanced_cast_custom.cpp new file mode 100644 index 0000000000..e45b8b4e95 --- /dev/null +++ b/tests/test_docs_advanced_cast_custom.cpp @@ -0,0 +1,99 @@ +// Copyright (c) 2022 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// ######################################################################### +// PLEASE UPDATE docs/advanced/cast/custom.rst IF ANY CHANGES ARE MADE HERE. +// ######################################################################### + +#include "pybind11_tests.h" + +#include + +// 0: Use the older pybind11::detail::type_caster specialization mechanism. +// 1: Use pybind11_select_caster declaration in user_space. +// 2: Use pybind11_select_caster friend function declaration. +#ifndef PYBIND11_TEST_DOCS_ADVANCED_CAST_CUSTOM_ALTERNATIVE +# define PYBIND11_TEST_DOCS_ADVANCED_CAST_CUSTOM_ALTERNATIVE 1 +#endif + +namespace user_space { + +#if PYBIND11_TEST_DOCS_ADVANCED_CAST_CUSTOM_ALTERNATIVE == 2 +struct inty_type_caster; +#endif + +struct inty { + long long_value; +#if PYBIND11_TEST_DOCS_ADVANCED_CAST_CUSTOM_ALTERNATIVE == 2 + friend inty_type_caster pybind11_select_caster(inty *); +#endif +}; + +std::string to_string(const inty &s) { return std::to_string(s.long_value); } + +inty return_42() { return inty{42}; } + +} // namespace user_space + +namespace user_space { + +struct inty_type_caster { +public: + // This macro establishes the name 'inty' in function signatures and declares a local variable + // 'value' of type inty. + PYBIND11_TYPE_CASTER(inty, pybind11::detail::const_name("inty")); + + // Python -> C++: convert a PyObject into an inty instance or return false upon failure. The + // second argument indicates whether implicit conversions should be allowed. + bool load(pybind11::handle src, bool) { + // Extract PyObject from handle. + PyObject *source = src.ptr(); + // Try converting into a Python integer value. + PyObject *tmp = PyNumber_Long(source); + if (!tmp) { + return false; + } + // Now try to convert into a C++ int. + value.long_value = PyLong_AsLong(tmp); + Py_DECREF(tmp); + // Ensure PyLong_AsLong succeeded (to catch out-of-range errors etc). + if (PyErr_Occurred()) { + PyErr_Clear(); + return false; + } + return true; + } + + // C++ -> Python: convert an inty instance into a Python object. The second and third arguments + // are used to indicate the return value policy and parent object (for + // return_value_policy::reference_internal) and are often ignored by custom casters. + static pybind11::handle + cast(inty src, pybind11::return_value_policy /* policy */, pybind11::handle /* parent */) { + return PyLong_FromLong(src.long_value); + } +}; + +} // namespace user_space + +#if PYBIND11_TEST_DOCS_ADVANCED_CAST_CUSTOM_ALTERNATIVE == 1 +namespace user_space { + +inty_type_caster pybind11_select_caster(inty *); + +} // namespace user_space +#endif + +#if PYBIND11_TEST_DOCS_ADVANCED_CAST_CUSTOM_ALTERNATIVE == 0 +namespace pybind11 { +namespace detail { +template <> +struct type_caster : user_space::inty_type_caster {}; +} // namespace detail +} // namespace pybind11 +#endif + +TEST_SUBMODULE(docs_advanced_cast_custom, m) { + m.def("to_string", user_space::to_string); + m.def("return_42", user_space::return_42); +} diff --git a/tests/test_docs_advanced_cast_custom.py b/tests/test_docs_advanced_cast_custom.py new file mode 100644 index 0000000000..5df41c07bf --- /dev/null +++ b/tests/test_docs_advanced_cast_custom.py @@ -0,0 +1,6 @@ +from pybind11_tests import docs_advanced_cast_custom as m + + +def test_all(): + assert m.to_string(135) == "135" + assert m.return_42() == 42 diff --git a/tests/test_select_caster.cpp b/tests/test_select_caster.cpp new file mode 100644 index 0000000000..b5e55d46d6 --- /dev/null +++ b/tests/test_select_caster.cpp @@ -0,0 +1,89 @@ +// Copyright (c) 2022 The Pybind Development Team. +// All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "pybind11_tests.h" + +namespace have_a_ns { +struct mock_caster { + static int num() { return 101; } +}; +struct type_mock { + friend mock_caster pybind11_select_caster(type_mock *); +}; + +} // namespace have_a_ns + +// namespace global { +struct global_ns_type_mock {}; +struct global_ns_mock_caster { + static int num() { return 202; } +}; +global_ns_mock_caster pybind11_select_caster(global_ns_type_mock *); +// } // namespace global + +namespace { +struct unnamed_ns_type_mock {}; +struct unnamed_ns_mock_caster { + static int num() { return 303; } +}; +// Some GCCs may not need this, but there is very little to gain for making this condition more +// complex. +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wunused-function" +#endif +PYBIND11_MAYBE_UNUSED unnamed_ns_mock_caster pybind11_select_caster(unnamed_ns_type_mock *); +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) +# pragma GCC diagnostic pop +#endif +} // namespace + +namespace mrc_ns { // minimal real caster + +struct minimal_real_caster; + +struct type_mrc { + int value = -9999; + friend minimal_real_caster pybind11_select_caster(type_mrc *); +}; + +struct minimal_real_caster { + static constexpr auto name = py::detail::const_name(); + + static py::handle + cast(type_mrc const &src, py::return_value_policy /*policy*/, py::handle /*parent*/) { + return py::int_(src.value + 1000).release(); + } + + // Maximizing simplicity. This will go terribly wrong for other arg types. + template + using cast_op_type = const type_mrc &; + + // NOLINTNEXTLINE(google-explicit-constructor) + operator type_mrc const &() { + static type_mrc obj; + obj.value = 404; + return obj; + } + + bool load(py::handle src, bool /*convert*/) { + // Only accepts str, but the value is ignored. + return py::isinstance(src); + } +}; + +} // namespace mrc_ns + +TEST_SUBMODULE(select_caster, m) { + m.def("have_a_ns_num", []() { return py::detail::make_caster::num(); }); + m.def("global_ns_num", []() { return py::detail::make_caster::num(); }); + m.def("unnamed_ns_num", []() { return py::detail::make_caster::num(); }); + + m.def("mrc_return", []() { + mrc_ns::type_mrc obj; + obj.value = 505; + return obj; + }); + m.def("mrc_arg", [](mrc_ns::type_mrc const &obj) { return obj.value + 2000; }); +} diff --git a/tests/test_select_caster.py b/tests/test_select_caster.py new file mode 100644 index 0000000000..5d515d38ed --- /dev/null +++ b/tests/test_select_caster.py @@ -0,0 +1,17 @@ +import pytest + +from pybind11_tests import select_caster as m + + +def test_mock_casters(): + assert m.have_a_ns_num() == 101 + assert m.global_ns_num() == 202 + assert m.unnamed_ns_num() == 303 + + +def test_minimal_real_caster(): + assert m.mrc_return() == 1505 + assert m.mrc_arg("ignored") == 2404 + with pytest.raises(TypeError) as excinfo: + m.mrc_arg(None) + assert "(arg0: mrc_ns::type_mrc) -> int" in str(excinfo.value)