diff --git a/docs/type_caster_iwyu.rst b/docs/type_caster_iwyu.rst new file mode 100644 index 00000000..c909db67 --- /dev/null +++ b/docs/type_caster_iwyu.rst @@ -0,0 +1,68 @@ +pybind11 ``type_caster`` design & `clangd Include Cleaner`_ +=========================================================== + +This document is purely to explain — in a nutshell — why pybind11 include +files with ``type_caster`` specializations require this: + +.. code-block:: cpp + + // IWYU pragma: always_keep + +For general technical information about the pybind11 ``type_caster`` mechanism +see the `Custom type casters`_ section. + +The problem +----------- + +The `clangd Include Cleaner`_ cannot possibly have the ability to detect which +``type_caster`` specialization is the correct one to use in a given C++ +translation unit. Without IWYU pragmas, Include Cleaner is likely to suggest +removing ``type_caster`` include files. + +The consequences +---------------- + +1. Incorrect runtime behavior. +2. `ODR violations`_. + +Example for 1. Incorrect runtime behavior +----------------------------------------- + +.. code-block:: cpp + + #include + #include + +If the stl.h include is removed (as suggested by Include Cleaner), conversions +between e.g. ``std::vector`` and Python ``list`` will no longer work. In most +cases this will be very obvious at runtime, but there are less obvious +situations, most notably for ``type_caster`` specializations that inherit from +``type_caster_base`` (e.g. `pybind11_abseil/status_caster.h`_). +Some conversions may still work correctly, while others will have unexpected +behavior. + +Explanation for 2. ODR violations +--------------------------------- + +This problem only arises if two or more C++ translation units are statically +linked (e.g. one ``.so`` Python extension built from multiple .cpp files), or +all visibility-restricting features (e.g. ``-fvisibility=hidden``, +``namespace pybind11`` ``__attribute__((visibility("hidden")))``, +``RTLD_LOCAL``) are disabled. + +Consider the same simple code example as above: if the stl.h include is missing +(as suggested by Include Cleaner) in one translation unit, but not in another +(maybe because Include Cleaner was never applied), the resulting Python +extension will have link incompatibilities, which is often referred to as +ODR violations. The behavior is entirely undefined. For better or worse, the +extension may perform as desired for years, until one day it suddenly does +not, because of some unrelated change in the environment (e.g. new compiler +version, a system library update), resulting in seemingly inexplicable failures + +See also: `pybind/pybind11#4022`_ (pybind11 smart_holder branch) + +.. _`clangd Include Cleaner`: https://clangd.llvm.org/design/include-cleaner +.. _`Custom type casters`: https://pybind11.readthedocs.io/en/stable/advanced/cast/custom.html +.. _`ODR violations`: https://en.cppreference.com/w/cpp/language/definition +.. _`pybind11_abseil/status_caster.h`: https://github.com/pybind/pybind11_abseil/blob/4b883e48ae749ff984c220484d54fdeb0cb4302c/pybind11_abseil/status_caster.h#L52-L53). +.. _`pybind/pybind11#4022`: https://github.com/pybind/pybind11/pull/4022 diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index 7d4a2216..e7724050 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -10,7 +10,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include "cast.h" #include diff --git a/include/pybind11/buffer_info.h b/include/pybind11/buffer_info.h index b99ee8be..af7f3095 100644 --- a/include/pybind11/buffer_info.h +++ b/include/pybind11/buffer_info.h @@ -9,7 +9,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 1ff870b2..aed727e3 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -8,8 +8,11 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" #include "detail/descr.h" #include "detail/native_enum_data.h" @@ -17,6 +20,7 @@ #include "detail/type_caster_base.h" #include "detail/type_caster_odr_guard.h" #include "detail/typeid.h" +// IWYU pragma: end_exports #include "pytypes.h" #include diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index 167ea0e3..48a5e10d 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -8,6 +8,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include "pybind11.h" diff --git a/include/pybind11/complex.h b/include/pybind11/complex.h index 8a831c12..087903c5 100644 --- a/include/pybind11/complex.h +++ b/include/pybind11/complex.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include "pybind11.h" diff --git a/include/pybind11/detail/abi_platform_id.h b/include/pybind11/detail/abi_platform_id.h index 5467322f..831d9878 100644 --- a/include/pybind11/detail/abi_platform_id.h +++ b/include/pybind11/detail/abi_platform_id.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "common.h" diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index c39ea165..c73ba3a8 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "../attr.h" diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index addaf91a..f8b9d8aa 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #define PYBIND11_VERSION_MAJOR 2 diff --git a/include/pybind11/detail/cross_extension_shared_state.h b/include/pybind11/detail/cross_extension_shared_state.h index ed36c830..b3e84d79 100644 --- a/include/pybind11/detail/cross_extension_shared_state.h +++ b/include/pybind11/detail/cross_extension_shared_state.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #define PYBIND11_HAS_CROSS_EXTENSION_SHARED_STATE diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index d447046d..304fb8d6 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -10,6 +10,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "common.h" diff --git a/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h b/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h index 7c00fe98..a4d38dc5 100644 --- a/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h +++ b/include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/smart_holder.h" + #pragma once #include "common.h" diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h index 9f59772f..c5877beb 100644 --- a/include/pybind11/detail/function_record_pyobject.h +++ b/include/pybind11/detail/function_record_pyobject.h @@ -4,6 +4,8 @@ // For background see the description of PR google/pywrapcc#30099. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "../attr.h" diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 0b601d6d..8b9b1a71 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "class.h" diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 5d330f88..c049728f 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "../pytypes.h" diff --git a/include/pybind11/detail/native_enum_data.h b/include/pybind11/detail/native_enum_data.h index 11e7f792..f1744079 100644 --- a/include/pybind11/detail/native_enum_data.h +++ b/include/pybind11/detail/native_enum_data.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #define PYBIND11_HAS_NATIVE_ENUM diff --git a/include/pybind11/detail/smart_holder_poc.h b/include/pybind11/detail/smart_holder_poc.h index 89742ab2..c61442ac 100644 --- a/include/pybind11/detail/smart_holder_poc.h +++ b/include/pybind11/detail/smart_holder_poc.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/smart_holder.h" + /* Proof-of-Concept for smart pointer interoperability. High-level aspects: diff --git a/include/pybind11/detail/smart_holder_sfinae_hooks_only.h b/include/pybind11/detail/smart_holder_sfinae_hooks_only.h index 5114607f..02377529 100644 --- a/include/pybind11/detail/smart_holder_sfinae_hooks_only.h +++ b/include/pybind11/detail/smart_holder_sfinae_hooks_only.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "common.h" diff --git a/include/pybind11/detail/smart_holder_type_casters.h b/include/pybind11/detail/smart_holder_type_casters.h index d138613f..87cd4c1a 100644 --- a/include/pybind11/detail/smart_holder_type_casters.h +++ b/include/pybind11/detail/smart_holder_type_casters.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/smart_holder.h" + #pragma once #include "../gil.h" diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index be7fd1cd..fb29ee18 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "../pytypes.h" diff --git a/include/pybind11/detail/type_caster_odr_guard.h b/include/pybind11/detail/type_caster_odr_guard.h index 46ea1af8..f118484a 100644 --- a/include/pybind11/detail/type_caster_odr_guard.h +++ b/include/pybind11/detail/type_caster_odr_guard.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "descr.h" diff --git a/include/pybind11/detail/type_map.h b/include/pybind11/detail/type_map.h index 74ca7bfe..6c8daa6e 100644 --- a/include/pybind11/detail/type_map.h +++ b/include/pybind11/detail/type_map.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include "common.h" diff --git a/include/pybind11/detail/typeid.h b/include/pybind11/detail/typeid.h index a67b5213..cbea6787 100644 --- a/include/pybind11/detail/typeid.h +++ b/include/pybind11/detail/typeid.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: private, include "third_party/pybind11/include/pybind11/pybind11.h" + #pragma once #include diff --git a/include/pybind11/eigen/matrix.h b/include/pybind11/eigen/matrix.h index eafbca14..75c186da 100644 --- a/include/pybind11/eigen/matrix.h +++ b/include/pybind11/eigen/matrix.h @@ -7,10 +7,14 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include "../numpy.h" +// IWYU pragma: begin_exports #include "common.h" +// IWYU pragma: end_exports /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. See also: diff --git a/include/pybind11/eigen/tensor.h b/include/pybind11/eigen/tensor.h index 3e78a641..73dd9e97 100644 --- a/include/pybind11/eigen/tensor.h +++ b/include/pybind11/eigen/tensor.h @@ -5,10 +5,14 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include "../numpy.h" +// IWYU pragma: begin_exports #include "common.h" +// IWYU pragma: end_exports #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index 01c3ea8f..253b4df7 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -7,6 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #define PYBIND11_HAS_TYPE_CASTER_STD_FUNCTION_SPECIALIZATIONS diff --git a/include/pybind11/gil.h b/include/pybind11/gil.h index da22f48d..0e4088bd 100644 --- a/include/pybind11/gil.h +++ b/include/pybind11/gil.h @@ -9,12 +9,16 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include #if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +// IWYU pragma: begin_exports # include "detail/internals.h" +// IWYU pragma: end_exports #endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) diff --git a/include/pybind11/gil_safe_call_once.h b/include/pybind11/gil_safe_call_once.h index eaf84d16..144909ae 100644 --- a/include/pybind11/gil_safe_call_once.h +++ b/include/pybind11/gil_safe_call_once.h @@ -2,7 +2,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include "gil.h" #include diff --git a/include/pybind11/native_enum.h b/include/pybind11/native_enum.h index 13c15ab9..ddf979ea 100644 --- a/include/pybind11/native_enum.h +++ b/include/pybind11/native_enum.h @@ -4,9 +4,11 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" #include "detail/native_enum_data.h" #include "detail/type_caster_base.h" +// IWYU pragma: end_exports #include "cast.h" #include diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 8551aa26..47195dfa 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -7,10 +7,14 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include "pybind11.h" +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include "complex.h" #include "gil_safe_call_once.h" #include "pytypes.h" diff --git a/include/pybind11/options.h b/include/pybind11/options.h index 1b212252..381d84ef 100644 --- a/include/pybind11/options.h +++ b/include/pybind11/options.h @@ -9,7 +9,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index c20ff766..ae7361e9 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -10,11 +10,13 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/class.h" #include "detail/function_record_pyobject.h" #include "detail/init.h" #include "detail/native_enum_data.h" #include "detail/smart_holder_sfinae_hooks_only.h" +// IWYU pragma: end_exports #include "attr.h" #include "gil.h" #include "gil_safe_call_once.h" diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 2492b774..c52efd89 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -9,7 +9,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include "buffer_info.h" #include diff --git a/include/pybind11/smart_holder.h b/include/pybind11/smart_holder.h index 03ec4bb8..56b14a54 100644 --- a/include/pybind11/smart_holder.h +++ b/include/pybind11/smart_holder.h @@ -6,7 +6,9 @@ #include "pybind11.h" #include "detail/common.h" +// IWYU pragma: begin_exports #include "detail/smart_holder_type_casters.h" +// IWYU pragma: end_exports #undef PYBIND11_SH_AVL // Undoing #define in pybind11.h diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 168c1b23..3f0c75ee 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -7,12 +7,16 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include "pybind11.h" +// IWYU pragma: begin_exports #include "detail/common.h" #include "detail/descr.h" #include "detail/type_caster_base.h" +// IWYU pragma: end_exports #include #include diff --git a/include/pybind11/stl/filesystem.h b/include/pybind11/stl/filesystem.h index e26f4217..83718460 100644 --- a/include/pybind11/stl/filesystem.h +++ b/include/pybind11/stl/filesystem.h @@ -2,6 +2,8 @@ // All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once #include "../pybind11.h" diff --git a/include/pybind11/stl_bind.h b/include/pybind11/stl_bind.h index 3d24bc5a..88fe11ab 100644 --- a/include/pybind11/stl_bind.h +++ b/include/pybind11/stl_bind.h @@ -7,10 +7,14 @@ BSD-style license that can be found in the LICENSE file. */ +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" #include "detail/type_caster_base.h" +// IWYU pragma: end_exports #include "cast.h" #include "operators.h" diff --git a/include/pybind11/type_caster_pyobject_ptr.h b/include/pybind11/type_caster_pyobject_ptr.h index 7789c040..279a2447 100644 --- a/include/pybind11/type_caster_pyobject_ptr.h +++ b/include/pybind11/type_caster_pyobject_ptr.h @@ -1,9 +1,13 @@ // Copyright (c) 2023 The pybind Community. +// IWYU pragma: always_keep // See pybind11/docs/type_caster_iwyu.rst + #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" #include "detail/descr.h" +// IWYU pragma: end_exports #include "cast.h" #include "pytypes.h" diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index b7b1a4e5..235281ac 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -10,7 +10,9 @@ #pragma once +// IWYU pragma: begin_exports #include "detail/common.h" +// IWYU pragma: end_exports #include "cast.h" #include "pytypes.h"