Revert "[MLIR][Python] use FetchContent_Declare for nanobind and remove pybind (#161230)" (#162309)

This reverts commit 84a2148.

This gives us more time to work out the alternative and also people to
migrate

Author: makslevental

.ci/all_requirements.txt

@@ -194,6 +194,10 @@ ml-dtypes==0.5.1 ; python_version < "3.13" \
     --hash=sha256:d13755f8e8445b3870114e5b6240facaa7cb0c3361e54beba3e07fa912a6e12b \
     --hash=sha256:fd918d4e6a4e0c110e2e05be7a7814d10dc1b95872accbf6512b80a109b71ae1
     # via -r mlir/python/requirements.txt
+nanobind==2.9.2 \
+    --hash=sha256:c37957ffd5eac7eda349cff3622ecd32e5ee1244ecc912c99b5bc8188bafd16e \
+    --hash=sha256:e7608472de99d375759814cab3e2c94aba3f9ec80e62cfef8ced495ca5c27d6e
+    # via -r mlir/python/requirements.txt
 numpy==2.0.2 \
     --hash=sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a \
     --hash=sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195 \
@@ -295,6 +299,10 @@ pyasn1-modules==0.4.2 \
     --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \
     --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6
     # via google-auth
+pybind11==2.13.6 \
+    --hash=sha256:237c41e29157b962835d356b370ededd57594a26d5894a795960f0047cb5caf5 \
+    --hash=sha256:ba6af10348c12b24e92fa086b39cfba0eff619b61ac77c406167d813b096d39a
+    # via -r mlir/python/requirements.txt
 pyyaml==6.0.1 \
     --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \
     --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \

mlir/cmake/modules/AddMLIRPython.cmake

@@ -123,12 +123,12 @@ function(mlir_generate_type_stubs)
     "IMPORT_PATHS;DEPENDS_TARGETS;OUTPUTS;DEPENDS_TARGET_SRC_DEPS"
     ${ARGN})
 
-  # for people installing a distro (e.g., pip install) of nanobind
+  # for people doing find_package(nanobind)
   if(EXISTS ${nanobind_DIR}/../src/stubgen.py)
     set(NB_STUBGEN "${nanobind_DIR}/../src/stubgen.py")
   elseif(EXISTS ${nanobind_DIR}/../stubgen.py)
     set(NB_STUBGEN "${nanobind_DIR}/../stubgen.py")
-  # for people using nanobind git source tree (e.g., FetchContent_Declare and FetchContent_MakeAvailable)
+  # for people using FetchContent_Declare and FetchContent_MakeAvailable
   elseif(EXISTS ${nanobind_SOURCE_DIR}/src/stubgen.py)
     set(NB_STUBGEN "${nanobind_SOURCE_DIR}/src/stubgen.py")
   elseif(EXISTS ${nanobind_SOURCE_DIR}/stubgen.py)
@@ -226,10 +226,11 @@ endfunction()
 #   EMBED_CAPI_LINK_LIBS: Dependent CAPI libraries that this extension depends
 #     on. These will be collected for all extensions and put into an
 #     aggregate dylib that is linked against.
+#   PYTHON_BINDINGS_LIBRARY: Either pybind11 or nanobind.
 function(declare_mlir_python_extension name)
   cmake_parse_arguments(ARG
     ""
-    "ROOT_DIR;MODULE_NAME;ADD_TO_PARENT"
+    "ROOT_DIR;MODULE_NAME;ADD_TO_PARENT;PYTHON_BINDINGS_LIBRARY"
     "SOURCES;PRIVATE_LINK_LIBS;EMBED_CAPI_LINK_LIBS"
     ${ARGN})
 
@@ -238,15 +239,20 @@ function(declare_mlir_python_extension name)
   endif()
   set(_install_destination "src/python/${name}")
 
+  if(NOT ARG_PYTHON_BINDINGS_LIBRARY)
+    set(ARG_PYTHON_BINDINGS_LIBRARY "pybind11")
+  endif()
+
   add_library(${name} INTERFACE)
   set_target_properties(${name} PROPERTIES
     # Yes: Leading-lowercase property names are load bearing and the recommended
     # way to do this: https://gitlab.kitware.com/cmake/cmake/-/issues/19261
-    EXPORT_PROPERTIES "mlir_python_SOURCES_TYPE;mlir_python_EXTENSION_MODULE_NAME;mlir_python_EMBED_CAPI_LINK_LIBS;mlir_python_DEPENDS"
+    EXPORT_PROPERTIES "mlir_python_SOURCES_TYPE;mlir_python_EXTENSION_MODULE_NAME;mlir_python_EMBED_CAPI_LINK_LIBS;mlir_python_DEPENDS;mlir_python_BINDINGS_LIBRARY"
     mlir_python_SOURCES_TYPE extension
     mlir_python_EXTENSION_MODULE_NAME "${ARG_MODULE_NAME}"
     mlir_python_EMBED_CAPI_LINK_LIBS "${ARG_EMBED_CAPI_LINK_LIBS}"
     mlir_python_DEPENDS ""
+    mlir_python_BINDINGS_LIBRARY "${ARG_PYTHON_BINDINGS_LIBRARY}"
   )
 
   # Set the interface source and link_libs properties of the target
@@ -335,12 +341,14 @@ function(add_mlir_python_modules name)
     elseif(_source_type STREQUAL "extension")
       # Native CPP extension.
       get_target_property(_module_name ${sources_target} mlir_python_EXTENSION_MODULE_NAME)
+      get_target_property(_bindings_library ${sources_target} mlir_python_BINDINGS_LIBRARY)
       # Transform relative source to based on root dir.
       set(_extension_target "${modules_target}.extension.${_module_name}.dso")
       add_mlir_python_extension(${_extension_target} "${_module_name}"
         INSTALL_COMPONENT ${modules_target}
         INSTALL_DIR "${ARG_INSTALL_PREFIX}/_mlir_libs"
         OUTPUT_DIRECTORY "${ARG_ROOT_PREFIX}/_mlir_libs"
+        PYTHON_BINDINGS_LIBRARY ${_bindings_library}
         LINK_LIBS PRIVATE
           ${sources_target}
           ${ARG_COMMON_CAPI_LINK_LIBS}
@@ -745,69 +753,78 @@ endfunction()
 function(add_mlir_python_extension libname extname)
   cmake_parse_arguments(ARG
   ""
-  "INSTALL_COMPONENT;INSTALL_DIR;OUTPUT_DIRECTORY"
+  "INSTALL_COMPONENT;INSTALL_DIR;OUTPUT_DIRECTORY;PYTHON_BINDINGS_LIBRARY"
   "SOURCES;LINK_LIBS"
   ${ARGN})
   if(ARG_UNPARSED_ARGUMENTS)
     message(FATAL_ERROR "Unhandled arguments to add_mlir_python_extension(${libname}, ... : ${ARG_UNPARSED_ARGUMENTS}")
   endif()
 
   # The extension itself must be compiled with RTTI and exceptions enabled.
-  # Also, some warning classes triggered by nanobind are disabled.
+  # Also, some warning classes triggered by pybind11 are disabled.
   set(eh_rtti_enable)
   if (MSVC)
     set(eh_rtti_enable /EHsc /GR)
   elseif(LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL)
     set(eh_rtti_enable -frtti -fexceptions)
   endif ()
 
-  nanobind_add_module(${libname}
-    NB_DOMAIN ${MLIR_BINDINGS_PYTHON_NB_DOMAIN}
-    FREE_THREADED
-    ${ARG_SOURCES}
-  )
+  # The actual extension library produces a shared-object or DLL and has
+  # sources that must be compiled in accordance with pybind11 needs (RTTI and
+  # exceptions).
+  if(NOT DEFINED ARG_PYTHON_BINDINGS_LIBRARY OR ARG_PYTHON_BINDINGS_LIBRARY STREQUAL "pybind11")
+    pybind11_add_module(${libname}
+      ${ARG_SOURCES}
+    )
+  elseif(ARG_PYTHON_BINDINGS_LIBRARY STREQUAL "nanobind")
+    nanobind_add_module(${libname}
+      NB_DOMAIN ${MLIR_BINDINGS_PYTHON_NB_DOMAIN}
+      FREE_THREADED
+      ${ARG_SOURCES}
+    )
 
-  if (NOT MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES
-      AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))
-    # Avoid some warnings from upstream nanobind.
-    # If a superproject set MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES, let
-    # the super project handle compile options as it wishes.
-    get_property(NB_LIBRARY_TARGET_NAME TARGET ${libname} PROPERTY LINK_LIBRARIES)
-    target_compile_options(${NB_LIBRARY_TARGET_NAME}
-      PRIVATE
-        -Wall -Wextra -Wpedantic
-        -Wno-c++98-compat-extra-semi
-        -Wno-cast-qual
-        -Wno-covered-switch-default
-        -Wno-deprecated-literal-operator
-        -Wno-nested-anon-types
-        -Wno-unused-parameter
-        -Wno-zero-length-array
-        ${eh_rtti_enable})
-
-    target_compile_options(${libname}
-      PRIVATE
-        -Wall -Wextra -Wpedantic
-        -Wno-c++98-compat-extra-semi
-        -Wno-cast-qual
-        -Wno-covered-switch-default
-        -Wno-deprecated-literal-operator
-        -Wno-nested-anon-types
-        -Wno-unused-parameter
-        -Wno-zero-length-array
-        ${eh_rtti_enable})
-  endif()
+    if (NOT MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES
+        AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))
+      # Avoid some warnings from upstream nanobind.
+      # If a superproject set MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES, let
+      # the super project handle compile options as it wishes.
+      get_property(NB_LIBRARY_TARGET_NAME TARGET ${libname} PROPERTY LINK_LIBRARIES)
+      target_compile_options(${NB_LIBRARY_TARGET_NAME}
+        PRIVATE
+          -Wall -Wextra -Wpedantic
+          -Wno-c++98-compat-extra-semi
+          -Wno-cast-qual
+          -Wno-covered-switch-default
+          -Wno-deprecated-literal-operator
+          -Wno-nested-anon-types
+          -Wno-unused-parameter
+          -Wno-zero-length-array
+          ${eh_rtti_enable})
+
+      target_compile_options(${libname}
+        PRIVATE
+          -Wall -Wextra -Wpedantic
+          -Wno-c++98-compat-extra-semi
+          -Wno-cast-qual
+          -Wno-covered-switch-default
+          -Wno-deprecated-literal-operator
+          -Wno-nested-anon-types
+          -Wno-unused-parameter
+          -Wno-zero-length-array
+          ${eh_rtti_enable})
+    endif()
 
-  if(APPLE)
-    # NanobindAdaptors.h uses PyClassMethod_New to build `pure_subclass`es but nanobind
-    # doesn't declare this API as undefined in its linker flags. So we need to declare it as such
-    # for downstream users that do not do something like `-undefined dynamic_lookup`.
-    # Same for the rest.
-    target_link_options(${libname} PUBLIC
-      "LINKER:-U,_PyClassMethod_New"
-      "LINKER:-U,_PyCode_Addr2Location"
-      "LINKER:-U,_PyFrame_GetLasti"
-    )
+    if(APPLE)
+      # NanobindAdaptors.h uses PyClassMethod_New to build `pure_subclass`es but nanobind
+      # doesn't declare this API as undefined in its linker flags. So we need to declare it as such
+      # for downstream users that do not do something like `-undefined dynamic_lookup`.
+      # Same for the rest.
+      target_link_options(${libname} PUBLIC
+        "LINKER:-U,_PyClassMethod_New"
+        "LINKER:-U,_PyCode_Addr2Location"
+        "LINKER:-U,_PyFrame_GetLasti"
+      )
+    endif()
   endif()
 
   target_compile_options(${libname} PRIVATE ${eh_rtti_enable})
@@ -845,11 +862,11 @@ function(add_mlir_python_extension libname extname)
   if(WIN32)
     # On Windows, pyconfig.h (and by extension python.h) hardcode the version of the
     # python library which will be used for linkage depending on the flavor of the build.
-    # nanobind has a workaround which depends on the definition of Py_DEBUG (if Py_DEBUG
-    # is not passed in as a compile definition, nanobind undefs _DEBUG when including
+    # pybind11 has a workaround which depends on the definition of Py_DEBUG (if Py_DEBUG
+    # is not passed in as a compile definition, pybind11 undefs _DEBUG when including
     # python.h, so that the release python library would be used).
-    # Since mlir uses nanobind, we can leverage their workaround by never directly
-    # pyconfig.h or python.h and instead relying on the nanobind headers to include the
+    # Since mlir uses pybind11, we can leverage their workaround by never directly
+    # pyconfig.h or python.h and instead relying on the pybind11 headers to include the
     # necessary python headers. This results in mlir always linking against the
     # release python library via the (undocumented) cmake property Python3_LIBRARY_RELEASE.
     target_link_libraries(${libname} PRIVATE ${Python3_LIBRARY_RELEASE})

mlir/cmake/modules/MLIRDetectPythonEnv.cmake

@@ -46,20 +46,81 @@ macro(mlir_configure_python_dev_packages)
     message(STATUS "Found python include dirs: ${Python3_INCLUDE_DIRS}")
     message(STATUS "Found python libraries: ${Python3_LIBRARIES}")
     message(STATUS "Found numpy v${Python3_NumPy_VERSION}: ${Python3_NumPy_INCLUDE_DIRS}")
-    message(STATUS "Python extension suffix for modules: '${Python3_SOABI}'")
-    if(nanobind_DIR)
-      message(STATUS "Using explicit nanobind cmake directory: ${nanobind_DIR} (-Dnanobind_DIR to change)")
-      find_package(nanobind 2.9 CONFIG REQUIRED)
-    else()
-      include(FetchContent)
-      FetchContent_Declare(
-        nanobind
-        GIT_REPOSITORY https://github.com/wjakob/nanobind.git
-        GIT_TAG        v2.9.0
-        GIT_SHALLOW    TRUE
-      )
-      FetchContent_MakeAvailable(nanobind)
-    endif()
-    message(STATUS "Found nanobind: ${NB_DIR}")
+    mlir_detect_pybind11_install()
+    find_package(pybind11 2.10 CONFIG REQUIRED)
+    message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIR}")
+    message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
+                  "suffix = '${PYTHON_MODULE_SUFFIX}', "
+                  "extension = '${PYTHON_MODULE_EXTENSION}")
+
+    mlir_detect_nanobind_install()
+    find_package(nanobind 2.9 CONFIG REQUIRED)
+    message(STATUS "Found nanobind v${nanobind_VERSION}: ${nanobind_INCLUDE_DIR}")
+    message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
+                  "suffix = '${PYTHON_MODULE_SUFFIX}', "
+                  "extension = '${PYTHON_MODULE_EXTENSION}")
   endif()
 endmacro()
+
+# Detects a pybind11 package installed in the current python environment
+# and sets variables to allow it to be found. This allows pybind11 to be
+# installed via pip, which typically yields a much more recent version than
+# the OS install, which will be available otherwise.
+function(mlir_detect_pybind11_install)
+  if(pybind11_DIR)
+    message(STATUS "Using explicit pybind11 cmake directory: ${pybind11_DIR} (-Dpybind11_DIR to change)")
+  else()
+    message(STATUS "Checking for pybind11 in python path...")
+    execute_process(
+      COMMAND "${Python3_EXECUTABLE}"
+      -c "import pybind11;print(pybind11.get_cmake_dir(), end='')"
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      RESULT_VARIABLE STATUS
+      OUTPUT_VARIABLE PACKAGE_DIR
+      ERROR_QUIET)
+    if(NOT STATUS EQUAL "0")
+      message(STATUS "not found (install via 'pip install pybind11' or set pybind11_DIR)")
+      return()
+    endif()
+    message(STATUS "found (${PACKAGE_DIR})")
+    set(pybind11_DIR "${PACKAGE_DIR}" PARENT_SCOPE)
+  endif()
+endfunction()
+
+
+# Detects a nanobind package installed in the current python environment
+# and sets variables to allow it to be found. This allows nanobind to be
+# installed via pip, which typically yields a much more recent version than
+# the OS install, which will be available otherwise.
+function(mlir_detect_nanobind_install)
+  if(nanobind_DIR)
+    message(STATUS "Using explicit nanobind cmake directory: ${nanobind_DIR} (-Dnanobind_DIR to change)")
+  else()
+    message(STATUS "Checking for nanobind in python path...")
+    execute_process(
+      COMMAND "${Python3_EXECUTABLE}"
+      -c "import nanobind;print(nanobind.cmake_dir(), end='')"
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      RESULT_VARIABLE STATUS
+      OUTPUT_VARIABLE PACKAGE_DIR
+      ERROR_QUIET)
+    if(NOT STATUS EQUAL "0")
+      message(STATUS "not found (install via 'pip install nanobind' or set nanobind_DIR)")
+      return()
+    endif()
+    message(STATUS "found (${PACKAGE_DIR})")
+    set(nanobind_DIR "${PACKAGE_DIR}" PARENT_SCOPE)
+    execute_process(
+      COMMAND "${Python3_EXECUTABLE}"
+      -c "import nanobind;print(nanobind.include_dir(), end='')"
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      RESULT_VARIABLE STATUS
+      OUTPUT_VARIABLE PACKAGE_DIR
+      ERROR_QUIET)
+    if(NOT STATUS EQUAL "0")
+      message(STATUS "not found (install via 'pip install nanobind' or set nanobind_DIR)")
+      return()
+    endif()
+    set(nanobind_INCLUDE_DIR "${PACKAGE_DIR}" PARENT_SCOPE)
+  endif()
+endfunction()

mlir/docs/Dialects/Linalg/OpDSL.md

@@ -16,15 +16,15 @@ corresponding `linalg.generic` IR for the composition.
 ## Basic usage
 
 The tool is bundled with the MLIR Python bindings. To use from the CMake build
-tree, MLIR must be built with Python bindings enabled
+tree, MLIR must be build with Python bindings enabled
 (`-DMLIR_ENABLE_BINDINGS_PYTHON=ON`). Then add the `python` directory in the
 build tree to your `PYTHONPATH` environment variable (i.e. `export
 PYTHONPATH=$PWD/build/tools/mlir/python_packages/mlir_core`). Optionally, use an
 installed MLIR package, if available, to avoid building.
 
 ```shell
 # Dump the `core_named_ops.py` module as YAML.
-python -m mlir.dialects.linalg.opdsl.dump_oplib.ops.core_named_ops
+python -m mlir.dialects.linalg.opdsl.dump_oplib .ops.core_named_ops
 ```
 
 Alternatively, run the `$PWD/build/bin/update_core_linalg_named_ops.sh` script,

mlir/examples/standalone/pyproject.toml

@@ -23,7 +23,9 @@ Discussions = "https://discourse.llvm.org/"
 [build-system]
 requires = [
     "scikit-build-core>=0.10.7",
-    "typing_extensions>=4.12.2"
+    "typing_extensions>=4.12.2",
+    "nanobind>=2.9, <3.0",
+    "pybind11>=2.10.0, <=2.13.6",
 ]
 build-backend = "scikit_build_core.build"
 

mlir/examples/standalone/python/CMakeLists.txt

@@ -16,10 +16,27 @@ declare_mlir_dialect_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir_standalone"
   TD_FILE dialects/StandaloneOps.td
   SOURCES
+    dialects/standalone_pybind11.py
     dialects/standalone_nanobind.py
     _mlir_libs/_standaloneDialectsNanobind/py.typed
   DIALECT_NAME standalone)
 
+
+declare_mlir_python_extension(StandalonePythonSources.Pybind11Extension
+  MODULE_NAME _standaloneDialectsPybind11
+  ADD_TO_PARENT StandalonePythonSources
+  SOURCES
+    StandaloneExtensionPybind11.cpp
+  PRIVATE_LINK_LIBS
+    LLVMSupport
+  EMBED_CAPI_LINK_LIBS
+    MLIRCAPIIR
+    MLIRCAPIArith
+    MLIRCAPITransforms
+    StandaloneCAPI
+  PYTHON_BINDINGS_LIBRARY pybind11
+)
+
 declare_mlir_python_extension(StandalonePythonSources.NanobindExtension
   MODULE_NAME _standaloneDialectsNanobind
   ADD_TO_PARENT StandalonePythonSources
@@ -32,6 +49,7 @@ declare_mlir_python_extension(StandalonePythonSources.NanobindExtension
     MLIRCAPIArith
     MLIRCAPITransforms
     StandaloneCAPI
+  PYTHON_BINDINGS_LIBRARY nanobind
 )