[MLIR][Python] use FetchContent_Declare for nanobind and remove pybind (#161230)

Inspired by this comment
#157930 (comment)
(and long-standing issues related to finding nanobind/pybind in the
right place), this PR moves to using `FetchContent_Declare` to get the
nanobind dependency. This is pretty standard (see e.g.,
[IREE](https://github.com/iree-org/iree/blob/cf60359b7443b0e92e15fb6ffc011525dc40e772/CMakeLists.txt#L842-L848)).
This PR also removes pybind which has been deprecated for almost a year
(#117922) and which isn't
compatible (for whatever reason) with `FetchContent_Declare`.

---------

Co-authored-by: Jacques Pienaar <jpienaar@google.com>

Author: makslevental

.ci/all_requirements.txt

@@ -194,10 +194,6 @@ ml-dtypes==0.5.1 ; python_version < "3.13" \
     --hash=sha256:d13755f8e8445b3870114e5b6240facaa7cb0c3361e54beba3e07fa912a6e12b \
     --hash=sha256:fd918d4e6a4e0c110e2e05be7a7814d10dc1b95872accbf6512b80a109b71ae1
     # via -r mlir/python/requirements.txt
-nanobind==2.9.2 \
-    --hash=sha256:c37957ffd5eac7eda349cff3622ecd32e5ee1244ecc912c99b5bc8188bafd16e \
-    --hash=sha256:e7608472de99d375759814cab3e2c94aba3f9ec80e62cfef8ced495ca5c27d6e
-    # via -r mlir/python/requirements.txt
 numpy==2.0.2 \
     --hash=sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a \
     --hash=sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195 \
@@ -299,10 +295,6 @@ pyasn1-modules==0.4.2 \
     --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \
     --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6
     # via google-auth
-pybind11==2.13.6 \
-    --hash=sha256:237c41e29157b962835d356b370ededd57594a26d5894a795960f0047cb5caf5 \
-    --hash=sha256:ba6af10348c12b24e92fa086b39cfba0eff619b61ac77c406167d813b096d39a
-    # via -r mlir/python/requirements.txt
 pyyaml==6.0.1 \
     --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \
     --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \

mlir/cmake/modules/AddMLIRPython.cmake

@@ -123,12 +123,12 @@ function(mlir_generate_type_stubs)
     "IMPORT_PATHS;DEPENDS_TARGETS;OUTPUTS;DEPENDS_TARGET_SRC_DEPS"
     ${ARGN})
 
-  # for people doing find_package(nanobind)
+  # for people installing a distro (e.g., pip install) of nanobind
   if(EXISTS ${nanobind_DIR}/../src/stubgen.py)
     set(NB_STUBGEN "${nanobind_DIR}/../src/stubgen.py")
   elseif(EXISTS ${nanobind_DIR}/../stubgen.py)
     set(NB_STUBGEN "${nanobind_DIR}/../stubgen.py")
-  # for people using FetchContent_Declare and FetchContent_MakeAvailable
+  # for people using nanobind git source tree (e.g., FetchContent_Declare and FetchContent_MakeAvailable)
   elseif(EXISTS ${nanobind_SOURCE_DIR}/src/stubgen.py)
     set(NB_STUBGEN "${nanobind_SOURCE_DIR}/src/stubgen.py")
   elseif(EXISTS ${nanobind_SOURCE_DIR}/stubgen.py)
@@ -226,11 +226,10 @@ endfunction()
 #   EMBED_CAPI_LINK_LIBS: Dependent CAPI libraries that this extension depends
 #     on. These will be collected for all extensions and put into an
 #     aggregate dylib that is linked against.
-#   PYTHON_BINDINGS_LIBRARY: Either pybind11 or nanobind.
 function(declare_mlir_python_extension name)
   cmake_parse_arguments(ARG
     ""
-    "ROOT_DIR;MODULE_NAME;ADD_TO_PARENT;PYTHON_BINDINGS_LIBRARY"
+    "ROOT_DIR;MODULE_NAME;ADD_TO_PARENT"
     "SOURCES;PRIVATE_LINK_LIBS;EMBED_CAPI_LINK_LIBS"
     ${ARGN})
 
@@ -239,20 +238,15 @@ function(declare_mlir_python_extension name)
   endif()
   set(_install_destination "src/python/${name}")
 
-  if(NOT ARG_PYTHON_BINDINGS_LIBRARY)
-    set(ARG_PYTHON_BINDINGS_LIBRARY "pybind11")
-  endif()
-
   add_library(${name} INTERFACE)
   set_target_properties(${name} PROPERTIES
     # Yes: Leading-lowercase property names are load bearing and the recommended
     # way to do this: https://gitlab.kitware.com/cmake/cmake/-/issues/19261
-    EXPORT_PROPERTIES "mlir_python_SOURCES_TYPE;mlir_python_EXTENSION_MODULE_NAME;mlir_python_EMBED_CAPI_LINK_LIBS;mlir_python_DEPENDS;mlir_python_BINDINGS_LIBRARY"
+    EXPORT_PROPERTIES "mlir_python_SOURCES_TYPE;mlir_python_EXTENSION_MODULE_NAME;mlir_python_EMBED_CAPI_LINK_LIBS;mlir_python_DEPENDS"
     mlir_python_SOURCES_TYPE extension
     mlir_python_EXTENSION_MODULE_NAME "${ARG_MODULE_NAME}"
     mlir_python_EMBED_CAPI_LINK_LIBS "${ARG_EMBED_CAPI_LINK_LIBS}"
     mlir_python_DEPENDS ""
-    mlir_python_BINDINGS_LIBRARY "${ARG_PYTHON_BINDINGS_LIBRARY}"
   )
 
   # Set the interface source and link_libs properties of the target
@@ -341,14 +335,12 @@ function(add_mlir_python_modules name)
     elseif(_source_type STREQUAL "extension")
       # Native CPP extension.
       get_target_property(_module_name ${sources_target} mlir_python_EXTENSION_MODULE_NAME)
-      get_target_property(_bindings_library ${sources_target} mlir_python_BINDINGS_LIBRARY)
       # Transform relative source to based on root dir.
       set(_extension_target "${modules_target}.extension.${_module_name}.dso")
       add_mlir_python_extension(${_extension_target} "${_module_name}"
         INSTALL_COMPONENT ${modules_target}
         INSTALL_DIR "${ARG_INSTALL_PREFIX}/_mlir_libs"
         OUTPUT_DIRECTORY "${ARG_ROOT_PREFIX}/_mlir_libs"
-        PYTHON_BINDINGS_LIBRARY ${_bindings_library}
         LINK_LIBS PRIVATE
           ${sources_target}
           ${ARG_COMMON_CAPI_LINK_LIBS}
@@ -753,78 +745,69 @@ endfunction()
 function(add_mlir_python_extension libname extname)
   cmake_parse_arguments(ARG
   ""
-  "INSTALL_COMPONENT;INSTALL_DIR;OUTPUT_DIRECTORY;PYTHON_BINDINGS_LIBRARY"
+  "INSTALL_COMPONENT;INSTALL_DIR;OUTPUT_DIRECTORY"
   "SOURCES;LINK_LIBS"
   ${ARGN})
   if(ARG_UNPARSED_ARGUMENTS)
     message(FATAL_ERROR "Unhandled arguments to add_mlir_python_extension(${libname}, ... : ${ARG_UNPARSED_ARGUMENTS}")
   endif()
 
   # The extension itself must be compiled with RTTI and exceptions enabled.
-  # Also, some warning classes triggered by pybind11 are disabled.
+  # Also, some warning classes triggered by nanobind are disabled.
   set(eh_rtti_enable)
   if (MSVC)
     set(eh_rtti_enable /EHsc /GR)
   elseif(LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL)
     set(eh_rtti_enable -frtti -fexceptions)
   endif ()
 
-  # The actual extension library produces a shared-object or DLL and has
-  # sources that must be compiled in accordance with pybind11 needs (RTTI and
-  # exceptions).
-  if(NOT DEFINED ARG_PYTHON_BINDINGS_LIBRARY OR ARG_PYTHON_BINDINGS_LIBRARY STREQUAL "pybind11")
-    pybind11_add_module(${libname}
-      ${ARG_SOURCES}
-    )
-  elseif(ARG_PYTHON_BINDINGS_LIBRARY STREQUAL "nanobind")
-    nanobind_add_module(${libname}
-      NB_DOMAIN ${MLIR_BINDINGS_PYTHON_NB_DOMAIN}
-      FREE_THREADED
-      ${ARG_SOURCES}
-    )
+  nanobind_add_module(${libname}
+    NB_DOMAIN ${MLIR_BINDINGS_PYTHON_NB_DOMAIN}
+    FREE_THREADED
+    ${ARG_SOURCES}
+  )
 
-    if (NOT MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES
-        AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))
-      # Avoid some warnings from upstream nanobind.
-      # If a superproject set MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES, let
-      # the super project handle compile options as it wishes.
-      get_property(NB_LIBRARY_TARGET_NAME TARGET ${libname} PROPERTY LINK_LIBRARIES)
-      target_compile_options(${NB_LIBRARY_TARGET_NAME}
-        PRIVATE
-          -Wall -Wextra -Wpedantic
-          -Wno-c++98-compat-extra-semi
-          -Wno-cast-qual
-          -Wno-covered-switch-default
-          -Wno-deprecated-literal-operator
-          -Wno-nested-anon-types
-          -Wno-unused-parameter
-          -Wno-zero-length-array
-          ${eh_rtti_enable})
-
-      target_compile_options(${libname}
-        PRIVATE
-          -Wall -Wextra -Wpedantic
-          -Wno-c++98-compat-extra-semi
-          -Wno-cast-qual
-          -Wno-covered-switch-default
-          -Wno-deprecated-literal-operator
-          -Wno-nested-anon-types
-          -Wno-unused-parameter
-          -Wno-zero-length-array
-          ${eh_rtti_enable})
-    endif()
+  if (NOT MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES
+      AND (LLVM_COMPILER_IS_GCC_COMPATIBLE OR CLANG_CL))
+    # Avoid some warnings from upstream nanobind.
+    # If a superproject set MLIR_DISABLE_CONFIGURE_PYTHON_DEV_PACKAGES, let
+    # the super project handle compile options as it wishes.
+    get_property(NB_LIBRARY_TARGET_NAME TARGET ${libname} PROPERTY LINK_LIBRARIES)
+    target_compile_options(${NB_LIBRARY_TARGET_NAME}
+      PRIVATE
+        -Wall -Wextra -Wpedantic
+        -Wno-c++98-compat-extra-semi
+        -Wno-cast-qual
+        -Wno-covered-switch-default
+        -Wno-deprecated-literal-operator
+        -Wno-nested-anon-types
+        -Wno-unused-parameter
+        -Wno-zero-length-array
+        ${eh_rtti_enable})
+
+    target_compile_options(${libname}
+      PRIVATE
+        -Wall -Wextra -Wpedantic
+        -Wno-c++98-compat-extra-semi
+        -Wno-cast-qual
+        -Wno-covered-switch-default
+        -Wno-deprecated-literal-operator
+        -Wno-nested-anon-types
+        -Wno-unused-parameter
+        -Wno-zero-length-array
+        ${eh_rtti_enable})
+  endif()
 
-    if(APPLE)
-      # NanobindAdaptors.h uses PyClassMethod_New to build `pure_subclass`es but nanobind
-      # doesn't declare this API as undefined in its linker flags. So we need to declare it as such
-      # for downstream users that do not do something like `-undefined dynamic_lookup`.
-      # Same for the rest.
-      target_link_options(${libname} PUBLIC
-        "LINKER:-U,_PyClassMethod_New"
-        "LINKER:-U,_PyCode_Addr2Location"
-        "LINKER:-U,_PyFrame_GetLasti"
-      )
-    endif()
+  if(APPLE)
+    # NanobindAdaptors.h uses PyClassMethod_New to build `pure_subclass`es but nanobind
+    # doesn't declare this API as undefined in its linker flags. So we need to declare it as such
+    # for downstream users that do not do something like `-undefined dynamic_lookup`.
+    # Same for the rest.
+    target_link_options(${libname} PUBLIC
+      "LINKER:-U,_PyClassMethod_New"
+      "LINKER:-U,_PyCode_Addr2Location"
+      "LINKER:-U,_PyFrame_GetLasti"
+    )
   endif()
 
   target_compile_options(${libname} PRIVATE ${eh_rtti_enable})
@@ -862,11 +845,11 @@ function(add_mlir_python_extension libname extname)
   if(WIN32)
     # On Windows, pyconfig.h (and by extension python.h) hardcode the version of the
     # python library which will be used for linkage depending on the flavor of the build.
-    # pybind11 has a workaround which depends on the definition of Py_DEBUG (if Py_DEBUG
-    # is not passed in as a compile definition, pybind11 undefs _DEBUG when including
+    # nanobind has a workaround which depends on the definition of Py_DEBUG (if Py_DEBUG
+    # is not passed in as a compile definition, nanobind undefs _DEBUG when including
     # python.h, so that the release python library would be used).
-    # Since mlir uses pybind11, we can leverage their workaround by never directly
-    # pyconfig.h or python.h and instead relying on the pybind11 headers to include the
+    # Since mlir uses nanobind, we can leverage their workaround by never directly
+    # pyconfig.h or python.h and instead relying on the nanobind headers to include the
     # necessary python headers. This results in mlir always linking against the
     # release python library via the (undocumented) cmake property Python3_LIBRARY_RELEASE.
     target_link_libraries(${libname} PRIVATE ${Python3_LIBRARY_RELEASE})

mlir/cmake/modules/MLIRDetectPythonEnv.cmake

@@ -46,81 +46,20 @@ macro(mlir_configure_python_dev_packages)
     message(STATUS "Found python include dirs: ${Python3_INCLUDE_DIRS}")
     message(STATUS "Found python libraries: ${Python3_LIBRARIES}")
     message(STATUS "Found numpy v${Python3_NumPy_VERSION}: ${Python3_NumPy_INCLUDE_DIRS}")
-    mlir_detect_pybind11_install()
-    find_package(pybind11 2.10 CONFIG REQUIRED)
-    message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIR}")
-    message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
-                  "suffix = '${PYTHON_MODULE_SUFFIX}', "
-                  "extension = '${PYTHON_MODULE_EXTENSION}")
-
-    mlir_detect_nanobind_install()
-    find_package(nanobind 2.9 CONFIG REQUIRED)
-    message(STATUS "Found nanobind v${nanobind_VERSION}: ${nanobind_INCLUDE_DIR}")
-    message(STATUS "Python prefix = '${PYTHON_MODULE_PREFIX}', "
-                  "suffix = '${PYTHON_MODULE_SUFFIX}', "
-                  "extension = '${PYTHON_MODULE_EXTENSION}")
-  endif()
-endmacro()
-
-# Detects a pybind11 package installed in the current python environment
-# and sets variables to allow it to be found. This allows pybind11 to be
-# installed via pip, which typically yields a much more recent version than
-# the OS install, which will be available otherwise.
-function(mlir_detect_pybind11_install)
-  if(pybind11_DIR)
-    message(STATUS "Using explicit pybind11 cmake directory: ${pybind11_DIR} (-Dpybind11_DIR to change)")
-  else()
-    message(STATUS "Checking for pybind11 in python path...")
-    execute_process(
-      COMMAND "${Python3_EXECUTABLE}"
-      -c "import pybind11;print(pybind11.get_cmake_dir(), end='')"
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-      RESULT_VARIABLE STATUS
-      OUTPUT_VARIABLE PACKAGE_DIR
-      ERROR_QUIET)
-    if(NOT STATUS EQUAL "0")
-      message(STATUS "not found (install via 'pip install pybind11' or set pybind11_DIR)")
-      return()
-    endif()
-    message(STATUS "found (${PACKAGE_DIR})")
-    set(pybind11_DIR "${PACKAGE_DIR}" PARENT_SCOPE)
-  endif()
-endfunction()
-
-
-# Detects a nanobind package installed in the current python environment
-# and sets variables to allow it to be found. This allows nanobind to be
-# installed via pip, which typically yields a much more recent version than
-# the OS install, which will be available otherwise.
-function(mlir_detect_nanobind_install)
-  if(nanobind_DIR)
-    message(STATUS "Using explicit nanobind cmake directory: ${nanobind_DIR} (-Dnanobind_DIR to change)")
-  else()
-    message(STATUS "Checking for nanobind in python path...")
-    execute_process(
-      COMMAND "${Python3_EXECUTABLE}"
-      -c "import nanobind;print(nanobind.cmake_dir(), end='')"
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-      RESULT_VARIABLE STATUS
-      OUTPUT_VARIABLE PACKAGE_DIR
-      ERROR_QUIET)
-    if(NOT STATUS EQUAL "0")
-      message(STATUS "not found (install via 'pip install nanobind' or set nanobind_DIR)")
-      return()
+    message(STATUS "Python extension suffix for modules: '${Python3_SOABI}'")
+    if(nanobind_DIR)
+      message(STATUS "Using explicit nanobind cmake directory: ${nanobind_DIR} (-Dnanobind_DIR to change)")
+      find_package(nanobind 2.9 CONFIG REQUIRED)
+    else()
+      include(FetchContent)
+      FetchContent_Declare(
+        nanobind
+        GIT_REPOSITORY https://github.com/wjakob/nanobind.git
+        GIT_TAG        v2.9.0
+        GIT_SHALLOW    TRUE
+      )
+      FetchContent_MakeAvailable(nanobind)
     endif()
-    message(STATUS "found (${PACKAGE_DIR})")
-    set(nanobind_DIR "${PACKAGE_DIR}" PARENT_SCOPE)
-    execute_process(
-      COMMAND "${Python3_EXECUTABLE}"
-      -c "import nanobind;print(nanobind.include_dir(), end='')"
-      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-      RESULT_VARIABLE STATUS
-      OUTPUT_VARIABLE PACKAGE_DIR
-      ERROR_QUIET)
-    if(NOT STATUS EQUAL "0")
-      message(STATUS "not found (install via 'pip install nanobind' or set nanobind_DIR)")
-      return()
-    endif()
-    set(nanobind_INCLUDE_DIR "${PACKAGE_DIR}" PARENT_SCOPE)
+    message(STATUS "Found nanobind: ${NB_DIR}")
   endif()
-endfunction()
+endmacro()

mlir/docs/Dialects/Linalg/OpDSL.md

@@ -16,15 +16,15 @@ corresponding `linalg.generic` IR for the composition.
 ## Basic usage
 
 The tool is bundled with the MLIR Python bindings. To use from the CMake build
-tree, MLIR must be build with Python bindings enabled
+tree, MLIR must be built with Python bindings enabled
 (`-DMLIR_ENABLE_BINDINGS_PYTHON=ON`). Then add the `python` directory in the
 build tree to your `PYTHONPATH` environment variable (i.e. `export
 PYTHONPATH=$PWD/build/tools/mlir/python_packages/mlir_core`). Optionally, use an
 installed MLIR package, if available, to avoid building.
 
 ```shell
 # Dump the `core_named_ops.py` module as YAML.
-python -m mlir.dialects.linalg.opdsl.dump_oplib .ops.core_named_ops
+python -m mlir.dialects.linalg.opdsl.dump_oplib.ops.core_named_ops
 ```
 
 Alternatively, run the `$PWD/build/bin/update_core_linalg_named_ops.sh` script,

mlir/examples/standalone/pyproject.toml

@@ -23,9 +23,7 @@ Discussions = "https://discourse.llvm.org/"
 [build-system]
 requires = [
     "scikit-build-core>=0.10.7",
-    "typing_extensions>=4.12.2",
-    "nanobind>=2.9, <3.0",
-    "pybind11>=2.10.0, <=2.13.6",
+    "typing_extensions>=4.12.2"
 ]
 build-backend = "scikit_build_core.build"
 

mlir/examples/standalone/python/CMakeLists.txt

@@ -16,27 +16,10 @@ declare_mlir_dialect_python_bindings(
   ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/mlir_standalone"
   TD_FILE dialects/StandaloneOps.td
   SOURCES
-    dialects/standalone_pybind11.py
     dialects/standalone_nanobind.py
     _mlir_libs/_standaloneDialectsNanobind/py.typed
   DIALECT_NAME standalone)
 
-
-declare_mlir_python_extension(StandalonePythonSources.Pybind11Extension
-  MODULE_NAME _standaloneDialectsPybind11
-  ADD_TO_PARENT StandalonePythonSources
-  SOURCES
-    StandaloneExtensionPybind11.cpp
-  PRIVATE_LINK_LIBS
-    LLVMSupport
-  EMBED_CAPI_LINK_LIBS
-    MLIRCAPIIR
-    MLIRCAPIArith
-    MLIRCAPITransforms
-    StandaloneCAPI
-  PYTHON_BINDINGS_LIBRARY pybind11
-)
-
 declare_mlir_python_extension(StandalonePythonSources.NanobindExtension
   MODULE_NAME _standaloneDialectsNanobind
   ADD_TO_PARENT StandalonePythonSources
@@ -49,7 +32,6 @@ declare_mlir_python_extension(StandalonePythonSources.NanobindExtension
     MLIRCAPIArith
     MLIRCAPITransforms
     StandaloneCAPI
-  PYTHON_BINDINGS_LIBRARY nanobind
 )