Python source reorg (halide#6867)

* Move python binding sources to src/halide/halide_

* Rename native module to halide_

* Fix tests

* Avoid copying Python sources

* Fix installation rules

* Make diff smaller

* trigger buildbots

* Add issue todo

Co-authored-by: Steven Johnson <srj@google.com>

packaging/CMakeLists.txt

@@ -52,8 +52,18 @@ endforeach ()
 if (WITH_PYTHON_BINDINGS)
  set(Halide_INSTALL_PYTHONDIR "${CMAKE_INSTALL_LIBDIR}/python3/site-packages"
  CACHE STRING "Path to Halide Python bindings folder")
+
+ install(DIRECTORY "${Halide_SOURCE_DIR}/python_bindings/src/halide"
+ DESTINATION "${Halide_INSTALL_PYTHONDIR}"
+ COMPONENT Halide_Python
+ FILES_MATCHING
+ PATTERN "*.py"
+ PATTERN "*/halide_" EXCLUDE
+ PATTERN "*/__pycache__" EXCLUDE)
+
  install(TARGETS Halide_Python
- LIBRARY DESTINATION ${Halide_INSTALL_PYTHONDIR} COMPONENT Halide_Python
+ LIBRARY DESTINATION "${Halide_INSTALL_PYTHONDIR}/halide"
+ COMPONENT Halide_Python
  NAMELINK_COMPONENT Halide_Python)
 endif ()
 
@@ -101,6 +111,13 @@ if (TARGET Halide_Adams2019)
  COMPONENT Halide_Development)
 endif ()
 
+if (TARGET Halide_Python AND NOT CMAKE_INSTALL_RPATH)
+ file(RELATIVE_PATH lib_dir
+ "${CMAKE_CURRENT_BINARY_DIR}/${Halide_INSTALL_PYTHONDIR}/halide"
+ "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
+ set_target_properties(Halide_Python PROPERTIES INSTALL_RPATH "${rbase}/${lib_dir}")
+endif ()
+
 ##
 # READMEs and other top-level documentation
 ##
@@ -268,6 +285,11 @@ cpack_add_component(Halide_Development
  DESCRIPTION "Static Halide libraries and CMake development files"
  DEPENDS Halide_Runtime)
 
+cpack_add_component(Halide_Python
+ DISPLAY_NAME "Python bindings"
+ DESCRIPTION "Python package providing bindings to Halide"
+ DEPENDS Halide_Runtime)
+
 cpack_add_component(Halide_Documentation
  DISPLAY_NAME "Halide documentation"
  DESCRIPTION "Documentation for Halide")

python_bindings/CMakeLists.txt

@@ -18,7 +18,7 @@ endif ()
 # Add our sources to this sub-tree.
 ##
 
-add_subdirectory(src)
+add_subdirectory(src/halide)
 add_subdirectory(stub)
 
 option(WITH_TEST_PYTHON "Build Python tests" ON)

python_bindings/src/CMakeLists.txt → python_bindings/src/halide/CMakeLists.txt

@@ -29,12 +29,13 @@ set(SOURCES
  PyVar.cpp
  PyVarOrRVar.cpp
  )
+list(TRANSFORM SOURCES PREPEND "halide_/")
 
 pybind11_add_module(Halide_Python MODULE ${SOURCES})
 add_library(Halide::Python ALIAS Halide_Python)
 set_target_properties(Halide_Python
  PROPERTIES
- LIBRARY_OUTPUT_NAME halide
+ LIBRARY_OUTPUT_NAME halide_
  EXPORT_NAME Python)
 target_link_libraries(Halide_Python PRIVATE Halide::Halide)
 

python_bindings/src/halide/__init__.py

@@ -0,0 +1,24 @@
+# TODO(#6870): The following three lines are a stop-gap. This file should just
+# contain the last two imports, at least until the pure-Python part of the
+# library grows.
+#
+# There are three main reasons why this exists:
+#
+# 1. relative imports cannot be overloaded with sys.path
+# 2. for a variety of reasons, copying the python sources next to the
+# halide_.*.so module is difficult to make work in 100% of cases in CMake
+# 3. even if we could copy them reliably, copying these files into the build
+# folder seems inelegant
+#
+# Fortunately, there are apparently other hooks besides sys.path that we could
+# use to redirect a failing relative import.
+#
+# https://docs.python.org/3/reference/import.html#finders-and-loaders
+# https://github.com/halide/Halide/issues/6870
+
+import sys
+from pathlib import Path
+sys.path.append(str(Path(__file__).parent.resolve(strict=True)))
+
+from halide_ import *
+from halide_ import _, _1, _2, _3, _4, _5, _6, _7, _8, _9

python_bindings/src/PyHalide.cpp → python_bindings/src/halide/halide_/PyHalide.cpp

@@ -34,7 +34,7 @@ static_assert(PY_VERSION_HEX >= 0x03000000,
  "We appear to be compiling against Python 2.x rather than 3.x, which is not supported.");
 
 #ifndef HALIDE_PYBIND_MODULE_NAME
-#define HALIDE_PYBIND_MODULE_NAME halide
+#define HALIDE_PYBIND_MODULE_NAME halide_
 #endif
 
 PYBIND11_MODULE(HALIDE_PYBIND_MODULE_NAME, m) {

python_bindings/test/apps/CMakeLists.txt

@@ -5,8 +5,14 @@ set(SCRIPTS
  interpolate.py
  local_laplacian.py)
 
+make_shell_path(
+ PYTHONPATH
+ "$<TARGET_FILE_DIR:Halide::Python>"
+ "${Halide_SOURCE_DIR}/python_bindings/src"
+)
+
 set(TEST_ENV
- "PYTHONPATH=$<SHELL_PATH:$<TARGET_FILE_DIR:Halide::Python>>"
+ "PYTHONPATH=${PYTHONPATH}"
  "HL_TARGET=${Halide_TARGET}"
  "TEST_TMPDIR=$<SHELL_PATH:${CMAKE_CURRENT_BINARY_DIR}>"
  "TEST_IMAGES_DIR=$<SHELL_PATH:${CMAKE_CURRENT_SOURCE_DIR}/../../../apps/images>"

python_bindings/test/correctness/CMakeLists.txt

@@ -27,11 +27,13 @@ set(TESTS
  )
 
 # Use generator expressions to get the true output paths of these files.
-make_shell_path(PYTHONPATH
- "$<TARGET_FILE_DIR:py_aot_bit>"
- "$<TARGET_FILE_DIR:py_stub_bit>"
- "$<TARGET_FILE_DIR:Halide::Python>"
- )
+make_shell_path(
+ PYTHONPATH
+ "$<TARGET_FILE_DIR:py_aot_bit>"
+ "$<TARGET_FILE_DIR:py_stub_bit>"
+ "$<TARGET_FILE_DIR:Halide::Python>"
+ "${Halide_SOURCE_DIR}/python_bindings/src"
+)
 
 foreach (TEST IN LISTS TESTS)
  get_filename_component(TEST_NAME ${TEST} NAME_WE)

python_bindings/tutorial/CMakeLists.txt

@@ -16,7 +16,12 @@ set(TESTS
  lesson_14_types.py
  )
 
-make_shell_path(PYTHONPATH "$<TARGET_FILE_DIR:lesson_10_halide>" "$<TARGET_FILE_DIR:Halide::Python>")
+make_shell_path(
+ PYTHONPATH
+ "$<TARGET_FILE_DIR:lesson_10_halide>"
+ "$<TARGET_FILE_DIR:Halide::Python>"
+ "${Halide_SOURCE_DIR}/python_bindings/src"
+)
 
 foreach (TEST IN LISTS TESTS)
  get_filename_component(TEST_NAME ${TEST} NAME_WE)

src/autoschedulers/li2018/CMakeLists.txt

@@ -44,7 +44,11 @@ if (WITH_PYTHON_BINDINGS)
  add_test(NAME gradient_autoscheduler_test_py
  COMMAND Python3::Interpreter "${CMAKE_CURRENT_SOURCE_DIR}/test.py")
 
- set(PYTHONPATH "$<SHELL_PATH:$<TARGET_FILE_DIR:Halide::Python>>")
+ make_shell_path(
+ PYTHONPATH
+ "$<TARGET_FILE_DIR:Halide::Python>"
+ "${Halide_SOURCE_DIR}/python_bindings/src"
+ )
 
  if (WIN32)
  set(SEP "\\$<SEMICOLON>")