Get the ASAN toolchain working again

README_cmake.md

@@ -11,11 +11,24 @@ The following sections cover each in detail.
 
 ## Table of Contents
 
+- [Halide and CMake](#halide-and-cmake)
+ - [Table of Contents](#table-of-contents)
 - [Getting started](#getting-started)
  - [Installing CMake](#installing-cmake)
+ - [Cross-platform](#cross-platform)
+ - [Windows](#windows)
+ - [macOS](#macos)
+ - [Ubuntu Linux](#ubuntu-linux)
  - [Installing dependencies](#installing-dependencies)
+ - [Windows](#windows-1)
+ - [macOS](#macos-1)
+ - [Ubuntu](#ubuntu)
 - [Building Halide with CMake](#building-halide-with-cmake)
  - [Basic build](#basic-build)
+ - [Windows](#windows-2)
+ - [macOS and Linux](#macos-and-linux)
+ - [CMake Presets](#cmake-presets)
+ - [Installing](#installing)
  - [Build options](#build-options)
  - [Find module options](#find-module-options)
 - [Using Halide from your CMake build](#using-halide-from-your-cmake-build)
@@ -33,6 +46,12 @@ The following sections cover each in detail.
  - [`add_halide_generator`](#add_halide_generator)
  - [`add_halide_python_extension_library`](#add_halide_python_extension_library)
  - [`add_halide_runtime`](#add_halide_runtime)
+ - [Cross compiling](#cross-compiling)
+ - [Use `add_halide_generator`](#use-add_halide_generator)
+ - [Use a super-build](#use-a-super-build)
+ - [Use `ExternalProject` directly](#use-externalproject-directly)
+ - [Use an emulator or run on device](#use-an-emulator-or-run-on-device)
+ - [Bypass CMake](#bypass-cmake)
 - [Contributing CMake code to Halide](#contributing-cmake-code-to-halide)
  - [General guidelines and best practices](#general-guidelines-and-best-practices)
  - [Prohibited commands list](#prohibited-commands-list)
@@ -750,8 +769,8 @@ Variables that control package loading:
 | Variable | Description |
 |----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `Halide_SHARED_LIBS` | override `BUILD_SHARED_LIBS` when loading the Halide package via `find_package`. Has no effect when using Halide via `add_subdirectory` as a Git or `FetchContent` submodule. |
-| `Halide_RUNTIME_NO_THREADS` | skip linking of Threads libraray to runtime. Should be set if your toolchain does not support it (e.g. baremetal). |
-| `Halide_RUNTIME_NO_DL_LIBS` | skip linking of DL libraray to runtime. Should be set if your toolchain does not support it (e.g. baremetal). |
+| `Halide_RUNTIME_NO_THREADS` | skip linking of Threads library to runtime. Should be set if your toolchain does not support it (e.g. baremetal). |
+| `Halide_RUNTIME_NO_DL_LIBS` | skip linking of DL library to runtime. Should be set if your toolchain does not support it (e.g. baremetal). |
 
 Variables set by the package:
 
@@ -767,6 +786,14 @@ Variables set by the package:
 | `Halide_ENABLE_EXCEPTIONS` | Whether Halide was compiled with exception support |
 | `Halide_ENABLE_RTTI` | Whether Halide was compiled with RTTI |
 
+Variables that control package behavior:
+
+| Variable | Description |
+|----------------------------|-------------|
+| `Halide_PYTHON_LAUNCHER` | Semicolon separated list containing a command to launch the Python interpreter. Can be used to set environment variables for Python generators. |
+| `Halide_NO_DEFAULT_FLAGS` | Off by default. When enabled, suppresses recommended compiler flags that would be added by `add_halide_generator` |
+
+
 ### Imported targets
 
 Halide defines the following targets that are available to users:
@@ -866,10 +893,11 @@ author warning will be issued.
 
 To use an autoscheduler, set the `AUTOSCHEDULER` argument to a target
 named like `Namespace::Scheduler`, for example `Halide::Adams19`. This will set
-the `autoscheduler` GeneratorParam on the generator command line to `Scheduler` and add the target to
-the list of plugins. Additional plugins can be loaded by setting the `PLUGINS`
-argument. If the argument to `AUTOSCHEDULER` does not contain `::` or it does
-not name a target, it will be passed to the `-s` flag verbatim.
+the `autoscheduler` GeneratorParam on the generator command line to `Scheduler`
+and add the target to the list of plugins. Additional plugins can be loaded by
+setting the `PLUGINS` argument. If the argument to `AUTOSCHEDULER` does not
+contain `::` or it does not name a target, it will be passed to the `-s` flag
+verbatim.
 
 If `GRADIENT_DESCENT` is set, then the module will be built suitably for
 gradient descent calculation in TensorFlow or PyTorch. See
@@ -954,8 +982,9 @@ and [apps/hannk](https://github.com/halide/Halide/tree/main/apps/hannk) for a co
 If `PYSTUB` is specified, then a Python Extension will be built that
 wraps the Generator with CPython glue to allow use of the Generator
 Python 3.x. The result will be a a shared library of the form
-`<target>_pystub.<soabi>.so`, where <soabi> describes the specific Python version and platform (e.g., `cpython-310-darwin` for Python 3.10 on macOS.) See
-`README_python.md` for examples of use.
+`<target>_pystub.<soabi>.so`, where <soabi> describes the specific Python
+version and platform (e.g., `cpython-310-darwin` for Python 3.10 on macOS.).
+See `README_python.md` for examples of use.
 
 #### `add_halide_python_extension_library`
 

cmake/HalideGeneratorHelpers.cmake

@@ -244,11 +244,23 @@ function(add_halide_library TARGET)
  if (NOT TARGET Halide::Python)
  message(FATAL_ERROR "This version of Halide was built without support for Python bindings; rebuild using WITH_PYTHON_BINDINGS=ON to use this rule with Python Generators.")
  endif ()
+
  if (NOT TARGET Python3::Interpreter)
  message(FATAL_ERROR "You must call find_package(Python3) in your CMake code in order to use this rule with Python Generators.")
  endif ()
- set(PYTHONPATH "$<TARGET_FILE_DIR:Halide::Python>/..")
- set(GENERATOR_CMD ${CMAKE_COMMAND} -E env PYTHONPATH=${PYTHONPATH} ${Python3_EXECUTABLE} $<SHELL_PATH:${py_src}>)
+
+ if (CMAKE_VERSION VERSION_LESS 3.24)
+ set(arg_sep "")
+ else ()
+ set(arg_sep "--")
+ endif ()
+
+ set(
+ GENERATOR_CMD
+ ${CMAKE_COMMAND} -E env "PYTHONPATH=$<TARGET_FILE_DIR:Halide::Python>/.." ${arg_sep}
+ ${Halide_PYTHON_LAUNCHER}
+ "$<TARGET_FILE:Python3::Interpreter>" $<SHELL_PATH:${py_src}>
+ )
  set(GENERATOR_CMD_DEPS ${ARG_FROM} Halide::Python ${py_src})
  else()
  set(GENERATOR_CMD "${ARG_FROM}")
@@ -785,4 +797,3 @@ function(_Halide_gengen_ensure)
  _Halide_place_dll(_Halide_gengen)
  endif ()
 endfunction()
-

cmake/toolchain.linux-x64-asan.cmake

@@ -33,3 +33,19 @@ set(CMAKE_CROSSCOMPILING_EMULATOR /usr/bin/env)
 
 # Can't mix -fsanitize=address with -fsanitize=fuzzer
 set(WITH_TEST_FUZZ OFF)
+
+if (NOT DEFINED Halide_SHARED_ASAN_RUNTIME_LIBRARY)
+ execute_process(
+ COMMAND ${CMAKE_CXX_COMPILER} "-print-file-name=libclang_rt.asan.so"
+ OUTPUT_VARIABLE Halide_SHARED_ASAN_RUNTIME_LIBRARY
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ )
+endif ()
+
+set(Halide_SHARED_ASAN_RUNTIME_LIBRARY "${Halide_SHARED_ASAN_RUNTIME_LIBRARY}"
+ CACHE FILEPATH "Library to preload when running Python tests.")
+
+set(
+ Halide_PYTHON_LAUNCHER
+ ${CMAKE_COMMAND} -E env ASAN_OPTIONS=detect_leaks=0 LD_PRELOAD=${Halide_SHARED_ASAN_RUNTIME_LIBRARY}
+)

python_bindings/CMakeLists.txt

@@ -62,43 +62,20 @@ endif ()
 # A helper for creating tests with correct PYTHONPATH and sanitizer preloading
 ##
 
-if (Halide_ASAN_ENABLED)
- if (NOT DEFINED Halide_Python_ASAN_LIBRARY)
- # TODO: this assumes clang-on-Linux, we could be smarter here and check
- # CMAKE_CXX_COMPILER_ID to behave differently on GNU, AppleClang, or
- # MSVC.
- execute_process(
- COMMAND ${CMAKE_CXX_COMPILER} "-print-file-name=libclang_rt.asan.so"
- OUTPUT_VARIABLE Halide_Python_ASAN_LIBRARY
- OUTPUT_STRIP_TRAILING_WHITESPACE
- )
- endif ()
-
- set(Halide_Python_ASAN_LIBRARY "${Halide_Python_ASAN_LIBRARY}"
- CACHE FILEPATH "Library to preload when running Python tests.")
-endif ()
-
 function(add_python_test)
  cmake_parse_arguments(ARG "" "FILE;LABEL" "PYTHONPATH;ENVIRONMENT;TEST_ARGS" ${ARGN})
 
  list(PREPEND ARG_PYTHONPATH "$<TARGET_FILE_DIR:Halide::Python>/..")
  list(TRANSFORM ARG_PYTHONPATH PREPEND "PYTHONPATH=path_list_prepend:")
 
  list(PREPEND ARG_ENVIRONMENT "HL_TARGET=${Halide_TARGET}")
- if (Halide_Python_ASAN_LIBRARY)
- if (APPLE)
- list(PREPEND ARG_ENVIRONMENT "DYLD_INSERT_LIBRARIES=${Halide_Python_ASAN_LIBRARY}")
- else ()
- list(PREPEND ARG_ENVIRONMENT "LD_PRELOAD=${Halide_Python_ASAN_LIBRARY}")
- endif ()
- endif ()
 
  cmake_path(GET ARG_FILE STEM test_name)
  set(test_name "${ARG_LABEL}_${test_name}")
 
  add_test(
  NAME "${test_name}"
- COMMAND Python3::Interpreter "$<SHELL_PATH:${CMAKE_CURRENT_SOURCE_DIR}/${ARG_FILE}>" ${ARG_TEST_ARGS}
+ COMMAND ${Halide_PYTHON_LAUNCHER} "$<TARGET_FILE:Python3::Interpreter>" "$<SHELL_PATH:${CMAKE_CURRENT_SOURCE_DIR}/${ARG_FILE}>" ${ARG_TEST_ARGS}
  )
  set_tests_properties(
  "${test_name}"
@@ -128,4 +105,3 @@ endif ()
 if (WITH_TUTORIALS)
  add_subdirectory(tutorial)
 endif ()
-

python_bindings/src/halide/CMakeLists.txt

@@ -51,6 +51,13 @@ set_target_properties(
  LIBRARY_OUTPUT_DIRECTORY "$<CONFIG>/halide"
  EXPORT_NAME Python
 )
+if (Halide_ASAN_ENABLED)
+ set_target_properties(
+ Halide_Python
+ PROPERTIES
+ CMAKE_SHARED_LINKER_FLAGS -shared-libasan
+ )
+endif ()
 target_link_libraries(Halide_Python PRIVATE Halide::Halide)
 
 # TODO: There's precious little information about why Python only sometimes prevents DLLs from loading from the PATH

src/autoschedulers/anderson2021/CMakeLists.txt

@@ -77,67 +77,56 @@ endif ()
 if (WITH_TESTS)
  ##
 
+ function(_add_test TARGET)
+ add_test(NAME ${TARGET} COMMAND ${TARGET})
+ set_tests_properties(${TARGET}
+ PROPERTIES
+ # This is a workaround for issues with the nvidia driver under ASAN
+ # https://forums.developer.nvidia.com/t/cuda-runtime-library-and-addresssanitizer-incompatibilty/63145
+ ENVIRONMENT ASAN_OPTIONS=protect_shadow_gap=0
+ LABELS Anderson2021)
+ endfunction()
+
  add_executable(anderson2021_test_function_dag test_function_dag.cpp FunctionDAG.cpp)
  target_include_directories(anderson2021_test_function_dag PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
  target_link_libraries(anderson2021_test_function_dag PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
- add_test(NAME anderson2021_test_function_dag COMMAND anderson2021_test_function_dag)
- set_tests_properties(anderson2021_test_function_dag
- PROPERTIES
- LABELS Anderson2021)
+ _add_test(anderson2021_test_function_dag)
 
  add_executable(anderson2021_test_bounds test/bounds.cpp FunctionDAG.cpp LoopNest.cpp GPULoopInfo.cpp Tiling.cpp)
  target_include_directories(anderson2021_test_bounds PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
  target_link_libraries(anderson2021_test_bounds PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
- add_test(NAME anderson2021_test_bounds COMMAND anderson2021_test_bounds)
- set_tests_properties(anderson2021_test_bounds
- PROPERTIES
- LABELS Anderson2021)
+ _add_test(anderson2021_test_bounds)
 
  add_executable(anderson2021_test_parser test/parser.cpp)
  target_include_directories(anderson2021_test_parser PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
  target_link_libraries(anderson2021_test_parser PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
- add_test(NAME anderson2021_test_parser COMMAND anderson2021_test_parser)
- set_tests_properties(anderson2021_test_parser
- PROPERTIES
- LABELS Anderson2021)
+ _add_test(anderson2021_test_parser)
 
  add_executable(anderson2021_test_state test/state.cpp FunctionDAG.cpp LoopNest.cpp GPULoopInfo.cpp State.cpp Tiling.cpp)
  target_include_directories(anderson2021_test_state PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
  target_link_libraries(anderson2021_test_state PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
- add_test(NAME anderson2021_test_state COMMAND anderson2021_test_state)
- set_tests_properties(anderson2021_test_state
- PROPERTIES
- LABELS Anderson2021)
+ _add_test(anderson2021_test_state)
 
  add_executable(anderson2021_test_storage_strides test/storage_strides.cpp FunctionDAG.cpp LoopNest.cpp GPULoopInfo.cpp State.cpp Tiling.cpp)
  target_include_directories(anderson2021_test_storage_strides PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
  target_link_libraries(anderson2021_test_storage_strides PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
- add_test(NAME anderson2021_test_storage_strides COMMAND anderson2021_test_storage_strides)
- set_tests_properties(anderson2021_test_storage_strides
- PROPERTIES
- LABELS Anderson2021)
+ _add_test(anderson2021_test_storage_strides)
 
  add_executable(anderson2021_test_thread_info test/thread_info.cpp LoopNest.cpp
  FunctionDAG.cpp GPULoopInfo.cpp Tiling.cpp)
  target_include_directories(anderson2021_test_thread_info PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
  target_link_libraries(anderson2021_test_thread_info PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
- add_test(NAME anderson2021_test_thread_info COMMAND anderson2021_test_thread_info)
- set_tests_properties(anderson2021_test_thread_info
- PROPERTIES
- LABELS Anderson2021)
+ _add_test(anderson2021_test_thread_info)
 
  add_executable(anderson2021_test_tiling test/tiling.cpp Tiling.cpp)
  target_include_directories(anderson2021_test_tiling PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
  target_link_libraries(anderson2021_test_tiling PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
- add_test(NAME anderson2021_test_tiling COMMAND anderson2021_test_tiling)
- set_tests_properties(anderson2021_test_tiling
- PROPERTIES
- LABELS Anderson2021)
+ _add_test(anderson2021_test_tiling)
 endif()

test/autoschedulers/li2018/CMakeLists.txt

@@ -33,8 +33,10 @@ if (WITH_PYTHON_BINDINGS)
  else()
  find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module)
 
- add_test(NAME li2018_gradient_autoscheduler_test_py
- COMMAND Python3::Interpreter "${CMAKE_CURRENT_SOURCE_DIR}/test.py" $<TARGET_FILE:Halide::Li2018>)
+ add_test(
+ NAME li2018_gradient_autoscheduler_test_py
+ COMMAND ${Halide_PYTHON_LAUNCHER} "$<TARGET_FILE:Python3::Interpreter>" "${CMAKE_CURRENT_SOURCE_DIR}/test.py" $<TARGET_FILE:Halide::Li2018>
+ )
 
  set(PYTHONPATH "$<TARGET_FILE_DIR:Halide::Python>/..")
  list(TRANSFORM PYTHONPATH PREPEND "PYTHONPATH=path_list_prepend:")