Get the ASAN toolchain working again

README_cmake.md

@@ -11,11 +11,24 @@ The following sections cover each in detail.
 
 ## Table of Contents
 
+- [Halide and CMake](#halide-and-cmake)
+  - [Table of Contents](#table-of-contents)
 - [Getting started](#getting-started)
   - [Installing CMake](#installing-cmake)
+    - [Cross-platform](#cross-platform)
+    - [Windows](#windows)
+    - [macOS](#macos)
+    - [Ubuntu Linux](#ubuntu-linux)
   - [Installing dependencies](#installing-dependencies)
+    - [Windows](#windows-1)
+    - [macOS](#macos-1)
+    - [Ubuntu](#ubuntu)
 - [Building Halide with CMake](#building-halide-with-cmake)
   - [Basic build](#basic-build)
+    - [Windows](#windows-2)
+    - [macOS and Linux](#macos-and-linux)
+    - [CMake Presets](#cmake-presets)
+  - [Installing](#installing)
   - [Build options](#build-options)
     - [Find module options](#find-module-options)
 - [Using Halide from your CMake build](#using-halide-from-your-cmake-build)
@@ -33,6 +46,12 @@ The following sections cover each in detail.
       - [`add_halide_generator`](#add_halide_generator)
       - [`add_halide_python_extension_library`](#add_halide_python_extension_library)
       - [`add_halide_runtime`](#add_halide_runtime)
+  - [Cross compiling](#cross-compiling)
+    - [Use `add_halide_generator`](#use-add_halide_generator)
+    - [Use a super-build](#use-a-super-build)
+    - [Use `ExternalProject` directly](#use-externalproject-directly)
+    - [Use an emulator or run on device](#use-an-emulator-or-run-on-device)
+    - [Bypass CMake](#bypass-cmake)
 - [Contributing CMake code to Halide](#contributing-cmake-code-to-halide)
   - [General guidelines and best practices](#general-guidelines-and-best-practices)
     - [Prohibited commands list](#prohibited-commands-list)
@@ -750,8 +769,8 @@ Variables that control package loading:
 | Variable             | Description                                                                                                                                                                   |
 |----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `Halide_SHARED_LIBS` | override `BUILD_SHARED_LIBS` when loading the Halide package via `find_package`. Has no effect when using Halide via `add_subdirectory` as a Git or `FetchContent` submodule. |
-| `Halide_RUNTIME_NO_THREADS` | skip linking of Threads libraray to runtime. Should be set if your toolchain does not support it (e.g. baremetal). |
-| `Halide_RUNTIME_NO_DL_LIBS` | skip linking of DL libraray to runtime. Should be set if your toolchain does not support it (e.g. baremetal). |
+| `Halide_RUNTIME_NO_THREADS` | skip linking of Threads library to runtime. Should be set if your toolchain does not support it (e.g. baremetal). |
+| `Halide_RUNTIME_NO_DL_LIBS` | skip linking of DL library to runtime. Should be set if your toolchain does not support it (e.g. baremetal). |
 
 Variables set by the package:
 
@@ -767,6 +786,14 @@ Variables set by the package:
 | `Halide_ENABLE_EXCEPTIONS` | Whether Halide was compiled with exception support                 |
 | `Halide_ENABLE_RTTI`       | Whether Halide was compiled with RTTI                              |
 
+Variables that control package behavior:
+
+| Variable                   | Description |
+|----------------------------|-------------|
+| `Halide_PYTHON_LAUNCHER`   | Semicolon separated list containing a command to launch the Python interpreter. Can be used to set environment variables for Python generators. |
+| `Halide_NO_DEFAULT_FLAGS`  | Off by default. When enabled, suppresses recommended compiler flags that would be added by `add_halide_generator` |
+
+
 ### Imported targets
 
 Halide defines the following targets that are available to users:
@@ -866,10 +893,11 @@ author warning will be issued.
 
 To use an autoscheduler, set the `AUTOSCHEDULER` argument to a target
 named like `Namespace::Scheduler`, for example `Halide::Adams19`. This will set
-the `autoscheduler` GeneratorParam on the generator command line to `Scheduler` and add the target to
-the list of plugins. Additional plugins can be loaded by setting the `PLUGINS`
-argument. If the argument to `AUTOSCHEDULER` does not contain `::` or it does
-not name a target, it will be passed to the `-s` flag verbatim.
+the `autoscheduler` GeneratorParam on the generator command line to `Scheduler`
+and add the target to the list of plugins. Additional plugins can be loaded by
+setting the `PLUGINS` argument. If the argument to `AUTOSCHEDULER` does not
+contain `::` or it does not name a target, it will be passed to the `-s` flag
+verbatim.
 
 If `GRADIENT_DESCENT` is set, then the module will be built suitably for
 gradient descent calculation in TensorFlow or PyTorch. See
@@ -954,8 +982,9 @@ and [apps/hannk](https://github.com/halide/Halide/tree/main/apps/hannk) for a co
 If `PYSTUB` is specified, then a Python Extension will be built that
 wraps the Generator with CPython glue to allow use of the Generator
 Python 3.x. The result will be a a shared library of the form
-`<target>_pystub.<soabi>.so`, where <soabi> describes the specific Python version and platform (e.g., `cpython-310-darwin` for Python 3.10 on macOS.) See
-`README_python.md` for examples of use.
+`<target>_pystub.<soabi>.so`, where <soabi> describes the specific Python
+version and platform (e.g., `cpython-310-darwin` for Python 3.10 on macOS.).
+See `README_python.md` for examples of use.
 
 #### `add_halide_python_extension_library`
 

cmake/HalideGeneratorHelpers.cmake

@@ -244,11 +244,23 @@ function(add_halide_library TARGET)
         if (NOT TARGET Halide::Python)
             message(FATAL_ERROR "This version of Halide was built without support for Python bindings; rebuild using WITH_PYTHON_BINDINGS=ON to use this rule with Python Generators.")
         endif ()
+
         if (NOT TARGET Python3::Interpreter)
             message(FATAL_ERROR "You must call find_package(Python3) in your CMake code in order to use this rule with Python Generators.")
         endif ()
-        set(PYTHONPATH "$<TARGET_FILE_DIR:Halide::Python>/..")
-        set(GENERATOR_CMD ${CMAKE_COMMAND} -E env PYTHONPATH=${PYTHONPATH} ${Python3_EXECUTABLE} $<SHELL_PATH:${py_src}>)
+
+        if (CMAKE_VERSION VERSION_LESS 3.24)
+            set(arg_sep "")
+        else ()
+            set(arg_sep "--")
+        endif ()
+
+        set(
+            GENERATOR_CMD
+            ${CMAKE_COMMAND} -E env "PYTHONPATH=$<TARGET_FILE_DIR:Halide::Python>/.." ${arg_sep}
+            ${Halide_PYTHON_LAUNCHER}
+            "$<TARGET_FILE:Python3::Interpreter>" $<SHELL_PATH:${py_src}>
+        )
         set(GENERATOR_CMD_DEPS ${ARG_FROM} Halide::Python ${py_src})
     else()
         set(GENERATOR_CMD "${ARG_FROM}")
@@ -785,4 +797,3 @@ function(_Halide_gengen_ensure)
         _Halide_place_dll(_Halide_gengen)
     endif ()
 endfunction()
-

cmake/toolchain.linux-x64-asan.cmake

@@ -33,3 +33,19 @@ set(CMAKE_CROSSCOMPILING_EMULATOR /usr/bin/env)
 
 # Can't mix -fsanitize=address with -fsanitize=fuzzer
 set(WITH_TEST_FUZZ OFF)
+
+if (NOT DEFINED Halide_SHARED_ASAN_RUNTIME_LIBRARY)
+    execute_process(
+        COMMAND ${CMAKE_CXX_COMPILER} "-print-file-name=libclang_rt.asan.so"
+        OUTPUT_VARIABLE Halide_SHARED_ASAN_RUNTIME_LIBRARY
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+endif ()
+
+set(Halide_SHARED_ASAN_RUNTIME_LIBRARY "${Halide_SHARED_ASAN_RUNTIME_LIBRARY}"
+    CACHE FILEPATH "Library to preload when running Python tests.")
+
+set(
+    Halide_PYTHON_LAUNCHER
+    ${CMAKE_COMMAND} -E env ASAN_OPTIONS=detect_leaks=0 LD_PRELOAD=${Halide_SHARED_ASAN_RUNTIME_LIBRARY}
+)

python_bindings/CMakeLists.txt

@@ -62,43 +62,20 @@ endif ()
 # A helper for creating tests with correct PYTHONPATH and sanitizer preloading
 ##
 
-if (Halide_ASAN_ENABLED)
-    if (NOT DEFINED Halide_Python_ASAN_LIBRARY)
-        # TODO: this assumes clang-on-Linux, we could be smarter here and check
-        #   CMAKE_CXX_COMPILER_ID to behave differently on GNU, AppleClang, or
-        #   MSVC.
-        execute_process(
-            COMMAND ${CMAKE_CXX_COMPILER} "-print-file-name=libclang_rt.asan.so"
-            OUTPUT_VARIABLE Halide_Python_ASAN_LIBRARY
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-        )
-    endif ()
-
-    set(Halide_Python_ASAN_LIBRARY "${Halide_Python_ASAN_LIBRARY}"
-        CACHE FILEPATH "Library to preload when running Python tests.")
-endif ()
-
 function(add_python_test)
     cmake_parse_arguments(ARG "" "FILE;LABEL" "PYTHONPATH;ENVIRONMENT;TEST_ARGS" ${ARGN})
 
     list(PREPEND ARG_PYTHONPATH "$<TARGET_FILE_DIR:Halide::Python>/..")
     list(TRANSFORM ARG_PYTHONPATH PREPEND "PYTHONPATH=path_list_prepend:")
 
     list(PREPEND ARG_ENVIRONMENT "HL_TARGET=${Halide_TARGET}")
-    if (Halide_Python_ASAN_LIBRARY)
-        if (APPLE)
-            list(PREPEND ARG_ENVIRONMENT "DYLD_INSERT_LIBRARIES=${Halide_Python_ASAN_LIBRARY}")
-        else ()
-            list(PREPEND ARG_ENVIRONMENT "LD_PRELOAD=${Halide_Python_ASAN_LIBRARY}")
-        endif ()
-    endif ()
 
     cmake_path(GET ARG_FILE STEM test_name)
     set(test_name "${ARG_LABEL}_${test_name}")
 
     add_test(
         NAME "${test_name}"
-        COMMAND Python3::Interpreter "$<SHELL_PATH:${CMAKE_CURRENT_SOURCE_DIR}/${ARG_FILE}>" ${ARG_TEST_ARGS}
+        COMMAND ${Halide_PYTHON_LAUNCHER} "$<TARGET_FILE:Python3::Interpreter>" "$<SHELL_PATH:${CMAKE_CURRENT_SOURCE_DIR}/${ARG_FILE}>" ${ARG_TEST_ARGS}
     )
     set_tests_properties(
         "${test_name}"
@@ -128,4 +105,3 @@ endif ()
 if (WITH_TUTORIALS)
     add_subdirectory(tutorial)
 endif ()
-

python_bindings/src/halide/CMakeLists.txt

@@ -51,6 +51,13 @@ set_target_properties(
     LIBRARY_OUTPUT_DIRECTORY "$<CONFIG>/halide"
     EXPORT_NAME Python
 )
+if (Halide_ASAN_ENABLED)
+    set_target_properties(
+        Halide_Python
+        PROPERTIES
+        CMAKE_SHARED_LINKER_FLAGS -shared-libasan
+    )
+endif ()
 target_link_libraries(Halide_Python PRIVATE Halide::Halide)
 
 # TODO: There's precious little information about why Python only sometimes prevents DLLs from loading from the PATH

src/autoschedulers/anderson2021/CMakeLists.txt

@@ -77,67 +77,56 @@ endif ()
 if (WITH_TESTS)
     ##
 
+    function(_add_test TARGET)
+        add_test(NAME ${TARGET} COMMAND ${TARGET})
+        set_tests_properties(${TARGET}
+                             PROPERTIES
+                             # This is a workaround for issues with the nvidia driver under ASAN
+                             # https://forums.developer.nvidia.com/t/cuda-runtime-library-and-addresssanitizer-incompatibilty/63145
+                             ENVIRONMENT ASAN_OPTIONS=protect_shadow_gap=0
+                             LABELS Anderson2021)
+    endfunction()
+
     add_executable(anderson2021_test_function_dag test_function_dag.cpp FunctionDAG.cpp)
     target_include_directories(anderson2021_test_function_dag PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
     target_link_libraries(anderson2021_test_function_dag PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
-    add_test(NAME anderson2021_test_function_dag COMMAND anderson2021_test_function_dag)
-    set_tests_properties(anderson2021_test_function_dag
-                         PROPERTIES
-                         LABELS Anderson2021)
+    _add_test(anderson2021_test_function_dag)
 
     add_executable(anderson2021_test_bounds test/bounds.cpp FunctionDAG.cpp LoopNest.cpp GPULoopInfo.cpp Tiling.cpp)
     target_include_directories(anderson2021_test_bounds PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
     target_link_libraries(anderson2021_test_bounds PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
-    add_test(NAME anderson2021_test_bounds COMMAND anderson2021_test_bounds)
-    set_tests_properties(anderson2021_test_bounds
-                         PROPERTIES
-                         LABELS Anderson2021)
+    _add_test(anderson2021_test_bounds)
 
     add_executable(anderson2021_test_parser test/parser.cpp)
     target_include_directories(anderson2021_test_parser PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
     target_link_libraries(anderson2021_test_parser PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
-    add_test(NAME anderson2021_test_parser COMMAND anderson2021_test_parser)
-    set_tests_properties(anderson2021_test_parser
-                         PROPERTIES
-                         LABELS Anderson2021)
+    _add_test(anderson2021_test_parser)
 
     add_executable(anderson2021_test_state test/state.cpp FunctionDAG.cpp LoopNest.cpp GPULoopInfo.cpp State.cpp Tiling.cpp)
     target_include_directories(anderson2021_test_state PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
     target_link_libraries(anderson2021_test_state PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
-    add_test(NAME anderson2021_test_state COMMAND anderson2021_test_state)
-    set_tests_properties(anderson2021_test_state
-                         PROPERTIES
-                         LABELS Anderson2021)
+    _add_test(anderson2021_test_state)
 
     add_executable(anderson2021_test_storage_strides test/storage_strides.cpp FunctionDAG.cpp LoopNest.cpp GPULoopInfo.cpp State.cpp Tiling.cpp)
     target_include_directories(anderson2021_test_storage_strides PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
     target_link_libraries(anderson2021_test_storage_strides PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
-    add_test(NAME anderson2021_test_storage_strides COMMAND anderson2021_test_storage_strides)
-    set_tests_properties(anderson2021_test_storage_strides
-                         PROPERTIES
-                         LABELS Anderson2021)
+    _add_test(anderson2021_test_storage_strides)
 
     add_executable(anderson2021_test_thread_info test/thread_info.cpp LoopNest.cpp
         FunctionDAG.cpp GPULoopInfo.cpp Tiling.cpp)
     target_include_directories(anderson2021_test_thread_info PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
     target_link_libraries(anderson2021_test_thread_info PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
-    add_test(NAME anderson2021_test_thread_info COMMAND anderson2021_test_thread_info)
-    set_tests_properties(anderson2021_test_thread_info
-                         PROPERTIES
-                         LABELS Anderson2021)
+    _add_test(anderson2021_test_thread_info)
 
     add_executable(anderson2021_test_tiling test/tiling.cpp Tiling.cpp)
     target_include_directories(anderson2021_test_tiling PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021")
     target_link_libraries(anderson2021_test_tiling PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin)
 
-    add_test(NAME anderson2021_test_tiling COMMAND anderson2021_test_tiling)
-    set_tests_properties(anderson2021_test_tiling
-                         PROPERTIES
-                         LABELS Anderson2021)
+    _add_test(anderson2021_test_tiling)
 endif()

test/autoschedulers/li2018/CMakeLists.txt

@@ -33,8 +33,10 @@ if (WITH_PYTHON_BINDINGS)
     else()
         find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module)
 
-        add_test(NAME li2018_gradient_autoscheduler_test_py
-                 COMMAND Python3::Interpreter "${CMAKE_CURRENT_SOURCE_DIR}/test.py" $<TARGET_FILE:Halide::Li2018>)
+        add_test(
+            NAME li2018_gradient_autoscheduler_test_py
+            COMMAND ${Halide_PYTHON_LAUNCHER} "$<TARGET_FILE:Python3::Interpreter>" "${CMAKE_CURRENT_SOURCE_DIR}/test.py" $<TARGET_FILE:Halide::Li2018>
+        )
 
         set(PYTHONPATH "$<TARGET_FILE_DIR:Halide::Python>/..")
         list(TRANSFORM PYTHONPATH PREPEND "PYTHONPATH=path_list_prepend:")