From 373f0d1b03d72656886f3383b9525900c02c4912 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 1 Jun 2023 14:56:28 -0700 Subject: [PATCH 01/10] Get the ASAN toolchain working again Various fixes to enable ASAN to finally work (linux x64 only). Note that this found several ASAN failures in the Anderson2021 autoscheduler tests, which are *not* fixed yet; I'll fix thus in a subsequent PR. --- cmake/HalideGeneratorHelpers.cmake | 10 ++++++++- cmake/toolchain.linux-x64-asan.cmake | 13 ++++++++++++ python_bindings/CMakeLists.txt | 25 +++-------------------- python_bindings/src/halide/CMakeLists.txt | 7 +++++++ src/CMakeLists.txt | 9 ++++++++ test/autoschedulers/li2018/CMakeLists.txt | 1 + 6 files changed, 42 insertions(+), 23 deletions(-) diff --git a/cmake/HalideGeneratorHelpers.cmake b/cmake/HalideGeneratorHelpers.cmake index d2fed8bae3ae..e9e52cde3ff6 100644 --- a/cmake/HalideGeneratorHelpers.cmake +++ b/cmake/HalideGeneratorHelpers.cmake @@ -248,7 +248,12 @@ function(add_halide_library TARGET) message(FATAL_ERROR "You must call find_package(Python3) in your CMake code in order to use this rule with Python Generators.") endif () set(PYTHONPATH "$/..") - set(GENERATOR_CMD ${CMAKE_COMMAND} -E env PYTHONPATH=${PYTHONPATH} ${Python3_EXECUTABLE} $) + if (Halide_SHARED_ASAN_RUNTIME_LIBRARY) + # ASAN's Leak detector will report that we leaked all the PyBind11 + # Module-support code, so disable it here. + set(EXTRA_ENV_VARS ASAN_OPTIONS=detect_leaks=0 ${Halide_SANITIZER_ENV_VARS}) + endif () + set(GENERATOR_CMD ${CMAKE_COMMAND} -E env PYTHONPATH=${PYTHONPATH} ${EXTRA_ENV_VARS} ${Python3_EXECUTABLE} $) set(GENERATOR_CMD_DEPS ${ARG_FROM} Halide::Python ${py_src}) else() set(GENERATOR_CMD "${ARG_FROM}") @@ -414,6 +419,9 @@ function(add_halide_library TARGET) set_target_properties("${TARGET}" PROPERTIES POSITION_INDEPENDENT_CODE ON LINKER_LANGUAGE CXX) + # Silence many useless warnings in generated C++ code compilation + target_compile_options("${TARGET}" PRIVATE + $<$:-Wno-psabi>) _Halide_fix_xcode("${TARGET}") endif () diff --git a/cmake/toolchain.linux-x64-asan.cmake b/cmake/toolchain.linux-x64-asan.cmake index b19e37f01162..b3152e6a0472 100644 --- a/cmake/toolchain.linux-x64-asan.cmake +++ b/cmake/toolchain.linux-x64-asan.cmake @@ -33,3 +33,16 @@ set(CMAKE_CROSSCOMPILING_EMULATOR /usr/bin/env) # Can't mix -fsanitize=address with -fsanitize=fuzzer set(WITH_TEST_FUZZ OFF) + +if (NOT DEFINED Halide_SHARED_ASAN_RUNTIME_LIBRARY) + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} "-print-file-name=libclang_rt.asan.so" + OUTPUT_VARIABLE Halide_SHARED_ASAN_RUNTIME_LIBRARY + OUTPUT_STRIP_TRAILING_WHITESPACE + ) +endif () + +set(Halide_SHARED_ASAN_RUNTIME_LIBRARY "${Halide_SHARED_ASAN_RUNTIME_LIBRARY}" + CACHE FILEPATH "Library to preload when running Python tests.") + +set(Halide_SANITIZER_ENV_VARS "LD_PRELOAD=${Halide_SHARED_ASAN_RUNTIME_LIBRARY}") diff --git a/python_bindings/CMakeLists.txt b/python_bindings/CMakeLists.txt index 41e85c0d2cc7..530393b4ee27 100644 --- a/python_bindings/CMakeLists.txt +++ b/python_bindings/CMakeLists.txt @@ -62,22 +62,6 @@ endif () # A helper for creating tests with correct PYTHONPATH and sanitizer preloading ## -if (Halide_ASAN_ENABLED) - if (NOT DEFINED Halide_Python_ASAN_LIBRARY) - # TODO: this assumes clang-on-Linux, we could be smarter here and check - # CMAKE_CXX_COMPILER_ID to behave differently on GNU, AppleClang, or - # MSVC. - execute_process( - COMMAND ${CMAKE_CXX_COMPILER} "-print-file-name=libclang_rt.asan.so" - OUTPUT_VARIABLE Halide_Python_ASAN_LIBRARY - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - endif () - - set(Halide_Python_ASAN_LIBRARY "${Halide_Python_ASAN_LIBRARY}" - CACHE FILEPATH "Library to preload when running Python tests.") -endif () - function(add_python_test) cmake_parse_arguments(ARG "" "FILE;LABEL" "PYTHONPATH;ENVIRONMENT;TEST_ARGS" ${ARGN}) @@ -85,12 +69,8 @@ function(add_python_test) list(TRANSFORM ARG_PYTHONPATH PREPEND "PYTHONPATH=path_list_prepend:") list(PREPEND ARG_ENVIRONMENT "HL_TARGET=${Halide_TARGET}") - if (Halide_Python_ASAN_LIBRARY) - if (APPLE) - list(PREPEND ARG_ENVIRONMENT "DYLD_INSERT_LIBRARIES=${Halide_Python_ASAN_LIBRARY}") - else () - list(PREPEND ARG_ENVIRONMENT "LD_PRELOAD=${Halide_Python_ASAN_LIBRARY}") - endif () + if (Halide_SANITIZER_ENV_VARS) + list(PREPEND ARG_ENVIRONMENT ${Halide_SANITIZER_ENV_VARS}) endif () cmake_path(GET ARG_FILE STEM test_name) @@ -100,6 +80,7 @@ function(add_python_test) NAME "${test_name}" COMMAND Python3::Interpreter "$" ${ARG_TEST_ARGS} ) + # message(STATUS "test_name ${test_name} ARG_ENVIRONMENT ${ARG_ENVIRONMENT} ARG_PYTHONPATH ${ARG_PYTHONPATH}") set_tests_properties( "${test_name}" PROPERTIES diff --git a/python_bindings/src/halide/CMakeLists.txt b/python_bindings/src/halide/CMakeLists.txt index 5b85d545965e..c0f48f569eb1 100644 --- a/python_bindings/src/halide/CMakeLists.txt +++ b/python_bindings/src/halide/CMakeLists.txt @@ -51,6 +51,13 @@ set_target_properties( LIBRARY_OUTPUT_DIRECTORY "$/halide" EXPORT_NAME Python ) +if (Halide_ASAN_ENABLED) + set_target_properties( + Halide_Python + PROPERTIES + CMAKE_SHARED_LINKER_FLAGS -shared-libasan + ) +endif () target_link_libraries(Halide_Python PRIVATE Halide::Halide) # TODO: There's precious little information about why Python only sometimes prevents DLLs from loading from the PATH diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 84f98033adb5..ee15964123cf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -427,6 +427,15 @@ target_link_libraries(Halide PUBLIC Halide::LanguageOptions) target_compile_definitions(Halide PRIVATE $<$,STATIC_LIBRARY>:Halide_STATIC_DEFINE>) target_compile_features(Halide PUBLIC cxx_std_17) +# get_property(halide_lib_type TARGET Halide PROPERTY TYPE) +# if (Halide_ASAN_ENABLED AND NOT "${halide_lib_type}" STREQUAL "STATIC_LIBRARY") +# set_target_properties( +# Halide +# PROPERTIES +# CMAKE_SHARED_LINKER_FLAGS -shared-libasan +# ) +# endif () + include(TargetExportScript) ## TODO: implement something similar for Windows/link.exe # https://github.com/halide/Halide/issues/4651 diff --git a/test/autoschedulers/li2018/CMakeLists.txt b/test/autoschedulers/li2018/CMakeLists.txt index a24ea4a7519c..79d727ded617 100644 --- a/test/autoschedulers/li2018/CMakeLists.txt +++ b/test/autoschedulers/li2018/CMakeLists.txt @@ -40,6 +40,7 @@ if (WITH_PYTHON_BINDINGS) set_tests_properties(li2018_gradient_autoscheduler_test_py PROPERTIES LABELS "li2018;autoschedulers;auto_schedule" + ENVIRONMENT "${Halide_SANITIZER_ENV_VARS}" ENVIRONMENT_MODIFICATION "${PYTHONPATH}") endif() endif () From 16895e506abe042c2d9385d11b1af1ed6027b950 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 1 Jun 2023 15:27:41 -0700 Subject: [PATCH 02/10] Remove stuff that I didn't mean to check in --- cmake/HalideGeneratorHelpers.cmake | 3 --- python_bindings/CMakeLists.txt | 1 - src/CMakeLists.txt | 9 --------- 3 files changed, 13 deletions(-) diff --git a/cmake/HalideGeneratorHelpers.cmake b/cmake/HalideGeneratorHelpers.cmake index e9e52cde3ff6..16472cc9a9cb 100644 --- a/cmake/HalideGeneratorHelpers.cmake +++ b/cmake/HalideGeneratorHelpers.cmake @@ -419,9 +419,6 @@ function(add_halide_library TARGET) set_target_properties("${TARGET}" PROPERTIES POSITION_INDEPENDENT_CODE ON LINKER_LANGUAGE CXX) - # Silence many useless warnings in generated C++ code compilation - target_compile_options("${TARGET}" PRIVATE - $<$:-Wno-psabi>) _Halide_fix_xcode("${TARGET}") endif () diff --git a/python_bindings/CMakeLists.txt b/python_bindings/CMakeLists.txt index 530393b4ee27..c5c47fafe25b 100644 --- a/python_bindings/CMakeLists.txt +++ b/python_bindings/CMakeLists.txt @@ -80,7 +80,6 @@ function(add_python_test) NAME "${test_name}" COMMAND Python3::Interpreter "$" ${ARG_TEST_ARGS} ) - # message(STATUS "test_name ${test_name} ARG_ENVIRONMENT ${ARG_ENVIRONMENT} ARG_PYTHONPATH ${ARG_PYTHONPATH}") set_tests_properties( "${test_name}" PROPERTIES diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ee15964123cf..84f98033adb5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -427,15 +427,6 @@ target_link_libraries(Halide PUBLIC Halide::LanguageOptions) target_compile_definitions(Halide PRIVATE $<$,STATIC_LIBRARY>:Halide_STATIC_DEFINE>) target_compile_features(Halide PUBLIC cxx_std_17) -# get_property(halide_lib_type TARGET Halide PROPERTY TYPE) -# if (Halide_ASAN_ENABLED AND NOT "${halide_lib_type}" STREQUAL "STATIC_LIBRARY") -# set_target_properties( -# Halide -# PROPERTIES -# CMAKE_SHARED_LINKER_FLAGS -shared-libasan -# ) -# endif () - include(TargetExportScript) ## TODO: implement something similar for Windows/link.exe # https://github.com/halide/Halide/issues/4651 From aacfa7a8337611b3e96da7c0cac1fe957cba105c Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 1 Jun 2023 15:35:58 -0700 Subject: [PATCH 03/10] Configure cuda-specific tests properly too --- .../anderson2021/CMakeLists.txt | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/src/autoschedulers/anderson2021/CMakeLists.txt b/src/autoschedulers/anderson2021/CMakeLists.txt index 60a4db197fd9..39bc91ae5929 100644 --- a/src/autoschedulers/anderson2021/CMakeLists.txt +++ b/src/autoschedulers/anderson2021/CMakeLists.txt @@ -77,67 +77,56 @@ endif () if (WITH_TESTS) ## + function(_add_test TARGET) + add_test(NAME ${TARGET} COMMAND ${TARGET}) + set_tests_properties(${TARGET} + PROPERTIES + # This is a workaround for issues with the nvidia driver under ASAN + # https://forums.developer.nvidia.com/t/cuda-runtime-library-and-addresssanitizer-incompatibilty/63145 + ENVIRONMENT ASAN_OPTIONS=protect_shadow_gap=0 + LABELS Anderson2021) + endfunction() + add_executable(anderson2021_test_function_dag test_function_dag.cpp FunctionDAG.cpp) target_include_directories(anderson2021_test_function_dag PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021") target_link_libraries(anderson2021_test_function_dag PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin) - add_test(NAME anderson2021_test_function_dag COMMAND anderson2021_test_function_dag) - set_tests_properties(anderson2021_test_function_dag - PROPERTIES - LABELS Anderson2021) + _add_test(anderson2021_test_function_dag) add_executable(anderson2021_test_bounds test/bounds.cpp FunctionDAG.cpp LoopNest.cpp GPULoopInfo.cpp Tiling.cpp) target_include_directories(anderson2021_test_bounds PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021") target_link_libraries(anderson2021_test_bounds PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin) - add_test(NAME anderson2021_test_bounds COMMAND anderson2021_test_bounds) - set_tests_properties(anderson2021_test_bounds - PROPERTIES - LABELS Anderson2021) + _add_test(anderson2021_test_bounds) add_executable(anderson2021_test_parser test/parser.cpp) target_include_directories(anderson2021_test_parser PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021") target_link_libraries(anderson2021_test_parser PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin) - add_test(NAME anderson2021_test_parser COMMAND anderson2021_test_parser) - set_tests_properties(anderson2021_test_parser - PROPERTIES - LABELS Anderson2021) + _add_test(anderson2021_test_parser) add_executable(anderson2021_test_state test/state.cpp FunctionDAG.cpp LoopNest.cpp GPULoopInfo.cpp State.cpp Tiling.cpp) target_include_directories(anderson2021_test_state PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021") target_link_libraries(anderson2021_test_state PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin) - add_test(NAME anderson2021_test_state COMMAND anderson2021_test_state) - set_tests_properties(anderson2021_test_state - PROPERTIES - LABELS Anderson2021) + _add_test(anderson2021_test_state) add_executable(anderson2021_test_storage_strides test/storage_strides.cpp FunctionDAG.cpp LoopNest.cpp GPULoopInfo.cpp State.cpp Tiling.cpp) target_include_directories(anderson2021_test_storage_strides PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021") target_link_libraries(anderson2021_test_storage_strides PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin) - add_test(NAME anderson2021_test_storage_strides COMMAND anderson2021_test_storage_strides) - set_tests_properties(anderson2021_test_storage_strides - PROPERTIES - LABELS Anderson2021) + _add_test(anderson2021_test_storage_strides) add_executable(anderson2021_test_thread_info test/thread_info.cpp LoopNest.cpp FunctionDAG.cpp GPULoopInfo.cpp Tiling.cpp) target_include_directories(anderson2021_test_thread_info PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021") target_link_libraries(anderson2021_test_thread_info PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin) - add_test(NAME anderson2021_test_thread_info COMMAND anderson2021_test_thread_info) - set_tests_properties(anderson2021_test_thread_info - PROPERTIES - LABELS Anderson2021) + _add_test(anderson2021_test_thread_info) add_executable(anderson2021_test_tiling test/tiling.cpp Tiling.cpp) target_include_directories(anderson2021_test_tiling PRIVATE "${Halide_SOURCE_DIR}/src/autoschedulers/anderson2021") target_link_libraries(anderson2021_test_tiling PRIVATE ASLog Halide::Halide Halide::Tools Halide::Plugin) - add_test(NAME anderson2021_test_tiling COMMAND anderson2021_test_tiling) - set_tests_properties(anderson2021_test_tiling - PROPERTIES - LABELS Anderson2021) + _add_test(anderson2021_test_tiling) endif() From 402c5f56e650d71e469765337d1f2e935d198957 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Fri, 2 Jun 2023 10:36:40 -0700 Subject: [PATCH 04/10] trigger buildbots From b518dcb6ce052b994f7a144f1a29f0b4555d30ba Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Fri, 2 Jun 2023 11:26:19 -0700 Subject: [PATCH 05/10] Update CodeGen_LLVM.cpp --- src/CodeGen_LLVM.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/CodeGen_LLVM.cpp b/src/CodeGen_LLVM.cpp index b7deaa67136a..e5a505048238 100644 --- a/src/CodeGen_LLVM.cpp +++ b/src/CodeGen_LLVM.cpp @@ -1072,6 +1072,8 @@ llvm::Function *CodeGen_LLVM::embed_metadata_getter(const std::string &metadata_ buffer_estimates_array_ptr = Constant::getNullValue(i64_t->getPointerTo()->getPointerTo()); } +std::cerr<<"name for arg " << arg << " named " << args[arg].name << " ...\n"; +std::cerr<<"... is " << map_string(args[arg].name) << " ...\n"; Constant *argument_fields[] = { create_string_constant(map_string(args[arg].name)), ConstantInt::get(i32_t, args[arg].kind), @@ -3573,10 +3575,12 @@ void CodeGen_LLVM::visit(const AssertStmt *op) { Constant *CodeGen_LLVM::create_string_constant(const string &s) { map::iterator iter = string_constants.find(s); if (iter == string_constants.end()) { + std::cerr << "create_string_constant: ("< data; data.reserve(s.size() + 1); data.insert(data.end(), s.begin(), s.end()); data.push_back(0); + std::cerr << "create_string_constant: constructed ("< &data, const string &name, bool constant) { + std::cerr << "create_binary_blob name="< data_array((const unsigned char *)&data[0], data.size()); global->setInitializer(ConstantDataArray::get(*context, data_array)); size_t alignment = 32; From 82fb2640d9a98fcd24ce396ebeeee7a21a003e9a Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Fri, 2 Jun 2023 11:40:01 -0700 Subject: [PATCH 06/10] Update CodeGen_LLVM.cpp --- src/CodeGen_LLVM.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/CodeGen_LLVM.cpp b/src/CodeGen_LLVM.cpp index e5a505048238..b7deaa67136a 100644 --- a/src/CodeGen_LLVM.cpp +++ b/src/CodeGen_LLVM.cpp @@ -1072,8 +1072,6 @@ llvm::Function *CodeGen_LLVM::embed_metadata_getter(const std::string &metadata_ buffer_estimates_array_ptr = Constant::getNullValue(i64_t->getPointerTo()->getPointerTo()); } -std::cerr<<"name for arg " << arg << " named " << args[arg].name << " ...\n"; -std::cerr<<"... is " << map_string(args[arg].name) << " ...\n"; Constant *argument_fields[] = { create_string_constant(map_string(args[arg].name)), ConstantInt::get(i32_t, args[arg].kind), @@ -3575,12 +3573,10 @@ void CodeGen_LLVM::visit(const AssertStmt *op) { Constant *CodeGen_LLVM::create_string_constant(const string &s) { map::iterator iter = string_constants.find(s); if (iter == string_constants.end()) { - std::cerr << "create_string_constant: ("< data; data.reserve(s.size() + 1); data.insert(data.end(), s.begin(), s.end()); data.push_back(0); - std::cerr << "create_string_constant: constructed ("< &data, const string &name, bool constant) { - std::cerr << "create_binary_blob name="< data_array((const unsigned char *)&data[0], data.size()); global->setInitializer(ConstantDataArray::get(*context, data_array)); size_t alignment = 32; From 5979b7de6f78fbbd826ac364bd4687d0d203d039 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Fri, 2 Jun 2023 14:01:54 -0700 Subject: [PATCH 07/10] Fix sloppiness? --- CMakeLists.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a7b194ff84b..9ecf6511822e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,11 +56,10 @@ mark_as_advanced(Halide_CCACHE_BUILD) if (Halide_CCACHE_BUILD) find_program(CCACHE_PROGRAM ccache REQUIRED) - # TODO: ccache recommends setting CCACHE_SLOPPINESS=pch_defines,time_macros to - # enable precompiled header caching. Our timing found it slightly faster with - # just CCACHE_SLOPPINESS=pch_defines, so that's what we're using. Maybe revisit - # if issues occur (but we don't use any of the time macros so should be irrelevant). - set(Halide_CCACHE_PARAMS CCACHE_CPP2=yes CCACHE_HASHDIR=yes CCACHE_SLOPPINESS=pch_defines + set(Halide_CCACHE_PARAMS + CCACHE_CPP2=yes + CCACHE_HASHDIR=yes + CCACHE_SLOPPINESS=pch_defines,time_macros,include_file_mtime,include_file_ctime CACHE STRING "Parameters to pass through to ccache") mark_as_advanced(Halide_CCACHE_PARAMS) From 7c116e8da9eda75a71f95c483b102f369db35371 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Fri, 2 Jun 2023 15:54:48 -0700 Subject: [PATCH 08/10] Update CMakeLists.txt --- test/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 78501a55c541..62d460af726d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,6 +7,11 @@ add_executable(_test_internal internal.cpp) target_link_libraries(_test_internal PRIVATE Halide::Test) target_include_directories(_test_internal PRIVATE "${Halide_SOURCE_DIR}/src") target_precompile_headers(_test_internal PRIVATE ) +if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_compile_options(_test_internal PRIVATE + "$<$:SHELL:-Xclang -fno-pch-timestamp>" + ) +endif() add_halide_test(_test_internal GROUPS internal) From c196d81ec9a6a249aadc3e1315fec15b87588266 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Fri, 2 Jun 2023 17:11:56 -0700 Subject: [PATCH 09/10] trigger buildbots From 2bb54ed6744c69565a8e856812bd3134ef21bb35 Mon Sep 17 00:00:00 2001 From: Alex Reinking Date: Fri, 23 Jun 2023 10:47:17 -0700 Subject: [PATCH 10/10] Use Halide_PYTHON_LAUNCHER to implement ASAN toolchain fixes (#7657) * Use new Halide_PYTHON_LAUNCHER to set env vars * Update CMake docs for Halide_SANITIZER_ENV_VARS --------- Co-authored-by: Alex Reinking --- README_cmake.md | 45 +++++++++++++++++++---- cmake/HalideGeneratorHelpers.cmake | 20 ++++++---- cmake/toolchain.linux-x64-asan.cmake | 5 ++- python_bindings/CMakeLists.txt | 6 +-- test/autoschedulers/li2018/CMakeLists.txt | 7 ++-- 5 files changed, 59 insertions(+), 24 deletions(-) diff --git a/README_cmake.md b/README_cmake.md index 51686ef8bc30..06f8c5dadfde 100644 --- a/README_cmake.md +++ b/README_cmake.md @@ -11,11 +11,24 @@ The following sections cover each in detail. ## Table of Contents +- [Halide and CMake](#halide-and-cmake) + - [Table of Contents](#table-of-contents) - [Getting started](#getting-started) - [Installing CMake](#installing-cmake) + - [Cross-platform](#cross-platform) + - [Windows](#windows) + - [macOS](#macos) + - [Ubuntu Linux](#ubuntu-linux) - [Installing dependencies](#installing-dependencies) + - [Windows](#windows-1) + - [macOS](#macos-1) + - [Ubuntu](#ubuntu) - [Building Halide with CMake](#building-halide-with-cmake) - [Basic build](#basic-build) + - [Windows](#windows-2) + - [macOS and Linux](#macos-and-linux) + - [CMake Presets](#cmake-presets) + - [Installing](#installing) - [Build options](#build-options) - [Find module options](#find-module-options) - [Using Halide from your CMake build](#using-halide-from-your-cmake-build) @@ -33,6 +46,12 @@ The following sections cover each in detail. - [`add_halide_generator`](#add_halide_generator) - [`add_halide_python_extension_library`](#add_halide_python_extension_library) - [`add_halide_runtime`](#add_halide_runtime) + - [Cross compiling](#cross-compiling) + - [Use `add_halide_generator`](#use-add_halide_generator) + - [Use a super-build](#use-a-super-build) + - [Use `ExternalProject` directly](#use-externalproject-directly) + - [Use an emulator or run on device](#use-an-emulator-or-run-on-device) + - [Bypass CMake](#bypass-cmake) - [Contributing CMake code to Halide](#contributing-cmake-code-to-halide) - [General guidelines and best practices](#general-guidelines-and-best-practices) - [Prohibited commands list](#prohibited-commands-list) @@ -750,8 +769,8 @@ Variables that control package loading: | Variable | Description | |----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `Halide_SHARED_LIBS` | override `BUILD_SHARED_LIBS` when loading the Halide package via `find_package`. Has no effect when using Halide via `add_subdirectory` as a Git or `FetchContent` submodule. | -| `Halide_RUNTIME_NO_THREADS` | skip linking of Threads libraray to runtime. Should be set if your toolchain does not support it (e.g. baremetal). | -| `Halide_RUNTIME_NO_DL_LIBS` | skip linking of DL libraray to runtime. Should be set if your toolchain does not support it (e.g. baremetal). | +| `Halide_RUNTIME_NO_THREADS` | skip linking of Threads library to runtime. Should be set if your toolchain does not support it (e.g. baremetal). | +| `Halide_RUNTIME_NO_DL_LIBS` | skip linking of DL library to runtime. Should be set if your toolchain does not support it (e.g. baremetal). | Variables set by the package: @@ -767,6 +786,14 @@ Variables set by the package: | `Halide_ENABLE_EXCEPTIONS` | Whether Halide was compiled with exception support | | `Halide_ENABLE_RTTI` | Whether Halide was compiled with RTTI | +Variables that control package behavior: + +| Variable | Description | +|----------------------------|-------------| +| `Halide_PYTHON_LAUNCHER` | Semicolon separated list containing a command to launch the Python interpreter. Can be used to set environment variables for Python generators. | +| `Halide_NO_DEFAULT_FLAGS` | Off by default. When enabled, suppresses recommended compiler flags that would be added by `add_halide_generator` | + + ### Imported targets Halide defines the following targets that are available to users: @@ -866,10 +893,11 @@ author warning will be issued. To use an autoscheduler, set the `AUTOSCHEDULER` argument to a target named like `Namespace::Scheduler`, for example `Halide::Adams19`. This will set -the `autoscheduler` GeneratorParam on the generator command line to `Scheduler` and add the target to -the list of plugins. Additional plugins can be loaded by setting the `PLUGINS` -argument. If the argument to `AUTOSCHEDULER` does not contain `::` or it does -not name a target, it will be passed to the `-s` flag verbatim. +the `autoscheduler` GeneratorParam on the generator command line to `Scheduler` +and add the target to the list of plugins. Additional plugins can be loaded by +setting the `PLUGINS` argument. If the argument to `AUTOSCHEDULER` does not +contain `::` or it does not name a target, it will be passed to the `-s` flag +verbatim. If `GRADIENT_DESCENT` is set, then the module will be built suitably for gradient descent calculation in TensorFlow or PyTorch. See @@ -954,8 +982,9 @@ and [apps/hannk](https://github.com/halide/Halide/tree/main/apps/hannk) for a co If `PYSTUB` is specified, then a Python Extension will be built that wraps the Generator with CPython glue to allow use of the Generator Python 3.x. The result will be a a shared library of the form -`_pystub..so`, where describes the specific Python version and platform (e.g., `cpython-310-darwin` for Python 3.10 on macOS.) See -`README_python.md` for examples of use. +`_pystub..so`, where describes the specific Python +version and platform (e.g., `cpython-310-darwin` for Python 3.10 on macOS.). +See `README_python.md` for examples of use. #### `add_halide_python_extension_library` diff --git a/cmake/HalideGeneratorHelpers.cmake b/cmake/HalideGeneratorHelpers.cmake index febe7ac4dbb5..f62da88b1f7b 100644 --- a/cmake/HalideGeneratorHelpers.cmake +++ b/cmake/HalideGeneratorHelpers.cmake @@ -244,16 +244,23 @@ function(add_halide_library TARGET) if (NOT TARGET Halide::Python) message(FATAL_ERROR "This version of Halide was built without support for Python bindings; rebuild using WITH_PYTHON_BINDINGS=ON to use this rule with Python Generators.") endif () + if (NOT TARGET Python3::Interpreter) message(FATAL_ERROR "You must call find_package(Python3) in your CMake code in order to use this rule with Python Generators.") endif () - set(PYTHONPATH "$/..") - if (Halide_SHARED_ASAN_RUNTIME_LIBRARY) - # ASAN's Leak detector will report that we leaked all the PyBind11 - # Module-support code, so disable it here. - set(EXTRA_ENV_VARS ASAN_OPTIONS=detect_leaks=0 ${Halide_SANITIZER_ENV_VARS}) + + if (CMAKE_VERSION VERSION_LESS 3.24) + set(arg_sep "") + else () + set(arg_sep "--") endif () - set(GENERATOR_CMD ${CMAKE_COMMAND} -E env PYTHONPATH=${PYTHONPATH} ${EXTRA_ENV_VARS} ${Python3_EXECUTABLE} $) + + set( + GENERATOR_CMD + ${CMAKE_COMMAND} -E env "PYTHONPATH=$/.." ${arg_sep} + ${Halide_PYTHON_LAUNCHER} + "$" $ + ) set(GENERATOR_CMD_DEPS ${ARG_FROM} Halide::Python ${py_src}) else() set(GENERATOR_CMD "${ARG_FROM}") @@ -790,4 +797,3 @@ function(_Halide_gengen_ensure) _Halide_place_dll(_Halide_gengen) endif () endfunction() - diff --git a/cmake/toolchain.linux-x64-asan.cmake b/cmake/toolchain.linux-x64-asan.cmake index b3152e6a0472..14f101042a23 100644 --- a/cmake/toolchain.linux-x64-asan.cmake +++ b/cmake/toolchain.linux-x64-asan.cmake @@ -45,4 +45,7 @@ endif () set(Halide_SHARED_ASAN_RUNTIME_LIBRARY "${Halide_SHARED_ASAN_RUNTIME_LIBRARY}" CACHE FILEPATH "Library to preload when running Python tests.") -set(Halide_SANITIZER_ENV_VARS "LD_PRELOAD=${Halide_SHARED_ASAN_RUNTIME_LIBRARY}") +set( + Halide_PYTHON_LAUNCHER + ${CMAKE_COMMAND} -E env ASAN_OPTIONS=detect_leaks=0 LD_PRELOAD=${Halide_SHARED_ASAN_RUNTIME_LIBRARY} +) diff --git a/python_bindings/CMakeLists.txt b/python_bindings/CMakeLists.txt index c5c47fafe25b..df904d95084c 100644 --- a/python_bindings/CMakeLists.txt +++ b/python_bindings/CMakeLists.txt @@ -69,16 +69,13 @@ function(add_python_test) list(TRANSFORM ARG_PYTHONPATH PREPEND "PYTHONPATH=path_list_prepend:") list(PREPEND ARG_ENVIRONMENT "HL_TARGET=${Halide_TARGET}") - if (Halide_SANITIZER_ENV_VARS) - list(PREPEND ARG_ENVIRONMENT ${Halide_SANITIZER_ENV_VARS}) - endif () cmake_path(GET ARG_FILE STEM test_name) set(test_name "${ARG_LABEL}_${test_name}") add_test( NAME "${test_name}" - COMMAND Python3::Interpreter "$" ${ARG_TEST_ARGS} + COMMAND ${Halide_PYTHON_LAUNCHER} "$" "$" ${ARG_TEST_ARGS} ) set_tests_properties( "${test_name}" @@ -108,4 +105,3 @@ endif () if (WITH_TUTORIALS) add_subdirectory(tutorial) endif () - diff --git a/test/autoschedulers/li2018/CMakeLists.txt b/test/autoschedulers/li2018/CMakeLists.txt index 0f721825327b..d108a27495a2 100644 --- a/test/autoschedulers/li2018/CMakeLists.txt +++ b/test/autoschedulers/li2018/CMakeLists.txt @@ -33,8 +33,10 @@ if (WITH_PYTHON_BINDINGS) else() find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module) - add_test(NAME li2018_gradient_autoscheduler_test_py - COMMAND Python3::Interpreter "${CMAKE_CURRENT_SOURCE_DIR}/test.py" $) + add_test( + NAME li2018_gradient_autoscheduler_test_py + COMMAND ${Halide_PYTHON_LAUNCHER} "$" "${CMAKE_CURRENT_SOURCE_DIR}/test.py" $ + ) set(PYTHONPATH "$/..") list(TRANSFORM PYTHONPATH PREPEND "PYTHONPATH=path_list_prepend:") @@ -42,7 +44,6 @@ if (WITH_PYTHON_BINDINGS) set_tests_properties(li2018_gradient_autoscheduler_test_py PROPERTIES DEPENDS li2018_test LABELS "li2018;autoschedulers;auto_schedule" - ENVIRONMENT "${Halide_SANITIZER_ENV_VARS}" ENVIRONMENT_MODIFICATION "${PYTHONPATH}") endif() endif ()