From 46916f307c3688112bd83284362762a11675f1b3 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 12 Sep 2024 22:51:41 -0400 Subject: [PATCH 01/55] Add HYPRE_Setup*Toolkit.cmake files --- src/config/cmake/HYPRE_SetupCUDAToolkit.cmake | 12 ++- src/config/cmake/HYPRE_SetupGPUToolkit.cmake | 12 +++ src/config/cmake/HYPRE_SetupHIPToolkit.cmake | 78 +++++++++++++++++++ src/config/cmake/HYPRE_SetupSYCLToolkit.cmake | 39 ++++++++++ 4 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 src/config/cmake/HYPRE_SetupGPUToolkit.cmake create mode 100644 src/config/cmake/HYPRE_SetupHIPToolkit.cmake create mode 100644 src/config/cmake/HYPRE_SetupSYCLToolkit.cmake diff --git a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake index e66d040e70..1bdb202a4c 100644 --- a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake @@ -12,9 +12,15 @@ # Collection of CUDA optional libraries set(EXPORT_INTERFACE_CUDA_LIBS "") -if (NOT CUDA_FOUND) - find_package(CUDA REQUIRED) -endif () +if (CMAKE_VERSION VERSION_LESS 3.17) + if (NOT CUDA_FOUND) + find_package(CUDA REQUIRED) + endif () +else() + if (NOT CUDAToolkit_FOUND) + find_package(CUDAToolkit REQUIRED) + endif () +endif() if (CMAKE_VERSION VERSION_LESS 3.17) diff --git a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake new file mode 100644 index 0000000000..3f79ff7812 --- /dev/null +++ b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake @@ -0,0 +1,12 @@ +# Copyright (c) 1998 Lawrence Livermore National Security, LLC and other +# HYPRE Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +if(HYPRE_WITH_CUDA) + include(HYPRE_SetupCUDAToolkit) +elseif(HYPRE_WITH_HIP) + include(HYPRE_SetupHIPToolkit) +else() + message(FATAL_ERROR "Neither CUDA nor HIP is enabled. Please enable one of them.") +endif() \ No newline at end of file diff --git a/src/config/cmake/HYPRE_SetupHIPToolkit.cmake b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake new file mode 100644 index 0000000000..447bab457a --- /dev/null +++ b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake @@ -0,0 +1,78 @@ +# Copyright (c) 1998 Lawrence Livermore National Security, LLC and other +# HYPRE Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# This handles the non-compiler aspect of the HIP toolkit. +# Uses cmake find_package to locate the AMD HIP tools +# for shared libraries. Otherwise for static libraries, assumes +# the libraries are located in ${ROCM_PATH}/lib or ${ROCM_PATH}/lib64. +# Please set environment variable ROCM_PATH or HIP_PATH. + +# Check for ROCM_PATH or HIP_PATH +if(DEFINED ENV{ROCM_PATH}) + set(HIP_PATH $ENV{ROCM_PATH}) +elseif(DEFINED ENV{HIP_PATH}) + set(HIP_PATH $ENV{HIP_PATH}) +else() + message(FATAL_ERROR "Neither ROCM_PATH nor HIP_PATH environment variable is set. Please set one of them to point to your ROCm installation.") +endif() + +# Add HIP_PATH to CMAKE_PREFIX_PATH +list(APPEND CMAKE_PREFIX_PATH ${HIP_PATH}) + +# Find HIP package +find_package(hip REQUIRED CONFIG) + +# Set HIP-specific variables +set(CMAKE_HIP_ARCHITECTURES "gfx940" CACHE STRING "target HIP architectures") +set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -fPIC" CACHE STRING "HIP compiler flags" FORCE) + +# Collection of HIP optional libraries +set(EXPORT_INTERFACE_HIP_LIBS "") + +if(HYPRE_SHARED OR WIN32) + set(HYPRE_HIP_TOOLKIT_STATIC FALSE) +else() + set(HYPRE_HIP_TOOLKIT_STATIC TRUE) +endif() + +# Function to find and add libraries +function(find_and_add_hip_library LIB_NAME) + if(HYPRE_ENABLE_${LIB_NAME}) + set(HYPRE_USING_${LIB_NAME} ON CACHE BOOL "" FORCE) + if(HYPRE_HIP_TOOLKIT_STATIC) + list(APPEND EXPORT_INTERFACE_HIP_LIBS roc::${LIB_NAME}_static) + else() + list(APPEND EXPORT_INTERFACE_HIP_LIBS roc::${LIB_NAME}) + endif() + set(EXPORT_INTERFACE_HIP_LIBS ${EXPORT_INTERFACE_HIP_LIBS} PARENT_SCOPE) + endif() +endfunction() + +# Find and add libraries +find_and_add_hip_library(rocblas) +find_and_add_hip_library(rocsparse) +find_and_add_hip_library(rocrand) +find_and_add_hip_library(rocsolver) + +if(HYPRE_ENABLE_GPU_PROFILING) + set(HYPRE_USING_ROCTRACER ON CACHE BOOL "" FORCE) + find_library(ROCTRACER_LIBRARY + NAMES libroctracer64.so + PATHS ${HIP_PATH}/lib ${HIP_PATH}/lib64 + NO_DEFAULT_PATH) + if(ROCTRACER_LIBRARY) + message(STATUS "ROCm tracer library found in ${ROCTRACER_LIBRARY}") + list(APPEND EXPORT_INTERFACE_HIP_LIBS ${ROCTRACER_LIBRARY}) + else() + message(WARNING "ROCm tracer library not found. GPU profiling may not work correctly.") + endif() +endif() + +# Make EXPORT_INTERFACE_HIP_LIBS available to parent scope +if(PARENT_SCOPE) + set(EXPORT_INTERFACE_HIP_LIBS ${EXPORT_INTERFACE_HIP_LIBS} PARENT_SCOPE) +else() + set(EXPORT_INTERFACE_HIP_LIBS ${EXPORT_INTERFACE_HIP_LIBS}) +endif() \ No newline at end of file diff --git a/src/config/cmake/HYPRE_SetupSYCLToolkit.cmake b/src/config/cmake/HYPRE_SetupSYCLToolkit.cmake new file mode 100644 index 0000000000..ed49f3826b --- /dev/null +++ b/src/config/cmake/HYPRE_SetupSYCLToolkit.cmake @@ -0,0 +1,39 @@ +# Copyright (c) 1998 Lawrence Livermore National Security, LLC and other +# HYPRE Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# This handles the non-compiler aspect of the SYCL toolkit. + +if (HYPRE_ENABLE_ONEMKLSPARSE) + set(HYPRE_USING_ONEMKLSPARSE ON CACHE BOOL "" FORCE) +endif() + +if (HYPRE_ENABLE_ONEMKLBLAS) + set(HYPRE_USING_ONEMKLBLAS ON CACHE BOOL "" FORCE) +endif() + +if (HYPRE_ENABLE_ONEMKLRAND) + set(HYPRE_USING_ONEMKLRAND ON CACHE BOOL "" FORCE) +endif() + +if (HYPRE_USING_ONEMKLSPARSE OR HYPRE_USING_ONEMKLBLAS OR HYPRE_USING_ONEMKLRAND) + set(MKL_LINK static) + set(MKL_THREADING sequential) + find_package(MKL CONFIG REQUIRED HINTS "$ENV{MKLROOT}/lib/cmake/mkl") + target_compile_options(${PROJECT_NAME} PUBLIC $) + target_include_directories(${PROJECT_NAME} PUBLIC $) + target_link_libraries(${PROJECT_NAME} PUBLIC $) +endif() + +target_include_directories(${PROJECT_NAME} PUBLIC $ENV{DPLROOT}/include) + +if (HYPRE_SYCL_TARGET) + target_compile_options(${PROJECT_NAME} PUBLIC -fsycl-targets=${HYPRE_SYCL_TARGET}) + target_link_options(${PROJECT_NAME} PUBLIC -fsycl-targets=${HYPRE_SYCL_TARGET}) +endif() + +if (HYPRE_SYCL_TARGET_BACKEND) + target_compile_options(${PROJECT_NAME} PUBLIC -fsycl-targets=${HYPRE_SYCL_TARGET_BACKEND}) + target_link_options(${PROJECT_NAME} PUBLIC -fsycl-targets=${HYPRE_SYCL_TARGET_BACKEND}) +endif() \ No newline at end of file From 75833902c82e23e17e3f135aafac01fa81652c4b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 12 Sep 2024 23:26:13 -0400 Subject: [PATCH 02/55] Add HYPRE_USING_GPU to define C++ source files --- src/IJ_mv/CMakeLists.txt | 2 +- src/parcsr_ls/CMakeLists.txt | 2 +- src/parcsr_mv/CMakeLists.txt | 2 +- src/seq_block_mv/CMakeLists.txt | 2 +- src/seq_mv/CMakeLists.txt | 2 +- src/sstruct_ls/CMakeLists.txt | 2 +- src/sstruct_mv/CMakeLists.txt | 2 +- src/struct_ls/CMakeLists.txt | 2 +- src/struct_mv/CMakeLists.txt | 2 +- src/utilities/CMakeLists.txt | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/IJ_mv/CMakeLists.txt b/src/IJ_mv/CMakeLists.txt index 6291d5186d..7449654aec 100644 --- a/src/IJ_mv/CMakeLists.txt +++ b/src/IJ_mv/CMakeLists.txt @@ -34,7 +34,7 @@ target_sources(${PROJECT_NAME} ${HDRS} ) -if (HYPRE_USING_CUDA OR HYPRE_USING_SYCL) +if (HYPRE_USING_GPU) set(GPU_SRCS IJMatrix_parcsr_device.c IJVector_parcsr_device.c diff --git a/src/parcsr_ls/CMakeLists.txt b/src/parcsr_ls/CMakeLists.txt index d46cfc1f72..87148a2427 100644 --- a/src/parcsr_ls/CMakeLists.txt +++ b/src/parcsr_ls/CMakeLists.txt @@ -155,7 +155,7 @@ target_sources(${PROJECT_NAME} ${HDRS} ) -if (HYPRE_USING_CUDA OR HYPRE_USING_SYCL) +if (HYPRE_USING_GPU) set(GPU_SRCS ams.c ads.c diff --git a/src/parcsr_mv/CMakeLists.txt b/src/parcsr_mv/CMakeLists.txt index c2277e571e..64583c9e79 100644 --- a/src/parcsr_mv/CMakeLists.txt +++ b/src/parcsr_mv/CMakeLists.txt @@ -46,7 +46,7 @@ target_sources(${PROJECT_NAME} ${HDRS} ) -if (HYPRE_USING_CUDA OR HYPRE_USING_SYCL) +if (HYPRE_USING_GPU) set(GPU_SRCS par_csr_matvec_device.c par_csr_fffc_device.c diff --git a/src/seq_block_mv/CMakeLists.txt b/src/seq_block_mv/CMakeLists.txt index 897f426be6..59fbccf7bb 100644 --- a/src/seq_block_mv/CMakeLists.txt +++ b/src/seq_block_mv/CMakeLists.txt @@ -17,7 +17,7 @@ target_sources(${PROJECT_NAME} ${HDRS} ) -# if (HYPRE_USING_CUDA OR HYPRE_USING_HIP) +# if (HYPRE_USING_GPU) # set(GPU_SRCS # # File name here # ) diff --git a/src/seq_mv/CMakeLists.txt b/src/seq_mv/CMakeLists.txt index 6a2a937d4b..c7962b47de 100644 --- a/src/seq_mv/CMakeLists.txt +++ b/src/seq_mv/CMakeLists.txt @@ -87,7 +87,7 @@ target_sources(${PROJECT_NAME} ${HDRS} ) -if (HYPRE_USING_CUDA OR HYPRE_USING_SYCL) +if (HYPRE_USING_GPU) set(GPU_SRCS csr_matop_device.c csr_matrix_cuda_utils.c diff --git a/src/sstruct_ls/CMakeLists.txt b/src/sstruct_ls/CMakeLists.txt index 0c49254d21..2d35908a5e 100644 --- a/src/sstruct_ls/CMakeLists.txt +++ b/src/sstruct_ls/CMakeLists.txt @@ -79,7 +79,7 @@ target_sources(${PROJECT_NAME} ${HDRS} ) -if (HYPRE_USING_CUDA OR HYPRE_USING_SYCL) +if (HYPRE_USING_GPU) set(GPU_SRCS fac_amr_fcoarsen.c fac_amr_rap.c diff --git a/src/sstruct_mv/CMakeLists.txt b/src/sstruct_mv/CMakeLists.txt index 50a6aeb11f..3b674ab9c7 100644 --- a/src/sstruct_mv/CMakeLists.txt +++ b/src/sstruct_mv/CMakeLists.txt @@ -36,7 +36,7 @@ target_sources(${PROJECT_NAME} ${HDRS} ) -if (HYPRE_USING_CUDA OR HYPRE_USING_SYCL) +if (HYPRE_USING_GPU) set(GPU_SRCS sstruct_matrix.c sstruct_vector.c diff --git a/src/struct_ls/CMakeLists.txt b/src/struct_ls/CMakeLists.txt index 2df3d0ab68..952398324d 100644 --- a/src/struct_ls/CMakeLists.txt +++ b/src/struct_ls/CMakeLists.txt @@ -81,7 +81,7 @@ target_sources(${PROJECT_NAME} ${HDRS} ) -if (HYPRE_USING_CUDA OR HYPRE_USING_SYCL) +if (HYPRE_USING_GPU) set(GPU_SRCS cyclic_reduction.c HYPRE_struct_int.c diff --git a/src/struct_mv/CMakeLists.txt b/src/struct_mv/CMakeLists.txt index acc6f40c56..20e4dc1f59 100644 --- a/src/struct_mv/CMakeLists.txt +++ b/src/struct_mv/CMakeLists.txt @@ -44,7 +44,7 @@ target_sources(${PROJECT_NAME} ${HDRS} ) -if (HYPRE_USING_CUDA OR HYPRE_USING_SYCL) +if (HYPRE_USING_GPU) set(GPU_SRCS struct_axpy.c struct_communication.c diff --git a/src/utilities/CMakeLists.txt b/src/utilities/CMakeLists.txt index 648c50b495..27af9d86cf 100644 --- a/src/utilities/CMakeLists.txt +++ b/src/utilities/CMakeLists.txt @@ -59,7 +59,7 @@ target_sources(${PROJECT_NAME} ${HDRS} ) -if (HYPRE_USING_CUDA OR HYPRE_USING_SYCL) +if (HYPRE_USING_GPU) set(GPU_SRCS device_utils.c general.c From d79a20460661d5bb77a31894f347c48dd6a7581e Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 12 Sep 2024 23:26:57 -0400 Subject: [PATCH 03/55] Initial working version for HIP support --- src/CMakeLists.txt | 243 ++++++------------ src/config/HYPRE_config.h.cmake.in | 15 ++ src/config/cmake/HYPRE_SetupCUDAToolkit.cmake | 5 +- src/config/cmake/HYPRE_SetupHIPToolkit.cmake | 11 +- 4 files changed, 105 insertions(+), 169 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b8da5f4be9..544ded961d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,6 +11,12 @@ else () cmake_policy(VERSION 3.16) endif () +# Set CMP0146 policy to NEW. This policy setting ensures we use the +# modern CUDA language support instead of the deprecated FindCUDA module +if(POLICY CMP0146) + cmake_policy(SET CMP0146 NEW) +endif() + # The version number. set(HYPRE_VERSION 2.31.0) set(HYPRE_NUMBER 23100) @@ -110,6 +116,7 @@ set(HYPRE_WITH_EXTRA_CFLAGS "" CACHE STRING "Define extra C compile flags" set(HYPRE_WITH_EXTRA_CXXFLAGS "" CACHE STRING "Define extra CXX compile flags") # GPU options option(HYPRE_WITH_CUDA "Use CUDA. Require cuda-8.0 or higher" OFF) +option(HYPRE_WITH_HIP "Use HIP" OFF) option(HYPRE_WITH_SYCL "Use SYCL" OFF) option(HYPRE_ENABLE_UNIFIED_MEMORY "Use unified memory for allocating the memory" OFF) option(HYPRE_ENABLE_DEVICE_MALLOC_ASYNC "Use device async malloc" OFF) @@ -122,6 +129,11 @@ option(HYPRE_ENABLE_CUBLAS "Use cuBLAS" OFF) option(HYPRE_ENABLE_CURAND "Use cuRAND" ON) option(HYPRE_ENABLE_GPU_PROFILING "Use NVTX on CUDA" OFF) set(HYPRE_CUDA_SM "70" CACHE STRING "Target CUDA architecture.") +# HIP options +option(HYPRE_ENABLE_ROCBLAS "Use rocBLAS" ON) +option(HYPRE_ENABLE_ROCSPARSE "Use rocSPARSE" ON) +option(HYPRE_ENABLE_ROCRAND "Use rocRAND" ON) +option(HYPRE_ENABLE_ROCSOLVER "Use rocSOLVER" ON) # oneAPI options option(HYPRE_ENABLE_ONEMKLSPARSE "Use oneMKL sparse" ON) option(HYPRE_ENABLE_ONEMKLBLAS "Use oneMKL blas" ON) @@ -262,164 +274,74 @@ if (HYPRE_WITH_UMPIRE_PINNED) set(HYPRE_USING_UMPIRE_PINNED ON CACHE BOOL "" FORCE) endif () -# CUDA -if (HYPRE_WITH_CUDA) +if (HYPRE_WITH_CUDA OR HYPRE_WITH_HIP OR HYPRE_WITH_SYCL) enable_language(CXX) message(STATUS "Enabled support for CXX.") - # Enforce C++11 - if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 11) - set(CMAKE_CXX_STANDARD 11) - endif () - set(CMAKE_CXX_STANDARD_REQUIRED ON) - - message(STATUS "Using CXX standard: c++${CMAKE_CXX_STANDARD}") - - # Add any extra CXX compiler flags HYPRE_WITH_EXTRA_CXXFLAGS - if (NOT HYPRE_WITH_EXTRA_CXXFLAGS STREQUAL "") - string(REPLACE " " ";" HYPRE_WITH_EXTRA_CXXFLAGS "${HYPRE_WITH_EXTRA_CXXFLAGS}") - add_compile_options("$<$:${HYPRE_WITH_EXTRA_CXXFLAGS}>") - endif () - - # Check if CUDA is available, then enable it - include(CheckLanguage) - check_language(CUDA) - - # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler. - if (NOT CMAKE_CUDA_HOST_COMPILER) - set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) - endif () - - if (CMAKE_CUDA_COMPILER) + set(HYPRE_USING_GPU ON CACHE BOOL "" FORCE) + if (HYPRE_WITH_CUDA) + set(HYPRE_USING_CUDA TRUE) enable_language(CUDA) - message(STATUS "Enabled support for CUDA.") - - if (NOT CMAKE_CUDA_STANDARD OR CMAKE_CUDA_STANDARD EQUAL 98) - set(CMAKE_CUDA_STANDARD 11) - endif () - - set(CMAKE_CUDA_STANDARD_REQUIRED ON CACHE BOOL "" FORCE) - - if (HYPRE_ENABLE_DEVICE_MALLOC_ASYNC) - set(HYPRE_USING_DEVICE_MALLOC_ASYNC ON CACHE BOOL "" FORCE) - endif () - - set(HYPRE_USING_CUDA ON CACHE BOOL "" FORCE) - set(HYPRE_USING_GPU ON CACHE BOOL "" FORCE) + include(HYPRE_SetupCUDAToolkit) - if (HYPRE_ENABLE_UNIFIED_MEMORY) - set(HYPRE_USING_UNIFIED_MEMORY ON CACHE BOOL "" FORCE) - else () - set(HYPRE_USING_DEVICE_MEMORY ON CACHE BOOL "" FORCE) + # Enforce C++11 + if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 11) + set(CMAKE_CXX_STANDARD 11) endif () - - # Check if examples are enabled, but not unified memory - if (HYPRE_BUILD_EXAMPLES AND NOT HYPRE_ENABLE_UNIFIED_MEMORY) - message(WARNING "Running the examples on GPUs requires Unified Memory! - Examples will not be built!") - set(HYPRE_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + elseif (HYPRE_WITH_HIP) + set(HYPRE_USING_HIP TRUE) + enable_language(HIP) + include(HYPRE_SetupHIPToolkit) + + # Enforce C++14 + if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) + set(CMAKE_CXX_STANDARD 14) endif () - - if (CMAKE_VERSION VERSION_LESS 3.18.0) - add_compile_options("$<$:-arch=sm_${HYPRE_CUDA_SM}>") - else () - set(CMAKE_CUDA_ARCHITECTURES "${HYPRE_CUDA_SM}") + elseif (HYPRE_WITH_SYCL) + set(HYPRE_USING_SYCL TRUE) + enable_language(SYCL) + include(HYPRE_SetupSYCLToolkit) + + # Enforce C++14 + if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) + set(CMAKE_CXX_STANDARD 14) endif () - message(STATUS "Using CUDA architecture: ${HYPRE_CUDA_SM}") - - add_compile_options("$<$:-expt-extended-lambda>") - - set(HYPRE_USING_HOST_MEMORY OFF CACHE BOOL "" FORCE) - - if (HYPRE_ENABLE_CUDA_STREAMS) - set(HYPRE_USING_CUDA_STREAMS ON CACHE BOOL "" FORCE) - endif (HYPRE_ENABLE_CUDA_STREAMS) - - if (HYPRE_ENABLE_DEVICE_POOL) - set(HYPRE_USING_DEVICE_POOL ON CACHE BOOL "" FORCE) - endif (HYPRE_ENABLE_DEVICE_POOL) - - # TODO Eventually should require cmake>=3.17 - # and use cmake's FindCUDAToolkit. Now collect - # CUDA optional libraries. - include(HYPRE_SetupCUDAToolkit) - else () - message(WARNING "No CUDA support!") - set(HYPRE_USING_HOST_MEMORY ON CACHE BOOL "" FORCE) - endif (CMAKE_CUDA_COMPILER) -endif (HYPRE_WITH_CUDA) - -# SYCL -if (HYPRE_WITH_SYCL) - enable_language(CXX) - message(STATUS "Enabled support for CXX.") - - # Enforce C++17 - if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 17) - set(CMAKE_CXX_STANDARD 17) endif () - set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_STANDARD_REQUIRED ON) message(STATUS "Using CXX standard: c++${CMAKE_CXX_STANDARD}") - # Set CXX compiler to dpcpp - # WM: note that dpcpp is deprecated, but for now oneMKL looks for the dpcpp compiler to configure things appropriately for the sycl backend - set(CMAKE_CXX_COMPILER "dpcpp") - - # Set linker to dpcpp - set(CMAKE_LINKER "dpcpp") - set(CMAKE_CXX_LINK_EXECUTABLE " -o ") - set(CMAKE_CXX_LINKER_WRAPPER_FLAG " ") - set(CMAKE_CXX_LINKER_WRAPPER_FLAG_SEP " ") - # Add any extra CXX compiler flags HYPRE_WITH_EXTRA_CXXFLAGS if (NOT HYPRE_WITH_EXTRA_CXXFLAGS STREQUAL "") - string(REPLACE " " ";" HYPRE_WITH_EXTRA_CXXFLAGS "${HYPRE_WITH_EXTRA_CXXFLAGS}") - add_compile_options("$<$:${HYPRE_WITH_EXTRA_CXXFLAGS}>") + string(REPLACE " " ";" HYPRE_WITH_EXTRA_CXXFLAGS_LIST ${HYPRE_WITH_EXTRA_CXXFLAGS}) + add_compile_options("$<$:${HYPRE_WITH_EXTRA_CXXFLAGS_LIST}>") endif () - set(HYPRE_USING_SYCL ON CACHE BOOL "" FORCE) - set(HYPRE_USING_GPU ON CACHE BOOL "" FORCE) + if (HYPRE_ENABLE_GPU_STREAMS) + set(HYPRE_USING_GPU_STREAMS ON CACHE BOOL "" FORCE) + endif () + if (HYPRE_ENABLE_DEVICE_POOL) + set(HYPRE_USING_DEVICE_POOL ON CACHE BOOL "" FORCE) + endif () + + set(HYPRE_USING_HOST_MEMORY OFF CACHE BOOL "" FORCE) if (HYPRE_ENABLE_UNIFIED_MEMORY) set(HYPRE_USING_UNIFIED_MEMORY ON CACHE BOOL "" FORCE) else () set(HYPRE_USING_DEVICE_MEMORY ON CACHE BOOL "" FORCE) endif () - # Check if examples are enabled, but not unified memory - if (HYPRE_BUILD_EXAMPLES AND NOT HYPRE_ENABLE_UNIFIED_MEMORY) - message(WARNING "Running the examples on GPUs requires Unified Memory! - Examples will not be built!") - set(HYPRE_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) - endif () - - add_compile_options("$<$:-fsycl>") - add_compile_options("$<$:-fsycl-unnamed-lambda>") - add_link_options("-fsycl") - add_link_options("-fsycl-device-code-split=per_kernel") - add_link_options("-Wl,--no-relax") - - if (HYPRE_SYCL_TARGET) - add_link_options("SHELL:-fsycl-targets=${HYPRE_SYCL_TARGET}") - endif () - if (HYPRE_SYCL_TARGET_BACKEND) - add_link_options("SHELL:-Xsycl-target-backend '${HYPRE_SYCL_TARGET_BACKEND}'") - endif () - - set(HYPRE_USING_HOST_MEMORY OFF CACHE BOOL "" FORCE) - - if (HYPRE_ENABLE_CUDA_STREAMS) - set(HYPRE_USING_CUDA_STREAMS ON CACHE BOOL "" FORCE) - endif (HYPRE_ENABLE_CUDA_STREAMS) - -endif (HYPRE_WITH_SYCL) +else () + message(STATUS "No GPU support enabled.") + set(HYPRE_USING_HOST_MEMORY ON CACHE BOOL "" FORCE) +endif () # Add any extra C compiler flags HYPRE_WITH_EXTRA_CFLAGS if (NOT HYPRE_WITH_EXTRA_CFLAGS STREQUAL "") - string(REPLACE " " ";" HYPRE_WITH_EXTRA_CFLAGS "${HYPRE_WITH_EXTRA_CFLAGS}") - add_compile_options("$<$:${HYPRE_WITH_EXTRA_CFLAGS}>") + string(REPLACE " " ";" HYPRE_WITH_EXTRA_CFLAGS_LIST ${HYPRE_WITH_EXTRA_CFLAGS}) + add_compile_options("$<$:${HYPRE_WITH_EXTRA_CFLAGS_LIST}>") endif () # Set library build type (must appear before add_library calls) @@ -577,27 +499,6 @@ if (HYPRE_USING_CUDA) endif () endif () -if (HYPRE_USING_SYCL) - target_include_directories(${PROJECT_NAME} PUBLIC $ENV{DPLROOT}/include) - if (HYPRE_ENABLE_ONEMKLSPARSE) - set(HYPRE_USING_ONEMKLSPARSE ON CACHE BOOL "" FORCE) - endif() - if (HYPRE_ENABLE_ONEMKLBLAS) - set(HYPRE_USING_ONEMKLBLAS ON CACHE BOOL "" FORCE) - endif() - if (HYPRE_ENABLE_ONEMKLRAND) - set(HYPRE_USING_ONEMKLRAND ON CACHE BOOL "" FORCE) - endif() - if (HYPRE_USING_ONEMKLSPARSE OR HYPRE_USING_ONEMKLBLAS OR HYPRE_USING_ONEMKLRAND) - set(MKL_LINK static) - set(MKL_THREADING sequential) - find_package(MKL CONFIG REQUIRED HINTS "$ENV{MKLROOT}/lib/cmake/mkl") - target_compile_options(${PROJECT_NAME} PUBLIC $) - target_include_directories(${PROJECT_NAME} PUBLIC $) - target_link_libraries(${PROJECT_NAME} PUBLIC $) - endif() -endif() - if (HYPRE_USING_CALIPER) if (NOT TPL_CALIPER_LIBRARIES OR NOT TPL_CALIPER_INCLUDE_DIRS) message(FATAL_ERROR "Both TPL_CALIPER_LIBRARIES and TPL_CALIPER_INCLUDE_DIRS options must be set for Caliper support.") @@ -612,7 +513,6 @@ if (HYPRE_USING_CALIPER) message(STATUS "Enabled support for using Caliper.") target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_CALIPER_LIBRARIES}) target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_CALIPER_INCLUDE_DIRS}) - endif() # Find Umpire, if requested @@ -655,19 +555,30 @@ target_include_directories(${PROJECT_NAME} PUBLIC $ ) -if (HYPRE_USING_CUDA) - set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE CUDA) -endif () - -if (HYPRE_USING_SYCL) - set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE CXX) -endif () - # Set MPI compile flags if (NOT HYPRE_SEQUENTIAL) find_program(MPIEXEC_EXECUTABLE NAMES mpiexec mpirun) find_package(MPI REQUIRED) target_link_libraries(${PROJECT_NAME} PUBLIC MPI::MPI_C) + + # Determine the correct MPI include directory + if(MPI_CXX_INCLUDE_DIRS) + set(MPI_INCLUDE_DIRS ${MPI_CXX_INCLUDE_DIRS}) + elseif(MPI_CXX_INCLUDE_PATH) + set(MPI_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH}) + elseif(MPI_CXX_COMPILER_INCLUDE_DIRS) + set(MPI_INCLUDE_DIRS ${MPI_CXX_COMPILER_INCLUDE_DIRS}) + elseif(MPI_C_COMPILER_INCLUDE_DIRS) + set(MPI_INCLUDE_DIRS ${MPI_C_COMPILER_INCLUDE_DIRS}) + elseif(MPI_C_INCLUDE_DIRS) + set(MPI_INCLUDE_DIRS ${MPI_C_INCLUDE_DIRS}) + elseif(MPI_C_INCLUDE_PATH) + set(MPI_INCLUDE_DIRS ${MPI_C_INCLUDE_PATH}) + elseif(MPI_INCLUDE_PATH) + set(MPI_INCLUDE_DIRS ${MPI_INCLUDE_PATH}) + else() + message(WARNING "MPI include directory not found. Compilation may fail.") + endif() endif (NOT HYPRE_SEQUENTIAL) # Set OpenMP compile flags @@ -676,6 +587,16 @@ if (HYPRE_USING_OPENMP) target_link_libraries(${PROJECT_NAME} PUBLIC OpenMP::OpenMP_C) endif (HYPRE_USING_OPENMP) +if (HYPRE_WITH_CUDA) + set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE CUDA) +elseif (HYPRE_WITH_HIP) + set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE HIP) + target_link_libraries(${PROJECT_NAME} PUBLIC ${EXPORT_INTERFACE_HIP_LIBS}) + target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIRS}) # Ensure MPI headers are included for HIP compilation +elseif (HYPRE_WITH_SYCL) + set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE SYCL) +endif () + if (MSVC) target_compile_definitions(${PROJECT_NAME} PRIVATE _CRT_SECURE_NO_WARNINGS) if (MSVC_VERSION LESS 1928) # Visual Studio 2019 version 16.8 claims full C11 support diff --git a/src/config/HYPRE_config.h.cmake.in b/src/config/HYPRE_config.h.cmake.in index 21fcc5bf6b..1de8e5ca70 100644 --- a/src/config/HYPRE_config.h.cmake.in +++ b/src/config/HYPRE_config.h.cmake.in @@ -92,6 +92,21 @@ /* Use if executing on device with SYCL */ #cmakedefine HYPRE_USING_SYCL 1 +/* Use rocBLAS */ +#cmakedefine HYPRE_USING_ROCBLAS 1 + +/* Use rocRAND */ +#cmakedefine HYPRE_USING_ROCRAND 1 + +/* Use rocSPARSE */ +#cmakedefine HYPRE_USING_ROCSPARSE 1 + +/* Use rocSOLVER */ +#cmakedefine HYPRE_USING_ROCSOLVER 1 + +/* Use rocTX profiling */ +#cmakedefine HYPRE_USING_ROCTX 1 + /* Use cuBLAS */ #cmakedefine HYPRE_USING_CUBLAS 1 diff --git a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake index 1bdb202a4c..1d1d03dbe6 100644 --- a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake @@ -4,10 +4,10 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) # This handles the non-compiler aspect of the CUDA toolkit. -# Uses cmake find_package to locate the NVIDIA CUDA C tools +# Uses cmake find_package to locate the NVIDIA CUDA C tools # for shared libraries. Otherwise for static libraries, assumes # the libraries are located in ${CUDA_TOOLKIT_ROOT_DIR}/lib64. -# Please set cmake variable CUDA_TOOLKIT_ROOT_DIR. +# Please set cmake variable CUDA_TOOLKIT_ROOT_DIR. # Collection of CUDA optional libraries set(EXPORT_INTERFACE_CUDA_LIBS "") @@ -23,7 +23,6 @@ else() endif() if (CMAKE_VERSION VERSION_LESS 3.17) - if (HYPRE_ENABLE_CUSPARSE) set(HYPRE_USING_CUSPARSE ON CACHE BOOL "" FORCE) if (HYPRE_SHARED) diff --git a/src/config/cmake/HYPRE_SetupHIPToolkit.cmake b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake index 447bab457a..8a1e9bd81e 100644 --- a/src/config/cmake/HYPRE_SetupHIPToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake @@ -4,7 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) # This handles the non-compiler aspect of the HIP toolkit. -# Uses cmake find_package to locate the AMD HIP tools +# Uses cmake find_package to locate the AMD HIP tools # for shared libraries. Otherwise for static libraries, assumes # the libraries are located in ${ROCM_PATH}/lib or ${ROCM_PATH}/lib64. # Please set environment variable ROCM_PATH or HIP_PATH. @@ -25,7 +25,6 @@ list(APPEND CMAKE_PREFIX_PATH ${HIP_PATH}) find_package(hip REQUIRED CONFIG) # Set HIP-specific variables -set(CMAKE_HIP_ARCHITECTURES "gfx940" CACHE STRING "target HIP architectures") set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -fPIC" CACHE STRING "HIP compiler flags" FORCE) # Collection of HIP optional libraries @@ -39,8 +38,10 @@ endif() # Function to find and add libraries function(find_and_add_hip_library LIB_NAME) - if(HYPRE_ENABLE_${LIB_NAME}) - set(HYPRE_USING_${LIB_NAME} ON CACHE BOOL "" FORCE) + string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) + if(HYPRE_ENABLE_${LIB_NAME_UPPER}) + find_package(${LIB_NAME} REQUIRED) + set(HYPRE_USING_${LIB_NAME_UPPER} ON CACHE BOOL "" FORCE) if(HYPRE_HIP_TOOLKIT_STATIC) list(APPEND EXPORT_INTERFACE_HIP_LIBS roc::${LIB_NAME}_static) else() @@ -75,4 +76,4 @@ if(PARENT_SCOPE) set(EXPORT_INTERFACE_HIP_LIBS ${EXPORT_INTERFACE_HIP_LIBS} PARENT_SCOPE) else() set(EXPORT_INTERFACE_HIP_LIBS ${EXPORT_INTERFACE_HIP_LIBS}) -endif() \ No newline at end of file +endif() From 777875d3e51a6fc5b3fe14786cb9cf94a53b855f Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Sep 2024 13:41:09 -0400 Subject: [PATCH 04/55] Add summary table + misc improvements --- src/CMakeLists.txt | 125 ++++++++++++------- src/config/cmake/HYPRE_CMakeUtilities.cmake | 55 ++++++++ src/config/cmake/HYPRE_SetupGPUToolkit.cmake | 4 +- 3 files changed, 135 insertions(+), 49 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 544ded961d..164e4a6dcb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -112,8 +112,6 @@ option(HYPRE_TIMING "Use HYPRE timing routines" OFF) option(HYPRE_BUILD_EXAMPLES "Build examples" OFF) option(HYPRE_BUILD_TESTS "Build tests" OFF) option(HYPRE_USING_HOST_MEMORY "Use host memory" ON) -set(HYPRE_WITH_EXTRA_CFLAGS "" CACHE STRING "Define extra C compile flags") -set(HYPRE_WITH_EXTRA_CXXFLAGS "" CACHE STRING "Define extra CXX compile flags") # GPU options option(HYPRE_WITH_CUDA "Use CUDA. Require cuda-8.0 or higher" OFF) option(HYPRE_WITH_HIP "Use HIP" OFF) @@ -138,17 +136,15 @@ option(HYPRE_ENABLE_ROCSOLVER "Use rocSOLVER" ON) option(HYPRE_ENABLE_ONEMKLSPARSE "Use oneMKL sparse" ON) option(HYPRE_ENABLE_ONEMKLBLAS "Use oneMKL blas" ON) option(HYPRE_ENABLE_ONEMKLRAND "Use oneMKL rand" ON) -set(HYPRE_SYCL_TARGET "" CACHE STRING "Target SYCL architecture, e.g. 'spir64_gen'.") -set(HYPRE_SYCL_TARGET_BACKEND "" CACHE STRING "Additional SYCL backend options, e.g. '-device 12.1.0,12.4.0'.") # Umpire resource management options option(HYPRE_WITH_UMPIRE "Use Umpire Allocator for device and unified memory" OFF) option(HYPRE_WITH_UMPIRE_HOST "Use Umpire Allocator for host memory" OFF) option(HYPRE_WITH_UMPIRE_DEVICE "Use Umpire Allocator for device memory" OFF) option(HYPRE_WITH_UMPIRE_UM "Use Umpire Allocator for unified memory" OFF) option(HYPRE_WITH_UMPIRE_PINNED "Use Umpire Allocator for pinned memory" OFF) +# TPL options option(TPL_UMPIRE_LIBRARIES "List of absolute paths to Umpire link libraries [].") option(TPL_UMPIRE_INCLUDE_DIRS "List of absolute paths to Umpire include directories [].") - option(TPL_SUPERLU_LIBRARIES "List of absolute paths to SuperLU link libraries [].") option(TPL_SUPERLU_INCLUDE_DIRS "List of absolute paths to SuperLU include directories [].") option(TPL_DSUPERLU_LIBRARIES "List of absolute paths to SuperLU_Dist link libraries [].") @@ -158,6 +154,11 @@ option(TPL_MAGMA_INCLUDE_DIRS "List of absolute paths to MAGMA include di option(TPL_BLAS_LIBRARIES "Optional list of absolute paths to BLAS libraries, otherwise use FindBLAS to locate [].") option(TPL_LAPACK_LIBRARIES "Optional list of absolute paths to LAPACK libraries, otherwise use FindLAPACK to locate [].") option(TPL_FEI_INCLUDE_DIRS "List of absolute paths to FEI include directories [].") +# Extra flags +set(HYPRE_WITH_EXTRA_CFLAGS "" CACHE STRING "Define extra C compile flags") +set(HYPRE_WITH_EXTRA_CXXFLAGS "" CACHE STRING "Define extra CXX compile flags") +set(HYPRE_SYCL_TARGET "" CACHE STRING "Target SYCL architecture, e.g. 'spir64_gen'.") +set(HYPRE_SYCL_TARGET_BACKEND "" CACHE STRING "Additional SYCL backend options, e.g. '-device 12.1.0,12.4.0'.") # Set config name values if (HYPRE_ENABLE_SHARED) @@ -354,6 +355,13 @@ endif () # Create the HYPRE library object add_library(${PROJECT_NAME}) +# Set output directory for the library +set_target_properties(HYPRE PROPERTIES + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" +) + # Headers and sources set(HYPRE_HEADERS "") @@ -419,10 +427,6 @@ if (HYPRE_USING_SUPERLU) target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_SUPERLU_INCLUDE_DIRS}) endif (HYPRE_USING_SUPERLU) -if (SUPERLU_FOUND) - set(HYPRE_USING_SUPERLU TRUE) -endif () - # Find DSUPERLU, if requested if (HYPRE_USING_DSUPERLU) if (NOT TPL_DSUPERLU_LIBRARIES) @@ -445,10 +449,6 @@ if (HYPRE_USING_DSUPERLU) target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_DSUPERLU_INCLUDE_DIRS}) endif (HYPRE_USING_DSUPERLU) -if (DSUPERLU_FOUND) - set(HYPRE_USING_DSUPERLU TRUE) -endif () - # TODO: Check for case if MAGMA without CUDA/HIP # Find MAGMA, if requested if (HYPRE_USING_MAGMA) @@ -491,14 +491,7 @@ if (HYPRE_USING_FEI) target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_FEI_INCLUDE_DIRS}) endif(HYPRE_USING_FEI) -if (HYPRE_USING_CUDA) - target_link_libraries(${PROJECT_NAME} PUBLIC "${EXPORT_INTERFACE_CUDA_LIBS}") - if (HYPRE_HAVE_MPI) - target_include_directories(${PROJECT_NAME} PUBLIC - ${MPI_CXX_INCLUDE_DIRS}) - endif () -endif () - +# Find Caliper, if requested if (HYPRE_USING_CALIPER) if (NOT TPL_CALIPER_LIBRARIES OR NOT TPL_CALIPER_INCLUDE_DIRS) message(FATAL_ERROR "Both TPL_CALIPER_LIBRARIES and TPL_CALIPER_INCLUDE_DIRS options must be set for Caliper support.") @@ -547,16 +540,23 @@ foreach (DIR IN LISTS HYPRE_DIRS) endforeach () # BINARY must be first in order to get the correct HYPRE_config.h file +#target_include_directories(${PROJECT_NAME} PUBLIC +# $ +# $ +# $ +# $ +# $ +# ) + +# Include directories target_include_directories(${PROJECT_NAME} PUBLIC - $ - $ - $ - $ - $ - ) + $ + $ + $ +) # Set MPI compile flags -if (NOT HYPRE_SEQUENTIAL) +if (HYPRE_WITH_MPI) find_program(MPIEXEC_EXECUTABLE NAMES mpiexec mpirun) find_package(MPI REQUIRED) target_link_libraries(${PROJECT_NAME} PUBLIC MPI::MPI_C) @@ -579,7 +579,7 @@ if (NOT HYPRE_SEQUENTIAL) else() message(WARNING "MPI include directory not found. Compilation may fail.") endif() -endif (NOT HYPRE_SEQUENTIAL) +endif () # Set OpenMP compile flags if (HYPRE_USING_OPENMP) @@ -589,14 +589,20 @@ endif (HYPRE_USING_OPENMP) if (HYPRE_WITH_CUDA) set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE CUDA) + target_link_libraries(${PROJECT_NAME} PUBLIC ${EXPORT_INTERFACE_CUDA_LIBS}) elseif (HYPRE_WITH_HIP) set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE HIP) target_link_libraries(${PROJECT_NAME} PUBLIC ${EXPORT_INTERFACE_HIP_LIBS}) - target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIRS}) # Ensure MPI headers are included for HIP compilation + target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIRS}) elseif (HYPRE_WITH_SYCL) set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE SYCL) endif () +# Ensure MPI headers are included for device compilation +if (HYPRE_HAVE_MPI AND HYPRE_USING_GPU) + target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIRS}) +endif () + if (MSVC) target_compile_definitions(${PROJECT_NAME} PRIVATE _CRT_SECURE_NO_WARNINGS) if (MSVC_VERSION LESS 1928) # Visual Studio 2019 version 16.8 claims full C11 support @@ -637,14 +643,15 @@ endif () if (HYPRE_BUILD_TESTS) add_subdirectory(test) endif () - +# Installation include(GNUInstallDirs) -install(TARGETS ${PROJECT_NAME} +install(TARGETS HYPRE EXPORT HYPRETargets - LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" - ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" - RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" - INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) install(FILES ${HYPRE_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") include(CMakePackageConfigHelpers) @@ -652,32 +659,54 @@ write_basic_package_version_file( HYPREConfigVersion.cmake VERSION ${PACKAGE_VERSION} COMPATIBILITY SameMajorVersion - ) +) +# Install the CMake export file install(EXPORT HYPRETargets FILE HYPRETargets.cmake NAMESPACE HYPRE:: - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE" - ) + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE +) configure_package_config_file( config/HYPREConfig.cmake.in HYPREConfig.cmake - INSTALL_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" - ) -install( - FILES - "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfigVersion.cmake" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE" - ) + INSTALL_DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake +) +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/HYPREConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE +) + +# Export from build tree export(EXPORT HYPRETargets FILE "${CMAKE_CURRENT_BINARY_DIR}/HYPRETargets.cmake" NAMESPACE HYPRE:: - ) +) # Declare an alias so that consumers can depend on HYPRE::HYPRE target # also when using HYPRE via add_directory or FetchContent add_library(HYPRE::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) +# Export the package export(PACKAGE ${PROJECT_NAME}) + +# Print the status of the options for debugging purposes +print_option_status( + HYPRE_ENABLE_SHARED + HYPRE_ENABLE_BIGINT + HYPRE_ENABLE_MIXEDINT + HYPRE_ENABLE_SINGLE + HYPRE_ENABLE_LONG_DOUBLE + HYPRE_ENABLE_COMPLEX + HYPRE_ENABLE_HYPRE_BLAS + HYPRE_ENABLE_HYPRE_LAPACK + HYPRE_ENABLE_PERSISTENT_COMM + HYPRE_ENABLE_FEI + HYPRE_WITH_MPI + HYPRE_WITH_OPENMP + HYPRE_WITH_CUDA + HYPRE_WITH_HIP + HYPRE_WITH_SYCL +) diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index a68acd1473..1b14e101b7 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -47,3 +47,58 @@ function(add_hypre_executables EXE_SRCS) target_link_libraries(${EXE_NAME} PRIVATE "${HYPRE_LIBS}") endforeach(SRC_FILE) endfunction() + +function(print_option_status) + # Define column widths + set(COLUMN1_WIDTH 40) + set(COLUMN2_WIDTH 10) + math(EXPR HEADER1_PAD "${COLUMN1_WIDTH} - 3") + math(EXPR HEADER2_PAD "${COLUMN2_WIDTH} - 1") + + # Create separator line + string(REPEAT "-" ${HEADER1_PAD} SEPARATOR1) + string(REPEAT "-" ${HEADER2_PAD} SEPARATOR2) + set(separator "+${SEPARATOR1}+${SEPARATOR2}+") + + # Create header and separator + message(STATUS "") + message(STATUS "HYPRE Configuration Summary:") + message(STATUS "${separator}") + message(STATUS "| Option | Status |") + message(STATUS "${separator}") + + # Iterate through each option and display its status + foreach(opt ${ARGN}) + # Determine the status string + if(${${opt}}) + set(status "ON") + else() + set(status "OFF") + endif() + + # Calculate padding for the option name + string(LENGTH "${opt}" opt_length) + math(EXPR padding "${COLUMN1_WIDTH} - ${opt_length} - 5") # 5 accounts for "| Option | Status |" + if(${padding} GREATER 0) + string(REPEAT " " ${padding} pad_spaces) + else() + set(pad_spaces "") + endif() + + # Calculate padding for the status + string(LENGTH "${status}" status_length) + math(EXPR status_padding "${COLUMN2_WIDTH} - ${status_length} - 3") # 3 accounts for "| " and space + if(${status_padding} GREATER 0) + string(REPEAT " " ${status_padding} status_pad_spaces) + else() + set(status_pad_spaces "") + endif() + + # Print the formatted row + message(STATUS "| ${opt}${pad_spaces} | ${status}${status_pad_spaces} |") + endforeach() + + # Print the footer separator + message(STATUS "${separator}") + message(STATUS "") +endfunction() \ No newline at end of file diff --git a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake index 3f79ff7812..c0f307cf87 100644 --- a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake @@ -7,6 +7,8 @@ if(HYPRE_WITH_CUDA) include(HYPRE_SetupCUDAToolkit) elseif(HYPRE_WITH_HIP) include(HYPRE_SetupHIPToolkit) +elseif(HYPRE_WITH_SYCL) + include(HYPRE_SetupSYCLToolkit) else() - message(FATAL_ERROR "Neither CUDA nor HIP is enabled. Please enable one of them.") + message(FATAL_ERROR "Neither CUDA nor HIP nor SYCL is enabled. Please enable one of them.") endif() \ No newline at end of file From 86900171131179b1a617ff9bee60744b2f5957e5 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Sep 2024 13:52:03 -0400 Subject: [PATCH 05/55] Change to modern CMake practice --- src/CMakeLists.txt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 164e4a6dcb..9041a0d09c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -419,7 +419,7 @@ if (HYPRE_USING_SUPERLU) if (NOT EXISTS ${dir}) message(FATAL_ERROR "SuperLU include directory not found: ${dir}") endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") + target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) endforeach () message(STATUS "Enabled support for using SUPERLU.") set(SUPERLU_FOUND TRUE) @@ -441,7 +441,7 @@ if (HYPRE_USING_DSUPERLU) if (NOT EXISTS ${dir}) message(FATAL_ERROR "SuperLU_Dist include directory not found: ${dir}") endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") + target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) endforeach () message(STATUS "Enabled support for using DSUPERLU.") set(DSUPERLU_FOUND TRUE) @@ -464,7 +464,7 @@ if (HYPRE_USING_MAGMA) if (NOT EXISTS ${dir}) message(FATAL_ERROR "MAGMA include directory not found: ${dir}") endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") + target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) endforeach () message(STATUS "Enabled support for using MAGMA.") target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_MAGMA_LIBRARIES} stdc++) @@ -483,8 +483,7 @@ if (HYPRE_USING_FEI) if (NOT EXISTS ${dir}) message(FATAL_ERROR "FEI include directory not found: ${dir}") endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") - set(CMAKE_CXX_FLAGS "-I${dir} ${CMAKE_CXX_FLAGS}") + target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) endforeach () message(STATUS "Enabled support for using FEI.") set(FEI_FOUND TRUE) @@ -501,7 +500,7 @@ if (HYPRE_USING_CALIPER) if (NOT EXISTS ${dir}) message(FATAL_ERROR "Caliper include directory not found: ${dir}") endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") + target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) endforeach () message(STATUS "Enabled support for using Caliper.") target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_CALIPER_LIBRARIES}) @@ -518,7 +517,7 @@ if (HYPRE_USING_UMPIRE) if (NOT EXISTS ${dir}) message(FATAL_ERROR "Umpire include directory not found: ${dir}") endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") + target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) endforeach () message(STATUS "Enabled support for using Umpire.") target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_UMPIRE_LIBRARIES}) From 31b0d971d1866bc4b990e562b36294a8679f02a4 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Sep 2024 13:52:20 -0400 Subject: [PATCH 06/55] Remove unused variable --- src/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9041a0d09c..cfd2c6631f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -556,7 +556,6 @@ target_include_directories(${PROJECT_NAME} PUBLIC # Set MPI compile flags if (HYPRE_WITH_MPI) - find_program(MPIEXEC_EXECUTABLE NAMES mpiexec mpirun) find_package(MPI REQUIRED) target_link_libraries(${PROJECT_NAME} PUBLIC MPI::MPI_C) From f7c411fcce6e459f82e8496de9900a0c2412874b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Sep 2024 14:54:14 -0400 Subject: [PATCH 07/55] Add configure_mpi_target function --- src/CMakeLists.txt | 51 +++++++-------------- src/config/cmake/HYPRE_CMakeUtilities.cmake | 24 ++++++++++ 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cfd2c6631f..266a5f180d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -30,6 +30,9 @@ project(${PROJECT_NAME} VERSION ${HYPRE_VERSION} LANGUAGES C) +# Create the HYPRE library object +add_library(${PROJECT_NAME}) + # We use C99 by default, but users are free to specify any newer standard version set(CMAKE_C_STANDARD 99) @@ -275,6 +278,18 @@ if (HYPRE_WITH_UMPIRE_PINNED) set(HYPRE_USING_UMPIRE_PINNED ON CACHE BOOL "" FORCE) endif () +# Set MPI compile flags +if (HYPRE_WITH_MPI) + configure_mpi_target() +endif () + +# Set OpenMP compile flags +if (HYPRE_USING_OPENMP) + find_package(OpenMP REQUIRED) + target_link_libraries(${PROJECT_NAME} PUBLIC OpenMP::OpenMP_C) +endif (HYPRE_USING_OPENMP) + +# Set GPU compile flags if (HYPRE_WITH_CUDA OR HYPRE_WITH_HIP OR HYPRE_WITH_SYCL) enable_language(CXX) message(STATUS "Enabled support for CXX.") @@ -352,9 +367,6 @@ else () set(BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE) endif () -# Create the HYPRE library object -add_library(${PROJECT_NAME}) - # Set output directory for the library set_target_properties(HYPRE PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" @@ -551,40 +563,11 @@ endforeach () target_include_directories(${PROJECT_NAME} PUBLIC $ $ + $ + $ $ ) -# Set MPI compile flags -if (HYPRE_WITH_MPI) - find_package(MPI REQUIRED) - target_link_libraries(${PROJECT_NAME} PUBLIC MPI::MPI_C) - - # Determine the correct MPI include directory - if(MPI_CXX_INCLUDE_DIRS) - set(MPI_INCLUDE_DIRS ${MPI_CXX_INCLUDE_DIRS}) - elseif(MPI_CXX_INCLUDE_PATH) - set(MPI_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH}) - elseif(MPI_CXX_COMPILER_INCLUDE_DIRS) - set(MPI_INCLUDE_DIRS ${MPI_CXX_COMPILER_INCLUDE_DIRS}) - elseif(MPI_C_COMPILER_INCLUDE_DIRS) - set(MPI_INCLUDE_DIRS ${MPI_C_COMPILER_INCLUDE_DIRS}) - elseif(MPI_C_INCLUDE_DIRS) - set(MPI_INCLUDE_DIRS ${MPI_C_INCLUDE_DIRS}) - elseif(MPI_C_INCLUDE_PATH) - set(MPI_INCLUDE_DIRS ${MPI_C_INCLUDE_PATH}) - elseif(MPI_INCLUDE_PATH) - set(MPI_INCLUDE_DIRS ${MPI_INCLUDE_PATH}) - else() - message(WARNING "MPI include directory not found. Compilation may fail.") - endif() -endif () - -# Set OpenMP compile flags -if (HYPRE_USING_OPENMP) - find_package(OpenMP REQUIRED) - target_link_libraries(${PROJECT_NAME} PUBLIC OpenMP::OpenMP_C) -endif (HYPRE_USING_OPENMP) - if (HYPRE_WITH_CUDA) set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE CUDA) target_link_libraries(${PROJECT_NAME} PUBLIC ${EXPORT_INTERFACE_CUDA_LIBS}) diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index 1b14e101b7..79fcbea89e 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -3,6 +3,30 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +function(configure_mpi_target) + find_package(MPI REQUIRED) + target_link_libraries(${PROJECT_NAME} PUBLIC MPI::MPI_C) + + # Determine the correct MPI include directory + if(MPI_CXX_INCLUDE_DIRS) + set(MPI_INCLUDE_DIRS ${MPI_CXX_INCLUDE_DIRS}) + elseif(MPI_CXX_INCLUDE_PATH) + set(MPI_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH}) + elseif(MPI_CXX_COMPILER_INCLUDE_DIRS) + set(MPI_INCLUDE_DIRS ${MPI_CXX_COMPILER_INCLUDE_DIRS}) + elseif(MPI_C_COMPILER_INCLUDE_DIRS) + set(MPI_INCLUDE_DIRS ${MPI_C_COMPILER_INCLUDE_DIRS}) + elseif(MPI_C_INCLUDE_DIRS) + set(MPI_INCLUDE_DIRS ${MPI_C_INCLUDE_DIRS}) + elseif(MPI_C_INCLUDE_PATH) + set(MPI_INCLUDE_DIRS ${MPI_C_INCLUDE_PATH}) + elseif(MPI_INCLUDE_PATH) + set(MPI_INCLUDE_DIRS ${MPI_INCLUDE_PATH}) + else() + message(WARNING "MPI include directory not found. Compilation may fail.") + endif() +endfunction() + # A handy function to add the current source directory to a local # filename. To be used for creating a list of sources. function(convert_filenames_to_full_paths NAMES) From 25491aef0fb514b37dd8eb840f211be29cb12e05 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Sep 2024 22:28:13 -0400 Subject: [PATCH 08/55] Modernizing CUDA cmake build --- src/CMakeLists.txt | 80 +++---- src/config/cmake/HYPRE_CMakeUtilities.cmake | 13 ++ src/config/cmake/HYPRE_SetupCUDAToolkit.cmake | 214 ++++++++---------- src/config/cmake/HYPRE_SetupGPUToolkit.cmake | 24 ++ 4 files changed, 163 insertions(+), 168 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 266a5f180d..a084ab1376 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,19 +3,11 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -cmake_minimum_required(VERSION 3.13...3.16) +cmake_minimum_required(VERSION 3.21) -if (${CMAKE_VERSION} VERSION_LESS 3.16) - cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) -else () - cmake_policy(VERSION 3.16) -endif () - -# Set CMP0146 policy to NEW. This policy setting ensures we use the -# modern CUDA language support instead of the deprecated FindCUDA module -if(POLICY CMP0146) - cmake_policy(SET CMP0146 NEW) -endif() +# Set cmake policies +cmake_policy(SET CMP0146 NEW) # Ensure CUDA toolkit is found +cmake_policy(SET CMP0104 NEW) # CUDA_ARCHITECTURES is allowed to be set in cache # The version number. set(HYPRE_VERSION 2.31.0) @@ -129,7 +121,6 @@ option(HYPRE_ENABLE_DEVICE_POOL "Use device memory pool" OFF) option(HYPRE_ENABLE_CUBLAS "Use cuBLAS" OFF) option(HYPRE_ENABLE_CURAND "Use cuRAND" ON) option(HYPRE_ENABLE_GPU_PROFILING "Use NVTX on CUDA" OFF) -set(HYPRE_CUDA_SM "70" CACHE STRING "Target CUDA architecture.") # HIP options option(HYPRE_ENABLE_ROCBLAS "Use rocBLAS" ON) option(HYPRE_ENABLE_ROCSPARSE "Use rocSPARSE" ON) @@ -145,7 +136,7 @@ option(HYPRE_WITH_UMPIRE_HOST "Use Umpire Allocator for host memory" OFF) option(HYPRE_WITH_UMPIRE_DEVICE "Use Umpire Allocator for device memory" OFF) option(HYPRE_WITH_UMPIRE_UM "Use Umpire Allocator for unified memory" OFF) option(HYPRE_WITH_UMPIRE_PINNED "Use Umpire Allocator for pinned memory" OFF) -# TPL options +# TPL options option(TPL_UMPIRE_LIBRARIES "List of absolute paths to Umpire link libraries [].") option(TPL_UMPIRE_INCLUDE_DIRS "List of absolute paths to Umpire include directories [].") option(TPL_SUPERLU_LIBRARIES "List of absolute paths to SuperLU link libraries [].") @@ -219,6 +210,18 @@ if (HYPRE_ENABLE_HOPSCOTCH) set(HYPRE_HOPSCOTCH ON CACHE BOOL "" FORCE) endif () +if (HYPRE_WITH_CUDA) + set(HYPRE_USING_CUDA ON CACHE BOOL "" FORCE) +endif () + +if (HYPRE_WITH_HIP) + set(HYPRE_USING_HIP ON CACHE BOOL "" FORCE) +endif () + +if (HYPRE_WITH_SYCL) + set(HYPRE_USING_SYCL ON CACHE BOOL "" FORCE) +endif () + if (HYPRE_WITH_SUPERLU) set(HYPRE_USING_SUPERLU ON CACHE BOOL "" FORCE) add_compile_definitions(HAVE_SUPERLU) @@ -295,39 +298,22 @@ if (HYPRE_WITH_CUDA OR HYPRE_WITH_HIP OR HYPRE_WITH_SYCL) message(STATUS "Enabled support for CXX.") set(HYPRE_USING_GPU ON CACHE BOOL "" FORCE) + include(HYPRE_SetupGPUToolkit) - if (HYPRE_WITH_CUDA) - set(HYPRE_USING_CUDA TRUE) - enable_language(CUDA) - include(HYPRE_SetupCUDAToolkit) - - # Enforce C++11 - if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 11) - set(CMAKE_CXX_STANDARD 11) - endif () - elseif (HYPRE_WITH_HIP) - set(HYPRE_USING_HIP TRUE) - enable_language(HIP) - include(HYPRE_SetupHIPToolkit) - - # Enforce C++14 - if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) - set(CMAKE_CXX_STANDARD 14) - endif () - elseif (HYPRE_WITH_SYCL) - set(HYPRE_USING_SYCL TRUE) - enable_language(SYCL) - include(HYPRE_SetupSYCLToolkit) - - # Enforce C++14 - if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) - set(CMAKE_CXX_STANDARD 14) - endif () - endif () + # Link device libraries + message(STATUS "Exported device libraries: ${EXPORT_DEVICE_LIBS}") + #target_link_libraries(${PROJECT_NAME} PUBLIC ${EXPORT_DEVICE_LIBS}) set(CMAKE_CXX_STANDARD_REQUIRED ON) message(STATUS "Using CXX standard: c++${CMAKE_CXX_STANDARD}") + # Check if examples are enabled, but not unified memory + if (HYPRE_BUILD_EXAMPLES AND NOT HYPRE_ENABLE_UNIFIED_MEMORY) + message(WARNING "Running the examples on GPUs requires Unified Memory! + Examples will not be built!") + set(HYPRE_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + endif () + # Add any extra CXX compiler flags HYPRE_WITH_EXTRA_CXXFLAGS if (NOT HYPRE_WITH_EXTRA_CXXFLAGS STREQUAL "") string(REPLACE " " ";" HYPRE_WITH_EXTRA_CXXFLAGS_LIST ${HYPRE_WITH_EXTRA_CXXFLAGS}) @@ -570,19 +556,17 @@ target_include_directories(${PROJECT_NAME} PUBLIC if (HYPRE_WITH_CUDA) set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE CUDA) - target_link_libraries(${PROJECT_NAME} PUBLIC ${EXPORT_INTERFACE_CUDA_LIBS}) elseif (HYPRE_WITH_HIP) set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE HIP) - target_link_libraries(${PROJECT_NAME} PUBLIC ${EXPORT_INTERFACE_HIP_LIBS}) - target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIRS}) elseif (HYPRE_WITH_SYCL) set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE SYCL) endif () # Ensure MPI headers are included for device compilation -if (HYPRE_HAVE_MPI AND HYPRE_USING_GPU) - target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIRS}) -endif () +#if (HYPRE_WITH_MPI AND HYPRE_USING_GPU) +# message(STATUS "Adding MPI include directory: ${MPI_INCLUDE_DIRS}") +# target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIRS}) +#endif () if (MSVC) target_compile_definitions(${PROJECT_NAME} PRIVATE _CRT_SECURE_NO_WARNINGS) diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index 79fcbea89e..0dd95b82fc 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -25,6 +25,11 @@ function(configure_mpi_target) else() message(WARNING "MPI include directory not found. Compilation may fail.") endif() + + if (HYPRE_WITH_CUDA OR HYPRE_WITH_HIP OR HYPRE_WITH_SYCL) + message(STATUS "Adding MPI include directory: ${MPI_INCLUDE_DIRS}") + target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIRS}) + endif () endfunction() # A handy function to add the current source directory to a local @@ -37,6 +42,14 @@ function(convert_filenames_to_full_paths NAMES) set(${NAMES} ${tmp_names} PARENT_SCOPE) endfunction() +# A function to add hypre subdirectories to the build +function(add_hypre_subdirectories DIRS) + foreach(DIR IN LISTS DIRS) + add_subdirectory(${DIR}) + target_include_directories(${PROJECT_NAME} PRIVATE $) + endforeach() +endfunction() + # A function to add each executable in the list to the build with the # correct flags, includes, and linkage. function(add_hypre_executables EXE_SRCS) diff --git a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake index 1d1d03dbe6..13fa4f4239 100644 --- a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake @@ -3,135 +3,109 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -# This handles the non-compiler aspect of the CUDA toolkit. -# Uses cmake find_package to locate the NVIDIA CUDA C tools -# for shared libraries. Otherwise for static libraries, assumes -# the libraries are located in ${CUDA_TOOLKIT_ROOT_DIR}/lib64. -# Please set cmake variable CUDA_TOOLKIT_ROOT_DIR. - # Collection of CUDA optional libraries -set(EXPORT_INTERFACE_CUDA_LIBS "") +set(CUDA_LIBS "") -if (CMAKE_VERSION VERSION_LESS 3.17) - if (NOT CUDA_FOUND) - find_package(CUDA REQUIRED) - endif () +# Check if CUDA is available and enable it if found +include(CheckLanguage) +check_language(CUDA) +if(CMAKE_CUDA_COMPILER) + enable_language(CUDA) else() - if (NOT CUDAToolkit_FOUND) - find_package(CUDAToolkit REQUIRED) - endif () + message(FATAL_ERROR "CUDA language not found. Please check your CUDA installation.") endif() -if (CMAKE_VERSION VERSION_LESS 3.17) - if (HYPRE_ENABLE_CUSPARSE) - set(HYPRE_USING_CUSPARSE ON CACHE BOOL "" FORCE) - if (HYPRE_SHARED) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS ${CUDA_cusparse_LIBRARY}) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcusparse_static.a) - endif () - endif () - - if (HYPRE_ENABLE_CURAND) - set(HYPRE_USING_CURAND ON CACHE BOOL "" FORCE) - if (HYPRE_SHARED) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS ${CUDA_curand_LIBRARY}) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcurand_static.a) - endif () - endif () - - if (HYPRE_ENABLE_CUBLAS) - set(HYPRE_USING_CUBLAS ON CACHE BOOL "" FORCE) - if (HYPRE_SHARED) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS ${CUDA_CUBLAS_LIBRARIES}) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcublas_static.a) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcublasLt_static.a) - endif (HYPRE_SHARED) - endif (HYPRE_ENABLE_CUBLAS) - - if (HYPRE_ENABLE_CUSOLVER) - set(HYPRE_USING_CUSOLVER ON CACHE BOOL "" FORCE) - if (HYPRE_SHARED) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS ${CUDA_cusolver_LIBRARY}) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcusolver_static.a) - endif () - endif () - - if (NOT HYPRE_SHARED) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libculibos.a) - endif () - - if (HYPRE_ENABLE_GPU_PROFILING) - set(HYPRE_USING_NVTX ON CACHE BOOL "" FORCE) - find_library(NVTX_LIBRARY - NAME libnvToolsExt.so - PATHS ${CUDA_TOOLKIT_ROOT_DIR}/lib64 ${CUDA_TOOLKIT_ROOT_DIR}/lib) - message(STATUS "NVidia tools extension library found in " ${NVTX_LIBRARY}) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS ${NVTX_LIBRARY}) - endif (HYPRE_ENABLE_GPU_PROFILING) +# Find the CUDA Toolkit +find_package(CUDAToolkit REQUIRED) + +# Set CUDA standard to match C++ standard if not already set +if(NOT DEFINED CMAKE_CUDA_STANDARD) + set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD}) + set(CMAKE_CUDA_STANDARD_REQUIRED ON) + set(CMAKE_CUDA_EXTENSIONS OFF) +endif() +# Show CUDA Toolkit location +if(CUDAToolkit_FOUND) + if (CUDAToolkit_ROOT) + message(STATUS "CUDA Toolkit found at: ${CUDAToolkit_ROOT}") + endif() + message(STATUS "CUDA Toolkit version: ${CUDAToolkit_VERSION}") + message(STATUS "CUDA Toolkit include directory: ${CUDAToolkit_INCLUDE_DIRS}") + message(STATUS "CUDA Toolkit library directory: ${CUDAToolkit_LIBRARY_DIR}") else() + message(FATAL_ERROR "CUDA Toolkit not found") +endif() + +# Check for Thrust headers +find_path(THRUST_INCLUDE_DIR thrust/version.h + HINTS ${CUDAToolkit_INCLUDE_DIRS} ${CUDAToolkit_INCLUDE_DIRS}/cuda-thrust + PATH_SUFFIXES thrust + NO_DEFAULT_PATH +) - find_package(CUDAToolkit REQUIRED) +if(THRUST_INCLUDE_DIR) + message(STATUS "CUDA Thrust headers found in: ${THRUST_INCLUDE_DIR}") +else() + message(FATAL_ERROR "CUDA Thrust headers not found! Please check your CUDA installation.") +endif() - if (HYPRE_SHARED OR WIN32) - set(HYPRE_CUDA_TOOLKIT_STATIC FALSE) - else() - set(HYPRE_CUDA_TOOLKIT_STATIC TRUE) +# Function to handle CUDA libraries +function(find_and_add_cuda_library LIB_NAME HYPRE_ENABLE_VAR) + if(${HYPRE_ENABLE_VAR}) + string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) + set(HYPRE_USING_${LIB_NAME_UPPER} ON CACHE BOOL "" FORCE) + if(TARGET CUDAToolkit::${LIB_NAME}) + message(STATUS "CUDAToolkit::${LIB_NAME} target found") + list(APPEND CUDA_LIBS CUDAToolkit::${LIB_NAME}) + else() + message(WARNING "CUDAToolkit::${LIB_NAME} target not found. Attempting manual linking.") + find_library(${LIB_NAME}_LIBRARY ${LIB_NAME} HINTS ${CUDAToolkit_LIBRARY_DIR}) + if(${LIB_NAME}_LIBRARY) + message(STATUS "Found ${LIB_NAME} library: ${${LIB_NAME}_LIBRARY}") + add_library(CUDAToolkit::${LIB_NAME} UNKNOWN IMPORTED) + set_target_properties(CUDAToolkit::${LIB_NAME} PROPERTIES + IMPORTED_LOCATION "${${LIB_NAME}_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${CUDAToolkit_INCLUDE_DIRS}") + list(APPEND CUDA_LIBS CUDAToolkit::${LIB_NAME}) + else() + message(FATAL_ERROR "Could not find ${LIB_NAME} library. Please check your CUDA installation.") + endif() + endif() + set(CUDA_LIBS ${CUDA_LIBS} PARENT_SCOPE) endif() +endfunction() + +# Handle CUDA libraries +find_and_add_cuda_library(cusparse HYPRE_ENABLE_CUSPARSE) +find_and_add_cuda_library(curand HYPRE_ENABLE_CURAND) +find_and_add_cuda_library(cublas HYPRE_ENABLE_CUBLAS) +find_and_add_cuda_library(cublasLt HYPRE_ENABLE_CUBLAS) +find_and_add_cuda_library(cusolver HYPRE_ENABLE_CUSOLVER) + +# Handle GPU Profiling with nvToolsExt +if(HYPRE_ENABLE_GPU_PROFILING) + find_package(nvToolsExt REQUIRED) + set(HYPRE_USING_NVTX ON CACHE BOOL "" FORCE) + list(APPEND CUDA_LIBS CUDA::nvToolsExt) +endif() - if (HYPRE_ENABLE_CUSPARSE) - set(HYPRE_USING_CUSPARSE ON CACHE BOOL "" FORCE) - if (HYPRE_CUDA_TOOLKIT_STATIC) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cusparse_static) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cusparse) - endif () - endif () - - if (HYPRE_ENABLE_CURAND) - set(HYPRE_USING_CURAND ON CACHE BOOL "" FORCE) - if (HYPRE_CUDA_TOOLKIT_STATIC) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::curand_static) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::curand) - endif () - endif () - - if (HYPRE_ENABLE_CUBLAS) - set(HYPRE_USING_CUBLAS ON CACHE BOOL "" FORCE) - if (HYPRE_CUDA_TOOLKIT_STATIC) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cublas_static CUDA::cublasLt_static) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cublas CUDA::cublasLt) - endif (HYPRE_CUDA_TOOLKIT_STATIC) - endif (HYPRE_ENABLE_CUBLAS) - - if (HYPRE_ENABLE_CUSOLVER) - set(HYPRE_USING_CUSOLVER ON CACHE BOOL "" FORCE) - if (HYPRE_CUDA_TOOLKIT_STATIC) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cusolver_static) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cusolver) - endif () - endif () - - if (HYPRE_CUDA_TOOLKIT_STATIC) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::culibos) - endif (HYPRE_CUDA_TOOLKIT_STATIC) - - if (HYPRE_ENABLE_GPU_PROFILING) - set(HYPRE_USING_NVTX ON CACHE BOOL "" FORCE) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::nvToolsExt) - endif (HYPRE_ENABLE_GPU_PROFILING) +# Return the list of CUDA libraries +set(EXPORT_INTERFACE_CUDA_LIBS ${CUDA_LIBS}) -endif() +# Add CUDA Toolkit include directories to the target +target_include_directories(HYPRE PUBLIC ${CUDAToolkit_INCLUDE_DIRS}) + +# Add Thrust include directory +target_include_directories(HYPRE PUBLIC ${THRUST_INCLUDE_DIR}) + +# Link CUDA libraries to the target +target_link_libraries(HYPRE PUBLIC ${CUDA_LIBS}) + +# Set CUDA flags +set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -ccbin=${CMAKE_CXX_COMPILER} -expt-extended-lambda") + +# Print CUDA info +message(STATUS "CUDA Standard: ${CMAKE_CUDA_STANDARD}") +message(STATUS "CUDA FLAGS: ${CMAKE_CUDA_FLAGS}") +message(STATUS "NVCC FLAGS: ${CUDA_NVCC_FLAGS}") diff --git a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake index c0f307cf87..6656785591 100644 --- a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake @@ -4,11 +4,35 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) if(HYPRE_WITH_CUDA) + # Enforce C++11 at least + if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 11) + set(CMAKE_CXX_STANDARD 11) + endif () include(HYPRE_SetupCUDAToolkit) + elseif(HYPRE_WITH_HIP) + # Enforce C++14 at least + if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) + set(CMAKE_CXX_STANDARD 14) + endif () + + message(STATUS "Enabling HIP toolkit") + set(HYPRE_USING_HIP ON) + enable_language(HIP) include(HYPRE_SetupHIPToolkit) + set(EXPORT_DEVICE_LIBS ${EXPORT_INTERFACE_HIP_LIBS}) + elseif(HYPRE_WITH_SYCL) + # Enforce C++14 at least + if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) + set(CMAKE_CXX_STANDARD 14) + endif () + + message(STATUS "Enabling SYCL toolkit") + enable_language(SYCL) include(HYPRE_SetupSYCLToolkit) + set(EXPORT_DEVICE_LIBS ${EXPORT_INTERFACE_SYCL_LIBS}) + else() message(FATAL_ERROR "Neither CUDA nor HIP nor SYCL is enabled. Please enable one of them.") endif() \ No newline at end of file From 1fda469a6bd261867a772bdd874eb589f24ba33a Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 14 Sep 2024 23:38:37 -0400 Subject: [PATCH 09/55] Add missing HYPRE_CUDA/HIP_CALL macros --- src/seq_mv/csr_spgemm_device_numblocks.c | 22 ++++++++++++---------- src/seq_mv/csr_spgemm_device_numer.h | 8 ++++---- src/seq_mv/csr_spgemm_device_symbl.h | 8 ++++---- src/utilities/device_utils.c | 8 ++++---- src/utilities/general.c | 16 +++++++++++++--- src/utilities/memory.c | 2 +- 6 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/seq_mv/csr_spgemm_device_numblocks.c b/src/seq_mv/csr_spgemm_device_numblocks.c index 183db2c910..9f2610f7a4 100644 --- a/src/seq_mv/csr_spgemm_device_numblocks.c +++ b/src/seq_mv/csr_spgemm_device_numblocks.c @@ -20,16 +20,18 @@ HYPRE_Int hypreDevice_CSRSpGemmBinnedGetBlockNumDim() hypre_HandleSpgemmNumBin(hypre_handle()) = num_bins; #if defined(HYPRE_USING_CUDA) - cudaDeviceGetAttribute(&multiProcessorCount, cudaDevAttrMultiProcessorCount, - hypre_HandleDevice(hypre_handle())); -#endif - -#if defined(HYPRE_USING_HIP) - hipDeviceGetAttribute(&multiProcessorCount, hipDeviceAttributeMultiprocessorCount, - hypre_HandleDevice(hypre_handle())); -#endif - -#if defined(HYPRE_USING_SYCL) + HYPRE_CUDA_CALL(cudaDeviceGetAttribute( + &multiProcessorCount, + cudaDevAttrMultiProcessorCount, + hypre_HandleDevice(hypre_handle()))); + +#elif defined(HYPRE_USING_HIP) + HYPRE_HIP_CALL(hipDeviceGetAttribute( + &multiProcessorCount, + hipDeviceAttributeMultiprocessorCount, + hypre_HandleDevice(hypre_handle()))); + +#elif defined(HYPRE_USING_SYCL) /* WM: todo - is this right? */ multiProcessorCount = hypre_HandleDevice( hypre_handle())->get_info(); diff --git a/src/seq_mv/csr_spgemm_device_numer.h b/src/seq_mv/csr_spgemm_device_numer.h index 4fb2f3d6d3..a75ac4383f 100644 --- a/src/seq_mv/csr_spgemm_device_numer.h +++ b/src/seq_mv/csr_spgemm_device_numer.h @@ -853,17 +853,17 @@ HYPRE_Int hypre_spgemm_numerical_max_num_blocks( HYPRE_Int multiProcessorCount, #endif #if defined(HYPRE_USING_CUDA) - cudaOccupancyMaxActiveBlocksPerMultiprocessor( + HYPRE_CUDA_CALL(cudaOccupancyMaxActiveBlocksPerMultiprocessor( &numBlocksPerSm, hypre_spgemm_numeric, - block_size, dynamic_shmem_size); + block_size, dynamic_shmem_size)); #endif #if defined(HYPRE_USING_HIP) - hipOccupancyMaxActiveBlocksPerMultiprocessor( + HYPRE_HIP_CALL(hipOccupancyMaxActiveBlocksPerMultiprocessor( &numBlocksPerSm, hypre_spgemm_numeric, - block_size, dynamic_shmem_size); + block_size, dynamic_shmem_size)); #endif #if defined(HYPRE_USING_SYCL) diff --git a/src/seq_mv/csr_spgemm_device_symbl.h b/src/seq_mv/csr_spgemm_device_symbl.h index 1bafc17ef9..d08ab6570f 100644 --- a/src/seq_mv/csr_spgemm_device_symbl.h +++ b/src/seq_mv/csr_spgemm_device_symbl.h @@ -600,17 +600,17 @@ HYPRE_Int hypre_spgemm_symbolic_max_num_blocks( HYPRE_Int multiProcessorCount, #endif #if defined(HYPRE_USING_CUDA) - cudaOccupancyMaxActiveBlocksPerMultiprocessor( + HYPRE_CUDA_CALL(cudaOccupancyMaxActiveBlocksPerMultiprocessor( &numBlocksPerSm, hypre_spgemm_symbolic, - block_size, dynamic_shmem_size); + block_size, dynamic_shmem_size)); #endif #if defined(HYPRE_USING_HIP) - hipOccupancyMaxActiveBlocksPerMultiprocessor( + HYPRE_HIP_CALL(hipOccupancyMaxActiveBlocksPerMultiprocessor( &numBlocksPerSm, hypre_spgemm_symbolic, - block_size, dynamic_shmem_size); + block_size, dynamic_shmem_size)); #endif #if defined(HYPRE_USING_SYCL) diff --git a/src/utilities/device_utils.c b/src/utilities/device_utils.c index 499fea8688..7abf86341a 100644 --- a/src/utilities/device_utils.c +++ b/src/utilities/device_utils.c @@ -202,9 +202,10 @@ HYPRE_Int hypre_ResetCudaDevice(hypre_Handle *hypre_handle) { #if defined(HYPRE_USING_CUDA) - cudaDeviceReset(); + HYPRE_CUDA_CALL(cudaDeviceReset()); + #elif defined(HYPRE_USING_HIP) - hipDeviceReset(); + HYPRE_HIP_CALL(hipDeviceReset()); #endif return hypre_error_flag; } @@ -2674,7 +2675,7 @@ hypre_CudaCompileFlagCheck() hypre_GetDevice(&device); struct cudaDeviceProp props; - cudaGetDeviceProperties(&props, device); + HYPRE_CUDA_CALL(cudaGetDeviceProperties(&props, device)); hypre_int cuda_arch_actual = props.major * 100 + props.minor * 10; hypre_int cuda_arch_compile = -1; dim3 gDim(1, 1, 1), bDim(1, 1, 1); @@ -3025,4 +3026,3 @@ hypre_bind_device( HYPRE_Int myid, { return hypre_bind_device_id(-1, myid, nproc, comm); } - diff --git a/src/utilities/general.c b/src/utilities/general.c index 71535e51d4..ad01d93051 100644 --- a/src/utilities/general.c +++ b/src/utilities/general.c @@ -147,11 +147,21 @@ hypre_GetDeviceMaxShmemSize(hypre_int device_id, } #if defined(HYPRE_USING_CUDA) - cudaDeviceGetAttribute(&max_size, cudaDevAttrMaxSharedMemoryPerBlock, device_id); - cudaDeviceGetAttribute(&max_size_optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, device_id); + HYPRE_CUDA_CALL(cudaDeviceGetAttribute( + &max_size, + cudaDevAttrMaxSharedMemoryPerBlock, + device_id)); + + HYPRE_CUDA_CALL(cudaDeviceGetAttribute( + &max_size_optin, + cudaDevAttrMaxSharedMemoryPerBlockOptin, + device_id)); #elif defined(HYPRE_USING_HIP) - hipDeviceGetAttribute(&max_size, hipDeviceAttributeMaxSharedMemoryPerBlock, device_id); + HYPRE_HIP_CALL(hipDeviceGetAttribute( + &max_size, + hipDeviceAttributeMaxSharedMemoryPerBlock, + device_id)); #elif defined(HYPRE_USING_SYCL) auto device = *hypre_HandleDevice(hypre_handle()); diff --git a/src/utilities/memory.c b/src/utilities/memory.c index be27a7ab70..5569377072 100644 --- a/src/utilities/memory.c +++ b/src/utilities/memory.c @@ -1153,7 +1153,7 @@ hypre_GetPointerLocation(const void *ptr, hypre_MemoryLocation *memory_location) ierr = 1; /* clear the error */ - hipGetLastError(); + (void) hipGetLastError(); if (err == hipErrorInvalidValue) { From f936f14aee014cfc51a673ac3bff995ebaeb9f3e Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sun, 15 Sep 2024 11:53:02 -0400 Subject: [PATCH 10/55] Allow CUDA/HIP builds with specific toolkit locations --- src/CMakeLists.txt | 16 ++-- src/config/cmake/HYPRE_SetupCUDAToolkit.cmake | 55 +++++++++--- src/config/cmake/HYPRE_SetupGPUToolkit.cmake | 7 +- src/config/cmake/HYPRE_SetupHIPToolkit.cmake | 88 ++++++++++++------- 4 files changed, 113 insertions(+), 53 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a084ab1376..74c3e27efd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,10 +4,16 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) cmake_minimum_required(VERSION 3.21) +message(STATUS "CMake Version: ${CMAKE_VERSION}") +message(STATUS "CMake Executable: ${CMAKE_COMMAND}") # Set cmake policies -cmake_policy(SET CMP0146 NEW) # Ensure CUDA toolkit is found -cmake_policy(SET CMP0104 NEW) # CUDA_ARCHITECTURES is allowed to be set in cache +if(POLICY CMP0146) + cmake_policy(SET CMP0146 NEW) # Ensure CUDA toolkit is found +endif() +if(POLICY CMP0104) + cmake_policy(SET CMP0104 NEW) # CUDA_ARCHITECTURES is allowed to be set in cache +endif() # The version number. set(HYPRE_VERSION 2.31.0) @@ -297,13 +303,11 @@ if (HYPRE_WITH_CUDA OR HYPRE_WITH_HIP OR HYPRE_WITH_SYCL) enable_language(CXX) message(STATUS "Enabled support for CXX.") + # Setup GPU libraries and include directories set(HYPRE_USING_GPU ON CACHE BOOL "" FORCE) include(HYPRE_SetupGPUToolkit) - # Link device libraries - message(STATUS "Exported device libraries: ${EXPORT_DEVICE_LIBS}") - #target_link_libraries(${PROJECT_NAME} PUBLIC ${EXPORT_DEVICE_LIBS}) - + # CXX standard info set(CMAKE_CXX_STANDARD_REQUIRED ON) message(STATUS "Using CXX standard: c++${CMAKE_CXX_STANDARD}") diff --git a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake index 13fa4f4239..784ebb0b16 100644 --- a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake @@ -3,14 +3,44 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -# Collection of CUDA optional libraries -set(CUDA_LIBS "") +message(STATUS "Enabling CUDA toolkit") + +# Check for CUDA_PATH, CUDA_HOME or CUDA_DIR +if(DEFINED CUDA_DIR) + set(CUDA_DIR ${CUDA_DIR}) +elseif(DEFINED ENV{CUDA_DIR}) + set(CUDA_DIR $ENV{CUDA_DIR}) +elseif(DEFINED CUDA_PATH) + set(CUDA_DIR ${CUDA_PATH}) +elseif(DEFINED ENV{CUDA_PATH}) + set(CUDA_DIR $ENV{CUDA_PATH}) +elseif(DEFINED CUDA_HOME) + set(CUDA_DIR ${CUDA_HOME}) +elseif(DEFINED ENV{CUDA_HOME}) + set(CUDA_DIR $ENV{CUDA_HOME}) +elseif(EXISTS "/opt/cuda") + set(CUDA_DIR "/opt/cuda") +elseif(EXISTS "/usr/bin/nvcc") + set(CUDA_DIR "/usr") +else() + message(FATAL_ERROR "CUDA_PATH or CUDA_HOME not set. Please set one of them to point to your CUDA installation.") +endif() +message(STATUS "Using CUDA installation: ${CUDA_DIR}") + +# Specify the path to the custom nvcc compiler +set(CMAKE_CUDA_COMPILER "${CUDA_DIR}/bin/nvcc" CACHE FILEPATH "CUDA compiler") + +# Specify the CUDA Toolkit root directory +set(CUDAToolkit_ROOT "${CUDA_DIR}" CACHE PATH "Path to the CUDA toolkit") + +# Optionally, prioritize the custom CUDA path in CMAKE_PREFIX_PATH +list(APPEND CMAKE_PREFIX_PATH "${CUDA_DIR}") # Check if CUDA is available and enable it if found include(CheckLanguage) check_language(CUDA) if(CMAKE_CUDA_COMPILER) - enable_language(CUDA) + enable_language(CUDA) else() message(FATAL_ERROR "CUDA language not found. Please check your CUDA installation.") endif() @@ -50,16 +80,23 @@ else() message(FATAL_ERROR "CUDA Thrust headers not found! Please check your CUDA installation.") endif() +# Collection of CUDA optional libraries +set(CUDA_LIBS "") + # Function to handle CUDA libraries function(find_and_add_cuda_library LIB_NAME HYPRE_ENABLE_VAR) + string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) if(${HYPRE_ENABLE_VAR}) - string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) set(HYPRE_USING_${LIB_NAME_UPPER} ON CACHE BOOL "" FORCE) + + # Use CUDAToolkit to find the component + find_package(CUDAToolkit REQUIRED COMPONENTS ${LIB_NAME}) + if(TARGET CUDAToolkit::${LIB_NAME}) message(STATUS "CUDAToolkit::${LIB_NAME} target found") list(APPEND CUDA_LIBS CUDAToolkit::${LIB_NAME}) else() - message(WARNING "CUDAToolkit::${LIB_NAME} target not found. Attempting manual linking.") + #message(WARNING "CUDAToolkit::${LIB_NAME} target not found. Attempting manual linking.") find_library(${LIB_NAME}_LIBRARY ${LIB_NAME} HINTS ${CUDAToolkit_LIBRARY_DIR}) if(${LIB_NAME}_LIBRARY) message(STATUS "Found ${LIB_NAME} library: ${${LIB_NAME}_LIBRARY}") @@ -72,6 +109,7 @@ function(find_and_add_cuda_library LIB_NAME HYPRE_ENABLE_VAR) message(FATAL_ERROR "Could not find ${LIB_NAME} library. Please check your CUDA installation.") endif() endif() + set(CUDA_LIBS ${CUDA_LIBS} PARENT_SCOPE) endif() endfunction() @@ -90,10 +128,7 @@ if(HYPRE_ENABLE_GPU_PROFILING) list(APPEND CUDA_LIBS CUDA::nvToolsExt) endif() -# Return the list of CUDA libraries -set(EXPORT_INTERFACE_CUDA_LIBS ${CUDA_LIBS}) - -# Add CUDA Toolkit include directories to the target +# Add CUDA Toolkit include directories to the target target_include_directories(HYPRE PUBLIC ${CUDAToolkit_INCLUDE_DIRS}) # Add Thrust include directory @@ -101,6 +136,7 @@ target_include_directories(HYPRE PUBLIC ${THRUST_INCLUDE_DIR}) # Link CUDA libraries to the target target_link_libraries(HYPRE PUBLIC ${CUDA_LIBS}) +message(STATUS "Linking to CUDA libraries: ${CUDA_LIBS}") # Set CUDA flags set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -ccbin=${CMAKE_CXX_COMPILER} -expt-extended-lambda") @@ -108,4 +144,3 @@ set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -ccbin=${CMAKE_CXX_COMPILER} -expt-ext # Print CUDA info message(STATUS "CUDA Standard: ${CMAKE_CUDA_STANDARD}") message(STATUS "CUDA FLAGS: ${CMAKE_CUDA_FLAGS}") -message(STATUS "NVCC FLAGS: ${CUDA_NVCC_FLAGS}") diff --git a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake index 6656785591..5349527b2e 100644 --- a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake @@ -15,12 +15,7 @@ elseif(HYPRE_WITH_HIP) if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) set(CMAKE_CXX_STANDARD 14) endif () - - message(STATUS "Enabling HIP toolkit") - set(HYPRE_USING_HIP ON) - enable_language(HIP) include(HYPRE_SetupHIPToolkit) - set(EXPORT_DEVICE_LIBS ${EXPORT_INTERFACE_HIP_LIBS}) elseif(HYPRE_WITH_SYCL) # Enforce C++14 at least @@ -35,4 +30,4 @@ elseif(HYPRE_WITH_SYCL) else() message(FATAL_ERROR "Neither CUDA nor HIP nor SYCL is enabled. Please enable one of them.") -endif() \ No newline at end of file +endif() diff --git a/src/config/cmake/HYPRE_SetupHIPToolkit.cmake b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake index 8a1e9bd81e..b9c8842c00 100644 --- a/src/config/cmake/HYPRE_SetupHIPToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake @@ -10,52 +10,71 @@ # Please set environment variable ROCM_PATH or HIP_PATH. # Check for ROCM_PATH or HIP_PATH -if(DEFINED ENV{ROCM_PATH}) +message(STATUS "Enabling HIP toolkit") +if(DEFINED ROCM_PATH) + set(HIP_PATH ${ROCM_PATH}) +elseif(DEFINED ENV{ROCM_PATH}) set(HIP_PATH $ENV{ROCM_PATH}) elseif(DEFINED ENV{HIP_PATH}) set(HIP_PATH $ENV{HIP_PATH}) +elseif(EXISTS "/opt/rocm") + set(HIP_PATH "/opt/rocm") else() - message(FATAL_ERROR "Neither ROCM_PATH nor HIP_PATH environment variable is set. Please set one of them to point to your ROCm installation.") + message(FATAL_ERROR "ROCM_PATH or HIP_PATH not set. Please set one of them to point to your ROCm installation.") endif() +message(STATUS "Using ROCm installation: ${HIP_PATH}") # Add HIP_PATH to CMAKE_PREFIX_PATH list(APPEND CMAKE_PREFIX_PATH ${HIP_PATH}) +# Check if HIP is available and enable it if found +include(CheckLanguage) +check_language(HIP) +if(CMAKE_HIP_COMPILER) + enable_language(HIP) +else() + message(FATAL_ERROR "HIP language not found. Please check your HIP/ROCm installation.") +endif() + # Find HIP package find_package(hip REQUIRED CONFIG) -# Set HIP-specific variables -set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -fPIC" CACHE STRING "HIP compiler flags" FORCE) - -# Collection of HIP optional libraries -set(EXPORT_INTERFACE_HIP_LIBS "") - -if(HYPRE_SHARED OR WIN32) - set(HYPRE_HIP_TOOLKIT_STATIC FALSE) -else() - set(HYPRE_HIP_TOOLKIT_STATIC TRUE) -endif() +# Collection of ROCm optional libraries +set(ROCM_LIBS "") # Function to find and add libraries -function(find_and_add_hip_library LIB_NAME) +function(find_and_add_rocm_library LIB_NAME) string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) - if(HYPRE_ENABLE_${LIB_NAME_UPPER}) - find_package(${LIB_NAME} REQUIRED) + set(HYPRE_ENABLE_VAR "HYPRE_ENABLE_${LIB_NAME_UPPER}") + if(${HYPRE_ENABLE_VAR}) set(HYPRE_USING_${LIB_NAME_UPPER} ON CACHE BOOL "" FORCE) - if(HYPRE_HIP_TOOLKIT_STATIC) - list(APPEND EXPORT_INTERFACE_HIP_LIBS roc::${LIB_NAME}_static) + find_package(${LIB_NAME} REQUIRED) + if(TARGET roc::${LIB_NAME}) + message(STATUS "roc::${LIB_NAME} target found") + list(APPEND ROCM_LIBS roc::${LIB_NAME}) else() - list(APPEND EXPORT_INTERFACE_HIP_LIBS roc::${LIB_NAME}) + #message(WARNING "roc::${LIB_NAME} target not found. Attempting manual linking.") + find_library(${LIB_NAME}_LIBRARY ${LIB_NAME} HINTS ${HIP_PATH}/lib ${HIP_PATH}/lib64) + if(${LIB_NAME}_LIBRARY) + message(STATUS "Found ${LIB_NAME} library: ${${LIB_NAME}_LIBRARY}") + add_library(roc::${LIB_NAME} UNKNOWN IMPORTED) + set_target_properties(roc::${LIB_NAME} PROPERTIES + IMPORTED_LOCATION "${${LIB_NAME}_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${HIP_PATH}/include") + list(APPEND ROCM_LIBS roc::${LIB_NAME}) + else() + message(FATAL_ERROR "Could not find ${LIB_NAME} library. Please check your ROCm installation.") + endif() endif() - set(EXPORT_INTERFACE_HIP_LIBS ${EXPORT_INTERFACE_HIP_LIBS} PARENT_SCOPE) + set(ROCM_LIBS ${ROCM_LIBS} PARENT_SCOPE) endif() endfunction() # Find and add libraries -find_and_add_hip_library(rocblas) -find_and_add_hip_library(rocsparse) -find_and_add_hip_library(rocrand) -find_and_add_hip_library(rocsolver) +find_and_add_rocm_library(rocblas) +find_and_add_rocm_library(rocsparse) +find_and_add_rocm_library(rocrand) +find_and_add_rocm_library(rocsolver) if(HYPRE_ENABLE_GPU_PROFILING) set(HYPRE_USING_ROCTRACER ON CACHE BOOL "" FORCE) @@ -65,15 +84,22 @@ if(HYPRE_ENABLE_GPU_PROFILING) NO_DEFAULT_PATH) if(ROCTRACER_LIBRARY) message(STATUS "ROCm tracer library found in ${ROCTRACER_LIBRARY}") - list(APPEND EXPORT_INTERFACE_HIP_LIBS ${ROCTRACER_LIBRARY}) + list(APPEND ROCM_LIBS ${ROCTRACER_LIBRARY}) else() message(WARNING "ROCm tracer library not found. GPU profiling may not work correctly.") endif() endif() -# Make EXPORT_INTERFACE_HIP_LIBS available to parent scope -if(PARENT_SCOPE) - set(EXPORT_INTERFACE_HIP_LIBS ${EXPORT_INTERFACE_HIP_LIBS} PARENT_SCOPE) -else() - set(EXPORT_INTERFACE_HIP_LIBS ${EXPORT_INTERFACE_HIP_LIBS}) -endif() +# Add HIP include directory +target_include_directories(HYPRE PUBLIC ${HIP_PATH}/include) + +# Link HIP libraries to the target +target_link_libraries(HYPRE PUBLIC ${ROCM_LIBS}) +message(STATUS "Linking to ROCm libraries: ${ROCM_LIBS}") + +# Set HIP-specific variables +set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -fPIC" CACHE STRING "HIP compiler flags" FORCE) + +# Print HIP info +message(STATUS "HIP Standard: ${CMAKE_HIP_STANDARD}") +message(STATUS "HIP FLAGS: ${CMAKE_HIP_FLAGS}") From ee9f9d50640c2161e7fd387e89498679ff337431 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sun, 15 Sep 2024 18:32:15 -0700 Subject: [PATCH 11/55] Improve CUDA and HIP builds robustness --- src/CMakeLists.txt | 2 + src/config/cmake/HYPRE_CMakeUtilities.cmake | 55 +++++++++------ src/config/cmake/HYPRE_SetupCUDAToolkit.cmake | 60 ++++++++++++++-- src/config/cmake/HYPRE_SetupHIPToolkit.cmake | 70 ++++++++++++++++--- 4 files changed, 151 insertions(+), 36 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 74c3e27efd..2679ed82f8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -599,6 +599,7 @@ if (MSVC) endif() endif () +# Build FEI, if requested (to be phased out) if (HYPRE_USING_FEI) add_subdirectory(FEI_mv) endif () @@ -612,6 +613,7 @@ endif () if (HYPRE_BUILD_TESTS) add_subdirectory(test) endif () + # Installation include(GNUInstallDirs) install(TARGETS HYPRE diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index 0dd95b82fc..5f908ca914 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -8,27 +8,29 @@ function(configure_mpi_target) target_link_libraries(${PROJECT_NAME} PUBLIC MPI::MPI_C) # Determine the correct MPI include directory - if(MPI_CXX_INCLUDE_DIRS) - set(MPI_INCLUDE_DIRS ${MPI_CXX_INCLUDE_DIRS}) + if(MPI_CXX_INCLUDE_DIR) + set(MPI_INCLUDE_DIR ${MPI_CXX_INCLUDE_DIR}) elseif(MPI_CXX_INCLUDE_PATH) - set(MPI_INCLUDE_DIRS ${MPI_CXX_INCLUDE_PATH}) - elseif(MPI_CXX_COMPILER_INCLUDE_DIRS) - set(MPI_INCLUDE_DIRS ${MPI_CXX_COMPILER_INCLUDE_DIRS}) - elseif(MPI_C_COMPILER_INCLUDE_DIRS) - set(MPI_INCLUDE_DIRS ${MPI_C_COMPILER_INCLUDE_DIRS}) - elseif(MPI_C_INCLUDE_DIRS) - set(MPI_INCLUDE_DIRS ${MPI_C_INCLUDE_DIRS}) + set(MPI_INCLUDE_DIR ${MPI_CXX_INCLUDE_PATH}) + elseif(MPI_CXX_COMPILER_INCLUDE_DIR) + set(MPI_INCLUDE_DIR ${MPI_CXX_COMPILER_INCLUDE_DIR}) + elseif(MPI_C_COMPILER_INCLUDE_DIR) + set(MPI_INCLUDE_DIR ${MPI_C_COMPILER_INCLUDE_DIR}) + elseif(MPI_C_INCLUDE_DIR) + set(MPI_INCLUDE_DIR ${MPI_C_INCLUDE_DIR}) elseif(MPI_C_INCLUDE_PATH) - set(MPI_INCLUDE_DIRS ${MPI_C_INCLUDE_PATH}) + set(MPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH}) elseif(MPI_INCLUDE_PATH) - set(MPI_INCLUDE_DIRS ${MPI_INCLUDE_PATH}) + set(MPI_INCLUDE_DIR ${MPI_INCLUDE_PATH}) + elseif(MPICH_DIR) + set(MPI_INCLUDE_DIR ${MPICH_DIR}/include) else() - message(WARNING "MPI include directory not found. Compilation may fail.") + message(WARNING "MPI include directory not found. Please specify -DMPI_INCLUDE_DIR or the compilation may fail.") endif() if (HYPRE_WITH_CUDA OR HYPRE_WITH_HIP OR HYPRE_WITH_SYCL) - message(STATUS "Adding MPI include directory: ${MPI_INCLUDE_DIRS}") - target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIRS}) + message(STATUS "Adding MPI include directory: ${MPI_INCLUDE_DIR}") + target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIR}) endif () endfunction() @@ -57,21 +59,32 @@ function(add_hypre_executables EXE_SRCS) foreach(SRC_FILE IN LISTS ${EXE_SRCS}) get_filename_component(SRC_FILENAME ${SRC_FILE} NAME) + # If CUDA is enabled, tag source files to be compiled with nvcc. if (HYPRE_USING_CUDA) - # If CUDA is enabled, tag source files to be compiled with nvcc. set_source_files_properties(${SRC_FILENAME} PROPERTIES LANGUAGE CUDA) - endif (HYPRE_USING_CUDA) + endif() + + # If HIP is enabled, tag source files to be compiled with hipcc/clang + if (HYPRE_USING_HIP) + set_source_files_properties(${SRC_FILENAME} PROPERTIES LANGUAGE HIP) + endif() + # If SYCL is enabled, tag source files to be compiled with dpcpp. if (HYPRE_USING_SYCL) - # If SYCL is enabled, tag source files to be compiled with dpcpp. set_source_files_properties(${SRC_FILENAME} PROPERTIES LANGUAGE CXX) - endif (HYPRE_USING_SYCL) - + endif() + # Get executable name string(REPLACE ".c" "" EXE_NAME ${SRC_FILENAME}) - # Actually add the exe + + # Add the executable add_executable(${EXE_NAME} ${SRC_FILE}) + # Explicitly specify the linker + if (HYPRE_USING_CUDA OR HYPRE_USING_HIP OR HYPRE_USING_SYCL) + set_target_properties(${EXE_NAME} PROPERTIES LINKER_LANGUAGE CXX) + endif() + # Link libraries set(HYPRE_LIBS "HYPRE") @@ -138,4 +151,4 @@ function(print_option_status) # Print the footer separator message(STATUS "${separator}") message(STATUS "") -endfunction() \ No newline at end of file +endfunction() diff --git a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake index 784ebb0b16..ce433f2f8d 100644 --- a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake @@ -36,6 +36,13 @@ set(CUDAToolkit_ROOT "${CUDA_DIR}" CACHE PATH "Path to the CUDA toolkit") # Optionally, prioritize the custom CUDA path in CMAKE_PREFIX_PATH list(APPEND CMAKE_PREFIX_PATH "${CUDA_DIR}") +# Set CUDA standard to match C++ standard if not already set +if(NOT DEFINED CMAKE_CUDA_STANDARD) + set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD}) +endif() +set(CMAKE_CUDA_STANDARD_REQUIRED ON) +set(CMAKE_CUDA_EXTENSIONS OFF) + # Check if CUDA is available and enable it if found include(CheckLanguage) check_language(CUDA) @@ -48,12 +55,50 @@ endif() # Find the CUDA Toolkit find_package(CUDAToolkit REQUIRED) -# Set CUDA standard to match C++ standard if not already set -if(NOT DEFINED CMAKE_CUDA_STANDARD) - set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD}) - set(CMAKE_CUDA_STANDARD_REQUIRED ON) - set(CMAKE_CUDA_EXTENSIONS OFF) +# Detection CUDA architecture if not given by the user +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + message(STATUS "Detecting CUDA GPU architectures using nvidia-smi...") + + # Execute nvidia-smi to get GPU compute capabilities + execute_process( + COMMAND nvidia-smi --query-gpu=compute_cap --format=csv,noheader + OUTPUT_VARIABLE NVIDIA_SMI_OUTPUT + RESULT_VARIABLE NVIDIA_SMI_RESULT + ERROR_VARIABLE NVIDIA_SMI_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + if(NOT NVIDIA_SMI_RESULT EQUAL 0) + message(WARNING "nvidia-smi failed to execute: ${NVIDIA_SMI_ERROR}") + set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "Default CUDA architectures" FORCE) + message(STATUS "Setting CMAKE_CUDA_ARCHITECTURES to default '70'") + else() + # Clean the output (remove extra newlines and spaces) + string(STRIP "${NVIDIA_SMI_OUTPUT}" CUDA_ARCHS) # Remove trailing/leading whitespaces + string(REPLACE "." "" CUDA_ARCHS "${CUDA_ARCHS}") # Replace '.' with nothing to format '7.0' as '70' + string(REPLACE "\n" ";" CUDA_ARCHS "${CUDA_ARCHS}") # Replace newline with semicolon for list format + + # Remove any duplicates CUDA archictectures + list(REMOVE_DUPLICATES CUDA_ARCHS) + + if(CUDA_ARCHS) + string(REPLACE ";" "," CUDA_ARCHS_STR "${CUDA_ARCHS}") + set(CMAKE_CUDA_ARCHITECTURES "${CUDA_ARCHS_STR}" CACHE STRING "Detected CUDA architectures" FORCE) + message(STATUS "Detected CUDA GPU architectures: ${CMAKE_CUDA_ARCHITECTURES}") + else() + message(WARNING "No GPUs detected. Setting CMAKE_CUDA_ARCHITECTURES to default '70'") + set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "Default CUDA architectures" FORCE) + endif() + endif() +else() + # Remove duplicates from the pre-set CMAKE_CUDA_ARCHITECTURES + string(REPLACE "," ";" CUDA_ARCH_LIST "${CMAKE_CUDA_ARCHITECTURES}") + list(REMOVE_DUPLICATES CUDA_ARCH_LIST) + string(REPLACE ";" "," CUDA_ARCH_STR "${CUDA_ARCH_LIST}") + set(CMAKE_CUDA_ARCHITECTURES "${CUDA_ARCH_STR}" CACHE STRING "Detected CUDA architectures" FORCE) + message(STATUS "CMAKE_CUDA_ARCHITECTURES is already set to: ${CMAKE_CUDA_ARCHITECTURES}") endif() +set_property(TARGET HYPRE PROPERTY CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}") # Show CUDA Toolkit location if(CUDAToolkit_FOUND) @@ -77,7 +122,7 @@ find_path(THRUST_INCLUDE_DIR thrust/version.h if(THRUST_INCLUDE_DIR) message(STATUS "CUDA Thrust headers found in: ${THRUST_INCLUDE_DIR}") else() - message(FATAL_ERROR "CUDA Thrust headers not found! Please check your CUDA installation.") + message(FATAL_ERROR "CUDA Thrust headers not found! Please specify -DTHRUST_INCLUDE_DIR.") endif() # Collection of CUDA optional libraries @@ -93,7 +138,7 @@ function(find_and_add_cuda_library LIB_NAME HYPRE_ENABLE_VAR) find_package(CUDAToolkit REQUIRED COMPONENTS ${LIB_NAME}) if(TARGET CUDAToolkit::${LIB_NAME}) - message(STATUS "CUDAToolkit::${LIB_NAME} target found") + message(STATUS "Found ${LIB_NAME} library: ${${LIB_NAME}_LIBRARY}") list(APPEND CUDA_LIBS CUDAToolkit::${LIB_NAME}) else() #message(WARNING "CUDAToolkit::${LIB_NAME} target not found. Attempting manual linking.") @@ -143,4 +188,5 @@ set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -ccbin=${CMAKE_CXX_COMPILER} -expt-ext # Print CUDA info message(STATUS "CUDA Standard: ${CMAKE_CUDA_STANDARD}") +message(STATUS "CUDA Archs: ${CMAKE_CUDA_ARCHITECTURES}") message(STATUS "CUDA FLAGS: ${CMAKE_CUDA_FLAGS}") diff --git a/src/config/cmake/HYPRE_SetupHIPToolkit.cmake b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake index b9c8842c00..99f3d6205a 100644 --- a/src/config/cmake/HYPRE_SetupHIPToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake @@ -3,12 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -# This handles the non-compiler aspect of the HIP toolkit. -# Uses cmake find_package to locate the AMD HIP tools -# for shared libraries. Otherwise for static libraries, assumes -# the libraries are located in ${ROCM_PATH}/lib or ${ROCM_PATH}/lib64. -# Please set environment variable ROCM_PATH or HIP_PATH. - # Check for ROCM_PATH or HIP_PATH message(STATUS "Enabling HIP toolkit") if(DEFINED ROCM_PATH) @@ -27,6 +21,12 @@ message(STATUS "Using ROCm installation: ${HIP_PATH}") # Add HIP_PATH to CMAKE_PREFIX_PATH list(APPEND CMAKE_PREFIX_PATH ${HIP_PATH}) +# Set HIP standard to match C++ standard if not already set +if(NOT DEFINED CMAKE_HIP_STANDARD) + set(CMAKE_HIP_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING "C++ standard for HIP" FORCE) +endif() +set(CMAKE_HIP_STANDARD_REQUIRED ON CACHE BOOL "Require C++ standard for HIP" FORCE) + # Check if HIP is available and enable it if found include(CheckLanguage) check_language(HIP) @@ -39,6 +39,59 @@ endif() # Find HIP package find_package(hip REQUIRED CONFIG) +# Function to detect GPU architectures using rocm-smi +if(NOT DEFINED CMAKE_HIP_ARCHITECTURES) + message(STATUS "Detecting GPU architectures using rocm-smi...") + + # Execute rocm-smi to get GPU architecture info + execute_process( + COMMAND rocm-smi --showUniqueId --format=json + OUTPUT_VARIABLE ROCM_SMI_OUTPUT + RESULT_VARIABLE ROCM_SMI_RESULT + ERROR_VARIABLE ROCM_SMI_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + if(NOT ROCM_SMI_RESULT EQUAL 0) + message(WARNING "rocm-smi failed to execute: ${ROCM_SMI_ERROR}") + set(CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "Default HIP architectures" FORCE) + message(STATUS "Setting CMAKE_HIP_ARCHITECTURES to default 'gfx90a'") + return() + endif() + + # Parse JSON output to extract GPU architectures + include(Jsoncpp) + jsoncpp_parse(json_output "${ROCM_SMI_OUTPUT}") + + # Extract GPU architecture codes + set(GPU_ARCHS "") + foreach(gpu ${json_output@/gpu/@}) + get_property(gpu_arch PROPERTY GPU ARCHITECTURE "${gpu}") + if(NOT gpu_arch STREQUAL "") + list(APPEND GPU_ARCHS ${gpu_arch}) + endif() + endforeach() + + list(REMOVE_DUPLICATES GPU_ARCHS) + + if(GPU_ARCHS) + string(REPLACE ";" "," GPU_ARCHS_STR "${GPU_ARCHS}") + set(CMAKE_HIP_ARCHITECTURES "${GPU_ARCHS_STR}" CACHE STRING "Detected HIP architectures" FORCE) + message(STATUS "Detected GPU architectures: ${CMAKE_HIP_ARCHITECTURES}") + else() + message(WARNING "No GPUs detected. Setting CMAKE_HIP_ARCHITECTURES to default 'gfx90a'") + set(CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "Default HIP architectures" FORCE) + endif() +else() + # Remove duplicates from the pre-set CMAKE_HIP_ARCHITECTURES + string(REPLACE "," ";" HIP_ARCH_LIST "${CMAKE_HIP_ARCHITECTURES}") + list(REMOVE_DUPLICATES HIP_ARCH_LIST) + string(REPLACE ";" "," HIP_ARCH_STR "${HIP_ARCH_LIST}") + set(CMAKE_HIP_ARCHITECTURES "${HIP_ARCH_STR}" CACHE STRING "Detected HIP architectures" FORCE) + message(STATUS "CMAKE_HIP_ARCHITECTURES is already set to: ${CMAKE_HIP_ARCHITECTURES}") +endif() +set_property(TARGET HYPRE PROPERTY HIP_ARCHITECTURES "${CMAKE_HIP_ARCHITECTURES}") + # Collection of ROCm optional libraries set(ROCM_LIBS "") @@ -50,7 +103,7 @@ function(find_and_add_rocm_library LIB_NAME) set(HYPRE_USING_${LIB_NAME_UPPER} ON CACHE BOOL "" FORCE) find_package(${LIB_NAME} REQUIRED) if(TARGET roc::${LIB_NAME}) - message(STATUS "roc::${LIB_NAME} target found") + message(STATUS "Found target: ${${LIB_NAME}_LIBRARY}") list(APPEND ROCM_LIBS roc::${LIB_NAME}) else() #message(WARNING "roc::${LIB_NAME} target not found. Attempting manual linking.") @@ -98,8 +151,9 @@ target_link_libraries(HYPRE PUBLIC ${ROCM_LIBS}) message(STATUS "Linking to ROCm libraries: ${ROCM_LIBS}") # Set HIP-specific variables -set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -fPIC" CACHE STRING "HIP compiler flags" FORCE) +target_compile_options(HYPRE PUBLIC $<$:-fPIC>) # Print HIP info message(STATUS "HIP Standard: ${CMAKE_HIP_STANDARD}") +message(STATUS "HIP Archs: ${CMAKE_HIP_ARCHITECTURES}") message(STATUS "HIP FLAGS: ${CMAKE_HIP_FLAGS}") From f4d3f4dae4f8151d116d699c842c17fdc32810c7 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sun, 15 Sep 2024 21:42:24 -0400 Subject: [PATCH 12/55] Adjustment to run nvidia-smi to detect cuda arch --- src/config/cmake/HYPRE_SetupCUDAToolkit.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake index ce433f2f8d..d89297f508 100644 --- a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake @@ -56,7 +56,7 @@ endif() find_package(CUDAToolkit REQUIRED) # Detection CUDA architecture if not given by the user -if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "52") message(STATUS "Detecting CUDA GPU architectures using nvidia-smi...") # Execute nvidia-smi to get GPU compute capabilities From 329729f806d253caab816d657ada669ee06518aa Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Sep 2024 16:55:00 -0400 Subject: [PATCH 13/55] Simplify build options setup with set_conditional_var + Simplify GPU build structure --- src/CMakeLists.txt | 127 +++--------------- src/config/cmake/HYPRE_CMakeUtilities.cmake | 8 ++ src/config/cmake/HYPRE_SetupCUDAToolkit.cmake | 6 +- src/config/cmake/HYPRE_SetupGPUToolkit.cmake | 52 +++++-- src/config/cmake/HYPRE_SetupHIPToolkit.cmake | 6 +- 5 files changed, 72 insertions(+), 127 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2679ed82f8..3d7e5869b6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -161,45 +161,23 @@ set(HYPRE_SYCL_TARGET "" CACHE STRING "Target SYCL architecture, e.g set(HYPRE_SYCL_TARGET_BACKEND "" CACHE STRING "Additional SYCL backend options, e.g. '-device 12.1.0,12.4.0'.") # Set config name values -if (HYPRE_ENABLE_SHARED) - set(HYPRE_SHARED ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_BIGINT) - set(HYPRE_BIGINT ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_MIXEDINT) - set(HYPRE_MIXEDINT ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_SINGLE) - set(HYPRE_SINGLE ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_LONG_DOUBLE) - set(HYPRE_LONG_DOUBLE ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_COMPLEX) - set(HYPRE_COMPLEX ON CACHE BOOL "" FORCE) -endif () - -if (CMAKE_BUILD_TYPE STREQUAL "Debug") - set(HYPRE_DEBUG ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_HYPRE_BLAS) - set(HYPRE_USING_HYPRE_BLAS ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_HYPRE_LAPACK) - set(HYPRE_USING_HYPRE_LAPACK ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_PERSISTENT_COMM) - set(HYPRE_USING_PERSISTENT_COMM ON CACHE BOOL "" FORCE) -endif () +set_conditional_var(HYPRE_ENABLE_SHARED HYPRE_SHARED) +set_conditional_var(HYPRE_ENABLE_BIGINT HYPRE_BIGINT) +set_conditional_var(HYPRE_ENABLE_MIXEDINT HYPRE_MIXEDINT) +set_conditional_var(HYPRE_ENABLE_SINGLE HYPRE_SINGLE) +set_conditional_var(HYPRE_ENABLE_LONG_DOUBLE HYPRE_LONG_DOUBLE) +set_conditional_var(HYPRE_ENABLE_COMPLEX HYPRE_COMPLEX) +set_conditional_var(CMAKE_BUILD_TYPE STREQUAL "Debug" HYPRE_DEBUG) +set_conditional_var(HYPRE_ENABLE_HYPRE_BLAS HYPRE_USING_HYPRE_BLAS) +set_conditional_var(HYPRE_ENABLE_HYPRE_LAPACK HYPRE_USING_HYPRE_LAPACK) +set_conditional_var(HYPRE_ENABLE_PERSISTENT_COMM HYPRE_USING_PERSISTENT_COMM) +set_conditional_var(HYPRE_WITH_OPENMP HYPRE_USING_OPENMP) +set_conditional_var(HYPRE_ENABLE_HOPSCOTCH HYPRE_HOPSCOTCH) +set_conditional_var(HYPRE_WITH_CUDA HYPRE_USING_CUDA) +set_conditional_var(HYPRE_WITH_HIP HYPRE_USING_HIP) +set_conditional_var(HYPRE_WITH_SYCL HYPRE_USING_SYCL) +set_conditional_var(HYPRE_WITH_MAGMA HYPRE_USING_MAGMA) +set_conditional_var(HYPRE_WITH_CALIPER HYPRE_USING_CALIPER) if (HYPRE_WITH_MPI) set(HYPRE_HAVE_MPI ON CACHE BOOL "" FORCE) @@ -208,26 +186,6 @@ else () set(HYPRE_SEQUENTIAL ON CACHE BOOL "" FORCE) endif () -if (HYPRE_WITH_OPENMP) - set(HYPRE_USING_OPENMP ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_HOPSCOTCH) - set(HYPRE_HOPSCOTCH ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_WITH_CUDA) - set(HYPRE_USING_CUDA ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_WITH_HIP) - set(HYPRE_USING_HIP ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_WITH_SYCL) - set(HYPRE_USING_SYCL ON CACHE BOOL "" FORCE) -endif () - if (HYPRE_WITH_SUPERLU) set(HYPRE_USING_SUPERLU ON CACHE BOOL "" FORCE) add_compile_definitions(HAVE_SUPERLU) @@ -239,21 +197,13 @@ if (HYPRE_WITH_DSUPERLU) set(HYPRE_USING_HYPRE_LAPACK OFF CACHE BOOL "" FORCE) endif () -if (HYPRE_WITH_MAGMA) - set(HYPRE_USING_MAGMA ON CACHE BOOL "" FORCE) -endif () - if (HYPRE_ENABLE_FEI) set(HYPRE_USING_FEI ON CACHE BOOL "" FORCE) message(WARNING "CMake support for FEI is not complete!") endif () -if (HYPRE_WITH_CALIPER) - set(HYPRE_USING_CALIPER ON CACHE BOOL "" FORCE) -endif () - +# FEI doesn't currently compile with shared if (HYPRE_SHARED OR HYPRE_BIGINT OR HYPRE_SINGLE OR HYPRE_LONG_DOUBLE) - # FEI doesn't currently compile with shared set(HYPRE_USING_FEI OFF CACHE BOOL "" FORCE) endif () @@ -300,45 +250,7 @@ endif (HYPRE_USING_OPENMP) # Set GPU compile flags if (HYPRE_WITH_CUDA OR HYPRE_WITH_HIP OR HYPRE_WITH_SYCL) - enable_language(CXX) - message(STATUS "Enabled support for CXX.") - - # Setup GPU libraries and include directories - set(HYPRE_USING_GPU ON CACHE BOOL "" FORCE) include(HYPRE_SetupGPUToolkit) - - # CXX standard info - set(CMAKE_CXX_STANDARD_REQUIRED ON) - message(STATUS "Using CXX standard: c++${CMAKE_CXX_STANDARD}") - - # Check if examples are enabled, but not unified memory - if (HYPRE_BUILD_EXAMPLES AND NOT HYPRE_ENABLE_UNIFIED_MEMORY) - message(WARNING "Running the examples on GPUs requires Unified Memory! - Examples will not be built!") - set(HYPRE_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) - endif () - - # Add any extra CXX compiler flags HYPRE_WITH_EXTRA_CXXFLAGS - if (NOT HYPRE_WITH_EXTRA_CXXFLAGS STREQUAL "") - string(REPLACE " " ";" HYPRE_WITH_EXTRA_CXXFLAGS_LIST ${HYPRE_WITH_EXTRA_CXXFLAGS}) - add_compile_options("$<$:${HYPRE_WITH_EXTRA_CXXFLAGS_LIST}>") - endif () - - if (HYPRE_ENABLE_GPU_STREAMS) - set(HYPRE_USING_GPU_STREAMS ON CACHE BOOL "" FORCE) - endif () - - if (HYPRE_ENABLE_DEVICE_POOL) - set(HYPRE_USING_DEVICE_POOL ON CACHE BOOL "" FORCE) - endif () - - set(HYPRE_USING_HOST_MEMORY OFF CACHE BOOL "" FORCE) - if (HYPRE_ENABLE_UNIFIED_MEMORY) - set(HYPRE_USING_UNIFIED_MEMORY ON CACHE BOOL "" FORCE) - else () - set(HYPRE_USING_DEVICE_MEMORY ON CACHE BOOL "" FORCE) - endif () - else () message(STATUS "No GPU support enabled.") set(HYPRE_USING_HOST_MEMORY ON CACHE BOOL "" FORCE) @@ -367,13 +279,14 @@ set_target_properties(HYPRE PROPERTIES # Headers and sources set(HYPRE_HEADERS "") -# Headers and sources: . +# Main headers set(HYPRE_MAIN_HEADERS ${CMAKE_CURRENT_BINARY_DIR}/HYPRE_config.h HYPREf.h HYPRE.h ) +# All headers set(HYPRE_HEADERS ${HYPRE_HEADERS} ${HYPRE_MAIN_HEADERS}) # Headers and sources: blas diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index 5f908ca914..c1263c913f 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -3,6 +3,14 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Function to set conditional variables +function(set_conditional_var condition var_name) + if(${condition}) + set(${var_name} ON CACHE BOOL "" FORCE) + endif() +endfunction() + +# Function to configure MPI target function(configure_mpi_target) find_package(MPI REQUIRED) target_link_libraries(${PROJECT_NAME} PUBLIC MPI::MPI_C) diff --git a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake index d89297f508..52bdd546f8 100644 --- a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake @@ -187,6 +187,6 @@ message(STATUS "Linking to CUDA libraries: ${CUDA_LIBS}") set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -ccbin=${CMAKE_CXX_COMPILER} -expt-extended-lambda") # Print CUDA info -message(STATUS "CUDA Standard: ${CMAKE_CUDA_STANDARD}") -message(STATUS "CUDA Archs: ${CMAKE_CUDA_ARCHITECTURES}") -message(STATUS "CUDA FLAGS: ${CMAKE_CUDA_FLAGS}") +message(STATUS "CUDA C++ standard: ${CMAKE_CUDA_STANDARD}") +message(STATUS "CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}") +message(STATUS "CUDA flags: ${CMAKE_CUDA_FLAGS}") diff --git a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake index 5349527b2e..c80b51faaa 100644 --- a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake @@ -3,26 +3,50 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Enable CXX language +enable_language(CXX) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +# Enforce C++14 at least +if(NOT DEFINED CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) + set(CMAKE_CXX_STANDARD 14) + set_property(TARGET HYPRE PROPERTY CXX_STANDARD 14) +endif() +message(STATUS "Enabled support for CXX.") +message(STATUS "Using CXX standard: C++${CMAKE_CXX_STANDARD}") + +# Set GPU-related variables +set(HYPRE_USING_GPU ON CACHE BOOL "" FORCE) +set(HYPRE_USING_HOST_MEMORY OFF CACHE BOOL "" FORCE) + +if(HYPRE_ENABLE_UNIFIED_MEMORY) + set(HYPRE_USING_UNIFIED_MEMORY ON CACHE BOOL "" FORCE) +else() + set(HYPRE_USING_DEVICE_MEMORY ON CACHE BOOL "" FORCE) +endif() + +# Check if examples are enabled, but not unified memory +if(HYPRE_BUILD_EXAMPLES AND NOT HYPRE_ENABLE_UNIFIED_MEMORY) + message(WARNING "Running the examples on GPUs requires Unified Memory! Examples will not be built!") + set(HYPRE_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +endif() + +# Add any extra CXX compiler flags +if(NOT HYPRE_WITH_EXTRA_CXXFLAGS STREQUAL "") + string(REPLACE " " ";" HYPRE_WITH_EXTRA_CXXFLAGS_LIST ${HYPRE_WITH_EXTRA_CXXFLAGS}) + add_compile_options("$<$:${HYPRE_WITH_EXTRA_CXXFLAGS_LIST}>") +endif() + +# Set conditional variables +set_conditional_var(HYPRE_ENABLE_GPU_STREAMS HYPRE_USING_GPU_STREAMS) +set_conditional_var(HYPRE_ENABLE_DEVICE_POOL HYPRE_USING_DEVICE_POOL) + if(HYPRE_WITH_CUDA) - # Enforce C++11 at least - if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 11) - set(CMAKE_CXX_STANDARD 11) - endif () include(HYPRE_SetupCUDAToolkit) elseif(HYPRE_WITH_HIP) - # Enforce C++14 at least - if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) - set(CMAKE_CXX_STANDARD 14) - endif () include(HYPRE_SetupHIPToolkit) elseif(HYPRE_WITH_SYCL) - # Enforce C++14 at least - if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) - set(CMAKE_CXX_STANDARD 14) - endif () - message(STATUS "Enabling SYCL toolkit") enable_language(SYCL) include(HYPRE_SetupSYCLToolkit) @@ -30,4 +54,4 @@ elseif(HYPRE_WITH_SYCL) else() message(FATAL_ERROR "Neither CUDA nor HIP nor SYCL is enabled. Please enable one of them.") -endif() +endif() \ No newline at end of file diff --git a/src/config/cmake/HYPRE_SetupHIPToolkit.cmake b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake index 99f3d6205a..b9029f39dd 100644 --- a/src/config/cmake/HYPRE_SetupHIPToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake @@ -154,6 +154,6 @@ message(STATUS "Linking to ROCm libraries: ${ROCM_LIBS}") target_compile_options(HYPRE PUBLIC $<$:-fPIC>) # Print HIP info -message(STATUS "HIP Standard: ${CMAKE_HIP_STANDARD}") -message(STATUS "HIP Archs: ${CMAKE_HIP_ARCHITECTURES}") -message(STATUS "HIP FLAGS: ${CMAKE_HIP_FLAGS}") +message(STATUS "HIP C++ standard: ${CMAKE_HIP_STANDARD}") +message(STATUS "HIP architectures: ${CMAKE_HIP_ARCHITECTURES}") +message(STATUS "HIP flags: ${CMAKE_HIP_FLAGS}") From c7da232107cf5fe2880140205721f57ba71407c2 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Sep 2024 22:16:57 -0400 Subject: [PATCH 14/55] Add setup_tpl to simplify TPLs usage --- src/CMakeLists.txt | 189 +++---------------- src/config/cmake/HYPRE_CMakeUtilities.cmake | 81 ++++++++ src/config/cmake/HYPRE_SetupGPUToolkit.cmake | 2 +- 3 files changed, 105 insertions(+), 167 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3d7e5869b6..c8d48e6df0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -263,11 +263,7 @@ if (NOT HYPRE_WITH_EXTRA_CFLAGS STREQUAL "") endif () # Set library build type (must appear before add_library calls) -if (HYPRE_SHARED) - set(BUILD_SHARED_LIBS ON CACHE INTERNAL "" FORCE) -else () - set(BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE) -endif () +set(BUILD_SHARED_LIBS ${HYPRE_SHARED} CACHE INTERNAL "Build shared libraries" FORCE) # Set output directory for the library set_target_properties(HYPRE PROPERTIES @@ -276,115 +272,13 @@ set_target_properties(HYPRE PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" ) -# Headers and sources -set(HYPRE_HEADERS "") - -# Main headers +# Set headers set(HYPRE_MAIN_HEADERS ${CMAKE_CURRENT_BINARY_DIR}/HYPRE_config.h HYPREf.h HYPRE.h - ) - -# All headers -set(HYPRE_HEADERS ${HYPRE_HEADERS} ${HYPRE_MAIN_HEADERS}) - -# Headers and sources: blas -if (HYPRE_USING_HYPRE_BLAS) - add_subdirectory(blas) -else () - # Use TPL_BLAS_LIBRARIES if set. - if (TPL_BLAS_LIBRARIES) - message(STATUS "Using TPL_BLAS_LIBRARIES='${TPL_BLAS_LIBRARIES}'") - target_link_libraries(${PROJECT_NAME} PUBLIC "${TPL_BLAS_LIBRARIES}") - else () - # Find system blas - find_package(BLAS REQUIRED) - target_link_libraries(${PROJECT_NAME} PUBLIC "${BLAS_LIBRARIES}") - endif () - target_compile_definitions(${PROJECT_NAME} PUBLIC "USE_VENDOR_BLAS") -endif () - -# Headers and sources: lapack -if (HYPRE_USING_HYPRE_LAPACK) - add_subdirectory(lapack) -else () - # Use TPL_LAPACK_LIBRARIES if set. - if (TPL_LAPACK_LIBRARIES) - message(STATUS "Using TPL_LAPACK_LIBRARIES='${TPL_LAPACK_LIBRARIES}'") - target_link_libraries(${PROJECT_NAME} PUBLIC "${TPL_LAPACK_LIBRARIES}") - else () - # Find system lapack - find_package(LAPACK REQUIRED) - target_link_libraries(${PROJECT_NAME} PUBLIC "${LAPACK_LIBRARIES}") - endif () -endif () - -# Find SUPERLU, if requested -if (HYPRE_USING_SUPERLU) - if (NOT TPL_SUPERLU_LIBRARIES) - message(FATAL_ERROR "TPL_SUPERLU_LIBRARIES option should be set for SuperLU support.") - endif () - - if (NOT TPL_SUPERLU_INCLUDE_DIRS) - message(FATAL_ERROR "TPL_SUPERLU_INCLUDE_DIRS option be set for SuperLU support.") - endif () - - foreach (dir ${TPL_SUPERLU_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "SuperLU include directory not found: ${dir}") - endif () - target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) - endforeach () - message(STATUS "Enabled support for using SUPERLU.") - set(SUPERLU_FOUND TRUE) - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_SUPERLU_LIBRARIES} stdc++) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_SUPERLU_INCLUDE_DIRS}) -endif (HYPRE_USING_SUPERLU) - -# Find DSUPERLU, if requested -if (HYPRE_USING_DSUPERLU) - if (NOT TPL_DSUPERLU_LIBRARIES) - message(FATAL_ERROR "TPL_DSUPERLU_LIBRARIES option should be set for SuperLU_Dist support.") - endif () - - if (NOT TPL_DSUPERLU_INCLUDE_DIRS) - message(FATAL_ERROR "TPL_DSUPERLU_INCLUDE_DIRS option be set for SuperLU_Dist support.") - endif () - - foreach (dir ${TPL_DSUPERLU_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "SuperLU_Dist include directory not found: ${dir}") - endif () - target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) - endforeach () - message(STATUS "Enabled support for using DSUPERLU.") - set(DSUPERLU_FOUND TRUE) - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_DSUPERLU_LIBRARIES} stdc++) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_DSUPERLU_INCLUDE_DIRS}) -endif (HYPRE_USING_DSUPERLU) - -# TODO: Check for case if MAGMA without CUDA/HIP -# Find MAGMA, if requested -if (HYPRE_USING_MAGMA) - if (NOT TPL_MAGMA_LIBRARIES) - message(FATAL_ERROR "TPL_MAGMA_LIBRARIES option should be set for MAGMA support.") - endif () - - if (NOT TPL_MAGMA_INCLUDE_DIRS) - message(FATAL_ERROR "TPL_MAGMA_INCLUDE_DIRS option be set for MAGMA support.") - endif () - - foreach (dir ${TPL_MAGMA_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "MAGMA include directory not found: ${dir}") - endif () - target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) - endforeach () - message(STATUS "Enabled support for using MAGMA.") - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_MAGMA_LIBRARIES} stdc++) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_MAGMA_INCLUDE_DIRS}) -endif (HYPRE_USING_MAGMA) +) +list(APPEND HYPRE_HEADERS ${HYPRE_MAIN_HEADERS}) # Find FEI, if requested if (HYPRE_USING_FEI) @@ -405,39 +299,14 @@ if (HYPRE_USING_FEI) target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_FEI_INCLUDE_DIRS}) endif(HYPRE_USING_FEI) -# Find Caliper, if requested -if (HYPRE_USING_CALIPER) - if (NOT TPL_CALIPER_LIBRARIES OR NOT TPL_CALIPER_INCLUDE_DIRS) - message(FATAL_ERROR "Both TPL_CALIPER_LIBRARIES and TPL_CALIPER_INCLUDE_DIRS options must be set for Caliper support.") - endif () - - foreach (dir ${TPL_CALIPER_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "Caliper include directory not found: ${dir}") - endif () - target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) - endforeach () - message(STATUS "Enabled support for using Caliper.") - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_CALIPER_LIBRARIES}) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_CALIPER_INCLUDE_DIRS}) -endif() - -# Find Umpire, if requested -if (HYPRE_USING_UMPIRE) - if (NOT TPL_UMPIRE_LIBRARIES OR NOT TPL_UMPIRE_INCLUDE_DIRS) - message(FATAL_ERROR "Both TPL_UMPIRE_LIBRARIES and TPL_UMPIRE_INCLUDE_DIRS options must be set for Umpire support.") - endif () - - foreach (dir ${TPL_UMPIRE_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "Umpire include directory not found: ${dir}") - endif () - target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) - endforeach () - message(STATUS "Enabled support for using Umpire.") - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_UMPIRE_LIBRARIES}) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_UMPIRE_INCLUDE_DIRS}) -endif () +# Setup TPLs +setup_tpl_or_internal(blas) +setup_tpl_or_internal(lapack) +setup_tpl(SUPERLU) +setup_tpl(DSUPERLU) +setup_tpl(MAGMA) +setup_tpl(CALIPER) +setup_tpl(UMPIRE) # Configure a header file to pass CMake settings to the source code configure_file( @@ -453,14 +322,14 @@ foreach (DIR IN LISTS HYPRE_DIRS) $) endforeach () -# BINARY must be first in order to get the correct HYPRE_config.h file -#target_include_directories(${PROJECT_NAME} PUBLIC -# $ -# $ -# $ -# $ -# $ -# ) +# Set the language for GPU sources (has to be done after the subdirectories are added) +if (HYPRE_WITH_CUDA) + set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE CUDA) +elseif (HYPRE_WITH_HIP) + set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE HIP) +elseif (HYPRE_WITH_SYCL) + set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE SYCL) +endif () # Include directories target_include_directories(${PROJECT_NAME} PUBLIC @@ -471,20 +340,7 @@ target_include_directories(${PROJECT_NAME} PUBLIC $ ) -if (HYPRE_WITH_CUDA) - set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE CUDA) -elseif (HYPRE_WITH_HIP) - set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE HIP) -elseif (HYPRE_WITH_SYCL) - set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE SYCL) -endif () - -# Ensure MPI headers are included for device compilation -#if (HYPRE_WITH_MPI AND HYPRE_USING_GPU) -# message(STATUS "Adding MPI include directory: ${MPI_INCLUDE_DIRS}") -# target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIRS}) -#endif () - +# Set file properties for MSVC if (MSVC) target_compile_definitions(${PROJECT_NAME} PRIVATE _CRT_SECURE_NO_WARNINGS) if (MSVC_VERSION LESS 1928) # Visual Studio 2019 version 16.8 claims full C11 support @@ -553,7 +409,8 @@ install(EXPORT HYPRETargets ) configure_package_config_file( - config/HYPREConfig.cmake.in HYPREConfig.cmake + ${CMAKE_CURRENT_SOURCE_DIR}/config/HYPREConfig.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake INSTALL_DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake ) diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index c1263c913f..48d18b3f62 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -42,6 +42,87 @@ function(configure_mpi_target) endif () endfunction() +# Function to handle TPL (Third-Party Library) setup +function(setup_tpl NAME) + if(HYPRE_USING_${NAME}) + if(NOT TPL_${NAME}_LIBRARIES OR NOT TPL_${NAME}_INCLUDE_DIRS) + message(FATAL_ERROR "Both TPL_${NAME}_LIBRARIES and TPL_${NAME}_INCLUDE_DIRS must be set for ${NAME} support.") + endif() + + foreach(dir ${TPL_${NAME}_INCLUDE_DIRS}) + if(NOT EXISTS ${dir}) + message(FATAL_ERROR "${NAME} include directory not found: ${dir}") + endif() + endforeach() + + message(STATUS "Enabled support for using ${NAME}.") + + foreach(lib ${TPL_${NAME}_LIBRARIES}) + if(EXISTS ${lib}) + message(STATUS "${NAME} library found: ${lib}") + else() + message(WARNING "${NAME} library not found at specified path: ${lib}") + endif() + endforeach() + + target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_${NAME}_LIBRARIES}) + target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_${NAME}_INCLUDE_DIRS}) + + if(${NAME} STREQUAL "SUPERLU" OR ${NAME} STREQUAL "DSUPERLU" OR ${NAME} STREQUAL "UMPIRE") + target_link_libraries(${PROJECT_NAME} PUBLIC stdc++) + endif() + + set(${NAME}_FOUND TRUE PARENT_SCOPE) + endif() +endfunction() + +# Function to setup TPL or internal library implementation +function(setup_tpl_or_internal LIB_NAME) + string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) + + if(HYPRE_USING_HYPRE_${LIB_NAME_UPPER}) + # Use internal library + add_subdirectory(${LIB_NAME}) + message(STATUS "Using internal ${LIB_NAME_UPPER}") + target_include_directories(${PROJECT_NAME} PUBLIC + $ + $ + ) + else() + # Use external library + if(TPL_${LIB_NAME_UPPER}_LIBRARIES) + # Use specified TPL libraries + message(STATUS "Enabled support for using ${LIB_NAME_UPPER}.") + foreach(lib ${TPL_${LIB_NAME_UPPER}_LIBRARIES}) + if(EXISTS ${lib}) + message(STATUS "${LIB_NAME_UPPER} library found: ${lib}") + else() + message(WARNING "${LIB_NAME_UPPER} library not found at specified path: ${lib}") + endif() + endforeach() + target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_${LIB_NAME_UPPER}_LIBRARIES}) + else() + # Find system library + find_package(${LIB_NAME_UPPER} REQUIRED) + if(${LIB_NAME_UPPER}_FOUND) + message(STATUS "Using system ${LIB_NAME_UPPER}") + if(TARGET ${LIB_NAME_UPPER}::${LIB_NAME_UPPER}) + target_link_libraries(${PROJECT_NAME} PUBLIC ${LIB_NAME_UPPER}::${LIB_NAME_UPPER}) + else() + target_link_libraries(${PROJECT_NAME} PUBLIC ${${LIB_NAME_UPPER}_LIBRARIES}) + endif() + else() + message(FATAL_ERROR "${LIB_NAME_UPPER} not found") + endif() + endif() + + # Add USE_VENDOR_BLAS definition for BLAS + if(${LIB_NAME_UPPER} STREQUAL "BLAS") + target_compile_definitions(${PROJECT_NAME} PUBLIC "USE_VENDOR_BLAS") + endif() + endif() +endfunction() + # A handy function to add the current source directory to a local # filename. To be used for creating a list of sources. function(convert_filenames_to_full_paths NAMES) diff --git a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake index c80b51faaa..b3f21162d9 100644 --- a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake @@ -11,7 +11,7 @@ if(NOT DEFINED CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) set(CMAKE_CXX_STANDARD 14) set_property(TARGET HYPRE PROPERTY CXX_STANDARD 14) endif() -message(STATUS "Enabled support for CXX.") +message(STATUS "Enabling support for CXX.") message(STATUS "Using CXX standard: C++${CMAKE_CXX_STANDARD}") # Set GPU-related variables From c1f996140a2464f1c30dfb548ffb7ba84fca6d09 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Sep 2024 23:38:08 -0400 Subject: [PATCH 15/55] Add find_package support to setup_tpl --- src/CMakeLists.txt | 10 +-- src/config/cmake/HYPRE_CMakeUtilities.cmake | 74 ++++++++++++------- src/config/cmake/HYPRE_SetupCUDAToolkit.cmake | 6 ++ src/config/cmake/HYPRE_SetupGPUToolkit.cmake | 7 +- src/config/cmake/HYPRE_SetupSYCLToolkit.cmake | 2 +- 5 files changed, 62 insertions(+), 37 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c8d48e6df0..1aaf89369b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -302,11 +302,11 @@ endif(HYPRE_USING_FEI) # Setup TPLs setup_tpl_or_internal(blas) setup_tpl_or_internal(lapack) -setup_tpl(SUPERLU) -setup_tpl(DSUPERLU) -setup_tpl(MAGMA) -setup_tpl(CALIPER) -setup_tpl(UMPIRE) +setup_tpl(superlu) +setup_tpl(dsuperlu) +setup_tpl(magma) +setup_tpl(caliper) +setup_tpl(umpire) # Configure a header file to pass CMake settings to the source code configure_file( diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index 48d18b3f62..392f3d55ec 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -44,42 +44,67 @@ endfunction() # Function to handle TPL (Third-Party Library) setup function(setup_tpl NAME) - if(HYPRE_USING_${NAME}) - if(NOT TPL_${NAME}_LIBRARIES OR NOT TPL_${NAME}_INCLUDE_DIRS) - message(FATAL_ERROR "Both TPL_${NAME}_LIBRARIES and TPL_${NAME}_INCLUDE_DIRS must be set for ${NAME} support.") - endif() + string(TOUPPER ${NAME} NAME_UPPER) + if(HYPRE_USING_${NAME_UPPER}) + if(TPL_${NAME_UPPER}_LIBRARIES AND TPL_${NAME_UPPER}_INCLUDE_DIRS) + # Use specified TPL libraries and include dirs + foreach(dir ${TPL_${NAME_UPPER}_INCLUDE_DIRS}) + if(NOT EXISTS ${dir}) + message(FATAL_ERROR "${NAME_UPPER} include directory not found: ${dir}") + endif() + endforeach() - foreach(dir ${TPL_${NAME}_INCLUDE_DIRS}) - if(NOT EXISTS ${dir}) - message(FATAL_ERROR "${NAME} include directory not found: ${dir}") - endif() - endforeach() + foreach(lib ${TPL_${NAME_UPPER}_LIBRARIES}) + if(EXISTS ${lib}) + message(STATUS "${NAME_UPPER} library found: ${lib}") + else() + message(WARNING "${NAME_UPPER} library not found at specified path: ${lib}") + endif() + endforeach() + + target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_${NAME_UPPER}_LIBRARIES}) + target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_${NAME_UPPER}_INCLUDE_DIRS}) + else() + # Use find_package + find_package(${NAME} REQUIRED CONFIG) + if(${NAME}_FOUND) + message(STATUS "Found ${NAME_UPPER}") - message(STATUS "Enabled support for using ${NAME}.") - - foreach(lib ${TPL_${NAME}_LIBRARIES}) - if(EXISTS ${lib}) - message(STATUS "${NAME} library found: ${lib}") + target_link_libraries(${PROJECT_NAME} PUBLIC ${NAME}) + target_include_directories(${PROJECT_NAME} PUBLIC ${${NAME}_INCLUDE_DIRS}) + + # Get and display information about the umpire target + get_target_property(INTERFACE_LINK_LIBRARIES ${NAME} INTERFACE_LINK_LIBRARIES) + get_target_property(INTERFACE_INCLUDE_DIRS ${NAME} INTERFACE_INCLUDE_DIRECTORIES) + + message(STATUS " linked libraries: ${INTERFACE_LINK_LIBRARIES}") + message(STATUS " include directories: ${INTERFACE_INCLUDE_DIRS}") else() - message(WARNING "${NAME} library not found at specified path: ${lib}") + message(FATAL_ERROR "${NAME_UPPER} target not found. Please check your ${NAME_UPPER} installation.") endif() - endforeach() - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_${NAME}_LIBRARIES}) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_${NAME}_INCLUDE_DIRS}) - - if(${NAME} STREQUAL "SUPERLU" OR ${NAME} STREQUAL "DSUPERLU" OR ${NAME} STREQUAL "UMPIRE") + if(DEFINED ${NAME}_VERSION) + message(STATUS " Version: ${${NAME}_VERSION}") + endif() + if(DEFINED ${NAME}_DIR) + message(STATUS " Config directory: ${${NAME}_DIR}") + endif() + endif() + + message(STATUS "Enabled support for using ${NAME_UPPER}.") + + if(${NAME_UPPER} STREQUAL "SUPERLU" OR ${NAME_UPPER} STREQUAL "DSUPERLU" OR ${NAME_UPPER} STREQUAL "UMPIRE") target_link_libraries(${PROJECT_NAME} PUBLIC stdc++) endif() - set(${NAME}_FOUND TRUE PARENT_SCOPE) + set(${NAME_UPPER}_FOUND TRUE PARENT_SCOPE) endif() endfunction() # Function to setup TPL or internal library implementation function(setup_tpl_or_internal LIB_NAME) string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) - + if(HYPRE_USING_HYPRE_${LIB_NAME_UPPER}) # Use internal library add_subdirectory(${LIB_NAME}) @@ -115,11 +140,6 @@ function(setup_tpl_or_internal LIB_NAME) message(FATAL_ERROR "${LIB_NAME_UPPER} not found") endif() endif() - - # Add USE_VENDOR_BLAS definition for BLAS - if(${LIB_NAME_UPPER} STREQUAL "BLAS") - target_compile_definitions(${PROJECT_NAME} PUBLIC "USE_VENDOR_BLAS") - endif() endif() endfunction() diff --git a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake index 52bdd546f8..326d0e6225 100644 --- a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake @@ -45,6 +45,7 @@ set(CMAKE_CUDA_EXTENSIONS OFF) # Check if CUDA is available and enable it if found include(CheckLanguage) +set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "" FORCE) check_language(CUDA) if(CMAKE_CUDA_COMPILER) enable_language(CUDA) @@ -55,6 +56,11 @@ endif() # Find the CUDA Toolkit find_package(CUDAToolkit REQUIRED) +# Add a dummy cuda target if it doesn't exist (avoid error when building with BLT dependencies) +if(NOT TARGET cuda) + add_library(cuda INTERFACE) +endif() + # Detection CUDA architecture if not given by the user if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "52") message(STATUS "Detecting CUDA GPU architectures using nvidia-smi...") diff --git a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake index b3f21162d9..5c0f20c711 100644 --- a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake @@ -6,11 +6,10 @@ # Enable CXX language enable_language(CXX) set(CMAKE_CXX_STANDARD_REQUIRED ON) -# Enforce C++14 at least if(NOT DEFINED CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) - set(CMAKE_CXX_STANDARD 14) - set_property(TARGET HYPRE PROPERTY CXX_STANDARD 14) + set(CMAKE_CXX_STANDARD 14) # Enforce C++14 at least endif() +set_property(TARGET HYPRE PROPERTY CXX_STANDARD ${CMAKE_CXX_STANDARD}) message(STATUS "Enabling support for CXX.") message(STATUS "Using CXX standard: C++${CMAKE_CXX_STANDARD}") @@ -54,4 +53,4 @@ elseif(HYPRE_WITH_SYCL) else() message(FATAL_ERROR "Neither CUDA nor HIP nor SYCL is enabled. Please enable one of them.") -endif() \ No newline at end of file +endif() diff --git a/src/config/cmake/HYPRE_SetupSYCLToolkit.cmake b/src/config/cmake/HYPRE_SetupSYCLToolkit.cmake index ed49f3826b..97738cea85 100644 --- a/src/config/cmake/HYPRE_SetupSYCLToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupSYCLToolkit.cmake @@ -36,4 +36,4 @@ endif() if (HYPRE_SYCL_TARGET_BACKEND) target_compile_options(${PROJECT_NAME} PUBLIC -fsycl-targets=${HYPRE_SYCL_TARGET_BACKEND}) target_link_options(${PROJECT_NAME} PUBLIC -fsycl-targets=${HYPRE_SYCL_TARGET_BACKEND}) -endif() \ No newline at end of file +endif() From 385951606f17e2c9ddd38536f5ee59a81696e818 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Sep 2024 23:54:20 -0400 Subject: [PATCH 16/55] Add VERSION and SOVERSION info --- src/CMakeLists.txt | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1aaf89369b..3da60ac9de 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -267,6 +267,8 @@ set(BUILD_SHARED_LIBS ${HYPRE_SHARED} CACHE INTERNAL "Build shared libraries" FO # Set output directory for the library set_target_properties(HYPRE PROPERTIES + VERSION ${HYPRE_VERSION} + SOVERSION ${HYPRE_NUMBER} ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" @@ -394,31 +396,29 @@ install(TARGETS HYPRE ) install(FILES ${HYPRE_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +# Export the package include(CMakePackageConfigHelpers) write_basic_package_version_file( - HYPREConfigVersion.cmake + "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfigVersion.cmake" VERSION ${PACKAGE_VERSION} COMPATIBILITY SameMajorVersion ) - -# Install the CMake export file -install(EXPORT HYPRETargets - FILE HYPRETargets.cmake - NAMESPACE HYPRE:: - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE -) - configure_package_config_file( - ${CMAKE_CURRENT_SOURCE_DIR}/config/HYPREConfig.cmake.in - ${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake + "${CMAKE_CURRENT_SOURCE_DIR}/config/HYPREConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" INSTALL_DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake ) - install(FILES - ${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake - ${CMAKE_CURRENT_BINARY_DIR}/HYPREConfigVersion.cmake + "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfigVersion.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE ) +# Install the CMake export file +install(EXPORT HYPRETargets + FILE HYPRETargets.cmake + NAMESPACE HYPRE:: + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE" +) # Export from build tree export(EXPORT HYPRETargets From 5b8a310c2a16d64209993f9ae6e4d0fbb56d18dd Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 18 Sep 2024 00:15:09 -0400 Subject: [PATCH 17/55] Modernize shared lib build: improve dependency linkage and CMake setup --- src/CMakeLists.txt | 36 +++++++++- src/config/HYPREConfig.cmake.in | 31 +++++++-- src/config/cmake/HYPRE_CMakeUtilities.cmake | 73 +++++++++++---------- src/test/CMakeLists.txt | 3 +- 4 files changed, 101 insertions(+), 42 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3da60ac9de..d4a0cc2a27 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -67,6 +67,12 @@ else () message(STATUS "NOTE: Could not find .git directory") endif () +# Ensure RPATH/RUNPATH is set properly during the build +set(CMAKE_SKIP_BUILD_RPATH FALSE) # Includes rpath in the binaries being built +set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) # Use different paths for build and install +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) # Use link-time paths +set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") # Where binaries can look for shared libraries + # Set cmake module path set(CMAKE_MODULE_PATH "${HYPRE_SOURCE_DIR}/config/cmake" "${CMAKE_MODULE_PATH}") include(HYPRE_CMakeUtilities) @@ -90,7 +96,7 @@ if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) endif () # Configuration options -option(HYPRE_ENABLE_SHARED "Build a shared library" OFF) +option(HYPRE_ENABLE_SHARED "Build a shared library" ${BUILD_SHARED_LIBS}) option(HYPRE_ENABLE_BIGINT "Use long long int for HYPRE_Int" OFF) option(HYPRE_ENABLE_MIXEDINT "Use long long int for HYPRE_BigInt, int for HYPRE_INT" OFF) option(HYPRE_ENABLE_SINGLE "Use float for HYPRE_Real" OFF) @@ -237,6 +243,17 @@ if (HYPRE_WITH_UMPIRE_PINNED) set(HYPRE_USING_UMPIRE_PINNED ON CACHE BOOL "" FORCE) endif () +# Ensure that HYPRE_ENABLE_SHARED and HYPRE_SHARED do not conflict with BUILD_SHARED_LIBS +if(DEFINED CACHE{BUILD_SHARED_LIBS} AND (DEFINED CACHE{HYPRE_ENABLE_SHARED} OR DEFINED CACHE{HYPRE_SHARED})) + if ((HYPRE_ENABLE_SHARED AND NOT BUILD_SHARED_LIBS) OR + (NOT HYPRE_ENABLE_SHARED AND BUILD_SHARED_LIBS)) + message(FATAL_ERROR "Incompatible options: HYPRE_ENABLE_SHARED (${HYPRE_ENABLE_SHARED}) and BUILD_SHARED_LIBS (${BUILD_SHARED_LIBS}) must have the same value.") + elseif ((HYPRE_SHARED AND NOT BUILD_SHARED_LIBS) OR + (NOT HYPRE_SHARED AND BUILD_SHARED_LIBS)) + message(FATAL_ERROR "Incompatible options: HYPRE_SHARED (${HYPRE_SHARED}) and BUILD_SHARED_LIBS (${BUILD_SHARED_LIBS}) must have the same value.") + endif() +endif() + # Set MPI compile flags if (HYPRE_WITH_MPI) configure_mpi_target() @@ -267,8 +284,8 @@ set(BUILD_SHARED_LIBS ${HYPRE_SHARED} CACHE INTERNAL "Build shared libraries" FO # Set output directory for the library set_target_properties(HYPRE PROPERTIES - VERSION ${HYPRE_VERSION} - SOVERSION ${HYPRE_NUMBER} + VERSION "${HYPRE_VERSION}" + SOVERSION "${HYPRE_VERSION}" ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" @@ -301,6 +318,10 @@ if (HYPRE_USING_FEI) target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_FEI_INCLUDE_DIRS}) endif(HYPRE_USING_FEI) +# TPL variables +set(HYPRE_DEPENDENCY_DIRS "" CACHE INTERNAL "List of absolute paths to TPL directories") +set(HYPRE_NEEDS_CXX OFF CACHE INTERNAL "Flag to indicate if C++ is needed for TPLs") + # Setup TPLs setup_tpl_or_internal(blas) setup_tpl_or_internal(lapack) @@ -310,6 +331,14 @@ setup_tpl(magma) setup_tpl(caliper) setup_tpl(umpire) +# Some TPLs need C++ to be enabled +if(HYPRE_NEEDS_CXX) + enable_language(CXX) +endif() + +# Print the directories used to hint the user about the active TPLs +message(STATUS "Dependency directories: ${HYPRE_DEPENDENCY_DIRS}") + # Configure a header file to pass CMake settings to the source code configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/config/HYPRE_config.h.cmake.in" @@ -435,6 +464,7 @@ export(PACKAGE ${PROJECT_NAME}) # Print the status of the options for debugging purposes print_option_status( + BUILD_SHARED_LIBS HYPRE_ENABLE_SHARED HYPRE_ENABLE_BIGINT HYPRE_ENABLE_MIXEDINT diff --git a/src/config/HYPREConfig.cmake.in b/src/config/HYPREConfig.cmake.in index abf6a99bfd..82c80648da 100644 --- a/src/config/HYPREConfig.cmake.in +++ b/src/config/HYPREConfig.cmake.in @@ -7,6 +7,7 @@ include(CMakeFindDependencyMacro) +set(HYPRE_DEPENDENCY_DIRS "@HYPRE_DEPENDENCY_DIRS@") set(HYPRE_ENABLE_SHARED @HYPRE_ENABLE_SHARED@) set(HYPRE_ENABLE_BIGINT @HYPRE_ENABLE_BIGINT@) set(HYPRE_ENABLE_MIXEDINT @HYPRE_ENABLE_MIXEDINT@) @@ -48,6 +49,19 @@ set(HYPRE_WITH_UMPIRE_PINNED @HYPRE_WITH_UMPIRE_PINNED@) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") +if (UNIX) + list(APPEND TPL_LIBRARIES m) +endif() + +if(HYPRE_WITH_CALIPER) + find_dependency(caliper REQUIRED HINTS ${HYPRE_DEPENDENCY_DIRS}) + if(caliper_FOUND) + message(STATUS "Caliper found: ${caliper_DIR}") + else() + message(FATAL_ERROR "Caliper not found.") + endif() +endif() + if(NOT HYPRE_ENABLE_HYPRE_BLAS) find_dependency(BLAS) endif() @@ -66,7 +80,7 @@ endif() if(HYPRE_WITH_MPI) enable_language(C) - find_dependency(MPI @MPI_C_VERSION@ EXACT COMPONENTS C) + find_dependency(MPI @MPI_C_VERSION@ COMPONENTS C) endif() if(HYPRE_WITH_OPENMP) @@ -74,9 +88,18 @@ if(HYPRE_WITH_OPENMP) endif() if(HYPRE_WITH_CUDA) - find_dependency(CUDA REQUIRED) - if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.17) - find_dependency(CUDAToolkit REQUIRED) + find_dependency(CUDAToolkit REQUIRED COMPONENTS cusparse cublas curand) + + # Diagnostic Messages + if(CUDAToolkit_FOUND) + message(STATUS "CUDAToolkit found: ${CUDAToolkit_VERSION}") + if(TARGET CUDAToolkit::cusparse AND TARGET CUDAToolkit::cublas AND TARGET CUDAToolkit::curand) + message(STATUS "Required CUDA components found: cusparse, cublas, curand.") + else() + message(FATAL_ERROR "One or more required CUDA components (cusparse, cublas, curand) not found.") + endif() + else() + message(FATAL_ERROR "CUDAToolkit not found.") endif() endif() diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index 392f3d55ec..32a71997a3 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -6,6 +6,7 @@ # Function to set conditional variables function(set_conditional_var condition var_name) if(${condition}) + #set(${var_name} ON CACHE INTERNAL "${var_name} set to ON because ${condition} is true") set(${var_name} ON CACHE BOOL "" FORCE) endif() endfunction() @@ -43,68 +44,74 @@ function(configure_mpi_target) endfunction() # Function to handle TPL (Third-Party Library) setup -function(setup_tpl NAME) - string(TOUPPER ${NAME} NAME_UPPER) - if(HYPRE_USING_${NAME_UPPER}) - if(TPL_${NAME_UPPER}_LIBRARIES AND TPL_${NAME_UPPER}_INCLUDE_DIRS) +function(setup_tpl LIBNAME) + string(TOUPPER ${LIBNAME} LIBNAME_UPPER) + if(HYPRE_USING_${LIBNAME_UPPER}) + if(TPL_${LIBNAME_UPPER}_LIBRARIES AND TPL_${LIBNAME_UPPER}_INCLUDE_DIRS) # Use specified TPL libraries and include dirs - foreach(dir ${TPL_${NAME_UPPER}_INCLUDE_DIRS}) + foreach(dir ${TPL_${LIBNAME_UPPER}_INCLUDE_DIRS}) if(NOT EXISTS ${dir}) - message(FATAL_ERROR "${NAME_UPPER} include directory not found: ${dir}") + message(FATAL_ERROR "${LIBNAME_UPPER} include directory not found: ${dir}") endif() endforeach() - foreach(lib ${TPL_${NAME_UPPER}_LIBRARIES}) + foreach(lib ${TPL_${LIBNAME_UPPER}_LIBRARIES}) if(EXISTS ${lib}) - message(STATUS "${NAME_UPPER} library found: ${lib}") + message(STATUS "${LIBNAME_UPPER} library found: ${lib}") + get_filename_component(LIB_DIR "${lib}" DIRECTORY) + set_property(TARGET ${PROJECT_NAME} APPEND PROPERTY INSTALL_RPATH "${LIB_DIR}") else() - message(WARNING "${NAME_UPPER} library not found at specified path: ${lib}") + message(WARNING "${LIBNAME_UPPER} library not found at specified path: ${lib}") endif() endforeach() - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_${NAME_UPPER}_LIBRARIES}) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_${NAME_UPPER}_INCLUDE_DIRS}) + target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_${LIBNAME_UPPER}_LIBRARIES}) + target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_${LIBNAME_UPPER}_INCLUDE_DIRS}) else() # Use find_package - find_package(${NAME} REQUIRED CONFIG) - if(${NAME}_FOUND) - message(STATUS "Found ${NAME_UPPER}") - - target_link_libraries(${PROJECT_NAME} PUBLIC ${NAME}) - target_include_directories(${PROJECT_NAME} PUBLIC ${${NAME}_INCLUDE_DIRS}) - - # Get and display information about the umpire target - get_target_property(INTERFACE_LINK_LIBRARIES ${NAME} INTERFACE_LINK_LIBRARIES) - get_target_property(INTERFACE_INCLUDE_DIRS ${NAME} INTERFACE_INCLUDE_DIRECTORIES) + find_package(${LIBNAME} REQUIRED CONFIG) + if(${LIBNAME}_FOUND) + message(STATUS "Found ${LIBNAME_UPPER}") + list(APPEND HYPRE_DEPENDENCY_DIRS "${${LIBNAME}_ROOT}") + set(HYPRE_DEPENDENCY_DIRS "${HYPRE_DEPENDENCY_DIRS}" CACHE INTERNAL "" FORCE) + + if(${LIBNAME} STREQUAL "caliper") + set(HYPRE_NEEDS_CXX TRUE PARENT_SCOPE) + endif() - message(STATUS " linked libraries: ${INTERFACE_LINK_LIBRARIES}") - message(STATUS " include directories: ${INTERFACE_INCLUDE_DIRS}") + if(TARGET ${LIBNAME}::${LIBNAME}) + target_link_libraries(${PROJECT_NAME} PUBLIC ${LIBNAME}::${LIBNAME}) + elseif(TARGET ${LIBNAME}) + target_link_libraries(${PROJECT_NAME} PUBLIC ${LIBNAME}) + else() + message(FATAL_ERROR "${LIBNAME} target not found. Please check your ${LIBNAME} installation.") + endif() else() - message(FATAL_ERROR "${NAME_UPPER} target not found. Please check your ${NAME_UPPER} installation.") + message(FATAL_ERROR "${LIBNAME_UPPER} target not found. Please check your ${LIBNAME_UPPER} installation.") endif() - if(DEFINED ${NAME}_VERSION) - message(STATUS " Version: ${${NAME}_VERSION}") + if(DEFINED ${LIBNAME}_VERSION) + message(STATUS " Version: ${${LIBNAME}_VERSION}") endif() - if(DEFINED ${NAME}_DIR) - message(STATUS " Config directory: ${${NAME}_DIR}") + if(DEFINED ${LIBNAME}_DIR) + message(STATUS " Config directory: ${${LIBNAME}_DIR}") endif() endif() - message(STATUS "Enabled support for using ${NAME_UPPER}.") + message(STATUS "Enabled support for using ${LIBNAME_UPPER}.") - if(${NAME_UPPER} STREQUAL "SUPERLU" OR ${NAME_UPPER} STREQUAL "DSUPERLU" OR ${NAME_UPPER} STREQUAL "UMPIRE") + if(${LIBNAME_UPPER} STREQUAL "SUPERLU" OR ${LIBNAME_UPPER} STREQUAL "DSUPERLU" OR ${LIBNAME_UPPER} STREQUAL "UMPIRE") target_link_libraries(${PROJECT_NAME} PUBLIC stdc++) endif() - set(${NAME_UPPER}_FOUND TRUE PARENT_SCOPE) + set(${LIBNAME_UPPER}_FOUND TRUE PARENT_SCOPE) endif() endfunction() # Function to setup TPL or internal library implementation function(setup_tpl_or_internal LIB_NAME) string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) - + if(HYPRE_USING_HYPRE_${LIB_NAME_UPPER}) # Use internal library add_subdirectory(${LIB_NAME}) @@ -260,4 +267,4 @@ function(print_option_status) # Print the footer separator message(STATUS "${separator}") message(STATUS "") -endfunction() +endfunction() \ No newline at end of file diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index d93d34de4e..aa3c67767b 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -21,5 +21,4 @@ set(TEST_SRCS ij_assembly.c ) -add_hypre_executables(TEST_SRCS) - +add_hypre_executables(TEST_SRCS) \ No newline at end of file From 6e08c7d4997fcb28ce75fedd288aabb5e33169e8 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 21 Sep 2024 00:05:16 -0400 Subject: [PATCH 18/55] Top-level CMakeLists refactoring --- src/CMakeLists.txt | 290 +++++++++--------- src/config/cmake/HYPRE_CMakeUtilities.cmake | 186 ++++++++--- src/config/cmake/HYPRE_SetupCUDAToolkit.cmake | 12 +- 3 files changed, 303 insertions(+), 185 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d4a0cc2a27..4df999b250 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,8 +4,11 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) cmake_minimum_required(VERSION 3.21) -message(STATUS "CMake Version: ${CMAKE_VERSION}") message(STATUS "CMake Executable: ${CMAKE_COMMAND}") +message(STATUS "CMake Version: ${CMAKE_VERSION}") + +# Include hypre's CMake utilities +include("${CMAKE_CURRENT_SOURCE_DIR}/config/cmake/HYPRE_CMakeUtilities.cmake") # Set cmake policies if(POLICY CMP0146) @@ -23,6 +26,16 @@ set(HYPRE_TIME 00:00:00) set(HYPRE_BUGS https://github.com/hypre-space/hypre/issues) set(HYPRE_SRCDIR "${PROJECT_SOURCE_DIR}") +# Display the hypre version +setup_git_version_info("${CMAKE_CURRENT_SOURCE_DIR}/../.git") +if (GIT_VERSION_FOUND) + message(STATUS "Hypre Version: ${HYPRE_DEVELOP_STRING} (${HYPRE_BRANCH_NAME})") +else() + message(STATUS "NOTE: Could not find .git directory") + message(STATUS "Hypre Version: ${HYPRE_VERSION}") +endif() + +# Set the project name set(PROJECT_NAME HYPRE) project(${PROJECT_NAME} VERSION ${HYPRE_VERSION} @@ -32,41 +45,13 @@ project(${PROJECT_NAME} add_library(${PROJECT_NAME}) # We use C99 by default, but users are free to specify any newer standard version -set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD 99 CACHE STRING "Standard to use for the C compiler") +# Raise an error if the source and binary directories are the same if (${HYPRE_SOURCE_DIR} STREQUAL ${HYPRE_BINARY_DIR}) message(FATAL_ERROR "In-place build not allowed! Please use a separate build directory. See the Users Manual or INSTALL file for details.") endif () -if (EXISTS ${HYPRE_SOURCE_DIR}/../.git) - execute_process(COMMAND git -C ${HYPRE_SOURCE_DIR} describe --match v* --long --abbrev=9 - OUTPUT_VARIABLE develop_string - OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND git -C ${HYPRE_SOURCE_DIR} describe --match v* --abbrev=0 - OUTPUT_VARIABLE develop_lastag - OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND git -C ${HYPRE_SOURCE_DIR} rev-list --count ${develop_lastag}..HEAD - OUTPUT_VARIABLE develop_number - OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND git -C ${HYPRE_SOURCE_DIR} rev-parse --abbrev-ref HEAD - OUTPUT_VARIABLE develop_branch - OUTPUT_STRIP_TRAILING_WHITESPACE) - if (${develop_string} MATCHES ".*") - set(HYPRE_DEVELOP_STRING ${develop_string}) - set(HYPRE_DEVELOP_NUMBER ${develop_number}) - set(HYPRE_BRANCH_NAME ${develop_branch}) - if (develop_branch MATCHES "master") - set(HYPRE_DEVELOP_BRANCH ${develop_branch}) - else () - message(STATUS "NOTE: On branch ${develop_branch}, not the main development branch") - endif () - else () - message(STATUS "NOTE: Could not describe development branch") - endif () -else () - message(STATUS "NOTE: Could not find .git directory") -endif () - # Ensure RPATH/RUNPATH is set properly during the build set(CMAKE_SKIP_BUILD_RPATH FALSE) # Includes rpath in the binaries being built set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) # Use different paths for build and install @@ -75,7 +60,6 @@ set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") # Where binaries can loo # Set cmake module path set(CMAKE_MODULE_PATH "${HYPRE_SOURCE_DIR}/config/cmake" "${CMAKE_MODULE_PATH}") -include(HYPRE_CMakeUtilities) # Set default installation directory, but provide a means for users to change set(HYPRE_INSTALL_PREFIX "${PROJECT_SOURCE_DIR}/hypre" CACHE PATH @@ -94,96 +78,95 @@ if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") endif () +message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") # Configuration options -option(HYPRE_ENABLE_SHARED "Build a shared library" ${BUILD_SHARED_LIBS}) -option(HYPRE_ENABLE_BIGINT "Use long long int for HYPRE_Int" OFF) -option(HYPRE_ENABLE_MIXEDINT "Use long long int for HYPRE_BigInt, int for HYPRE_INT" OFF) -option(HYPRE_ENABLE_SINGLE "Use float for HYPRE_Real" OFF) -option(HYPRE_ENABLE_LONG_DOUBLE "Use long double for HYPRE_Real" OFF) -option(HYPRE_ENABLE_COMPLEX "Use complex values" OFF) -option(HYPRE_ENABLE_HYPRE_BLAS "Use internal BLAS library" ON) -option(HYPRE_ENABLE_HYPRE_LAPACK "Use internal LAPACK library" ON) -option(HYPRE_ENABLE_PERSISTENT_COMM "Use persistent communication" OFF) -option(HYPRE_ENABLE_FEI "Use FEI" OFF) # TODO: Add this cmake feature -option(HYPRE_WITH_MPI "Compile with MPI" ON) -option(HYPRE_WITH_GPU_AWARE_MPI "Compile with device aware GPU support" OFF) -option(HYPRE_WITH_OPENMP "Use OpenMP" OFF) -option(HYPRE_ENABLE_HOPSCOTCH "Use hopscotch hashing with OpenMP" OFF) -option(HYPRE_WITH_SUPERLU "Use TPL SuperLU" OFF) -option(HYPRE_WITH_DSUPERLU "Use TPL SuperLU_Dist" OFF) -option(HYPRE_WITH_MAGMA "Use TPL MAGMA" OFF) -option(HYPRE_WITH_CALIPER "Use Caliper" OFF) -option(HYPRE_PRINT_ERRORS "Print HYPRE errors" OFF) -option(HYPRE_TIMING "Use HYPRE timing routines" OFF) -option(HYPRE_BUILD_EXAMPLES "Build examples" OFF) -option(HYPRE_BUILD_TESTS "Build tests" OFF) -option(HYPRE_USING_HOST_MEMORY "Use host memory" ON) +option(HYPRE_ENABLE_SHARED "Build a shared library" ${BUILD_SHARED_LIBS}) +option(HYPRE_ENABLE_BIGINT "Use long long int for HYPRE_Int" OFF) +option(HYPRE_ENABLE_MIXEDINT "Use long long int for HYPRE_BigInt, int for HYPRE_INT" OFF) +option(HYPRE_ENABLE_SINGLE "Use float for HYPRE_Real" OFF) +option(HYPRE_ENABLE_LONG_DOUBLE "Use long double for HYPRE_Real" OFF) +option(HYPRE_ENABLE_COMPLEX "Use complex values" OFF) +option(HYPRE_ENABLE_HYPRE_BLAS "Use internal BLAS library" ON) +option(HYPRE_ENABLE_HYPRE_LAPACK "Use internal LAPACK library" ON) +option(HYPRE_ENABLE_PERSISTENT_COMM "Use persistent communication" OFF) +option(HYPRE_ENABLE_FEI "Use FEI" OFF) # TODO: Add this cmake feature +option(HYPRE_WITH_MPI "Compile with MPI" ON) +option(HYPRE_WITH_GPU_AWARE_MPI "Compile with device aware GPU support" OFF) +option(HYPRE_WITH_OPENMP "Use OpenMP" OFF) +option(HYPRE_ENABLE_HOPSCOTCH "Use hopscotch hashing with OpenMP" OFF) +option(HYPRE_WITH_SUPERLU "Use TPL SuperLU" OFF) +option(HYPRE_WITH_DSUPERLU "Use TPL SuperLU_Dist" OFF) +option(HYPRE_WITH_MAGMA "Use TPL MAGMA" OFF) +option(HYPRE_WITH_CALIPER "Use Caliper" OFF) +option(HYPRE_PRINT_ERRORS "Print HYPRE errors" OFF) +option(HYPRE_TIMING "Use HYPRE timing routines" OFF) +option(HYPRE_BUILD_EXAMPLES "Build examples" OFF) +option(HYPRE_BUILD_TESTS "Build tests" OFF) +option(HYPRE_USING_HOST_MEMORY "Use host memory" ON) # GPU options -option(HYPRE_WITH_CUDA "Use CUDA. Require cuda-8.0 or higher" OFF) -option(HYPRE_WITH_HIP "Use HIP" OFF) -option(HYPRE_WITH_SYCL "Use SYCL" OFF) -option(HYPRE_ENABLE_UNIFIED_MEMORY "Use unified memory for allocating the memory" OFF) -option(HYPRE_ENABLE_DEVICE_MALLOC_ASYNC "Use device async malloc" OFF) +option(HYPRE_WITH_CUDA "Use CUDA. Require cuda-8.0 or higher" OFF) +option(HYPRE_WITH_HIP "Use HIP" OFF) +option(HYPRE_WITH_SYCL "Use SYCL" OFF) +option(HYPRE_ENABLE_UNIFIED_MEMORY "Use unified memory for allocating the memory" OFF) +option(HYPRE_ENABLE_DEVICE_MALLOC_ASYNC "Use device async malloc" OFF) # CUDA options -option(HYPRE_ENABLE_CUDA_STREAMS "Use CUDA streams" ON) -option(HYPRE_ENABLE_CUSPARSE "Use cuSPARSE" ON) -option(HYPRE_ENABLE_CUSOLVER "Use cuSOLVER" OFF) -option(HYPRE_ENABLE_DEVICE_POOL "Use device memory pool" OFF) -option(HYPRE_ENABLE_CUBLAS "Use cuBLAS" OFF) -option(HYPRE_ENABLE_CURAND "Use cuRAND" ON) -option(HYPRE_ENABLE_GPU_PROFILING "Use NVTX on CUDA" OFF) +option(HYPRE_ENABLE_CUDA_STREAMS "Use CUDA streams" ON) +option(HYPRE_ENABLE_CUSPARSE "Use cuSPARSE" ON) +option(HYPRE_ENABLE_CUSOLVER "Use cuSOLVER" OFF) +option(HYPRE_ENABLE_DEVICE_POOL "Use device memory pool" OFF) +option(HYPRE_ENABLE_CUBLAS "Use cuBLAS" OFF) +option(HYPRE_ENABLE_CURAND "Use cuRAND" ON) +option(HYPRE_ENABLE_GPU_PROFILING "Use NVTX on CUDA" OFF) # HIP options -option(HYPRE_ENABLE_ROCBLAS "Use rocBLAS" ON) -option(HYPRE_ENABLE_ROCSPARSE "Use rocSPARSE" ON) -option(HYPRE_ENABLE_ROCRAND "Use rocRAND" ON) -option(HYPRE_ENABLE_ROCSOLVER "Use rocSOLVER" ON) +option(HYPRE_ENABLE_ROCBLAS "Use rocBLAS" ON) +option(HYPRE_ENABLE_ROCSPARSE "Use rocSPARSE" ON) +option(HYPRE_ENABLE_ROCRAND "Use rocRAND" ON) +option(HYPRE_ENABLE_ROCSOLVER "Use rocSOLVER" ON) # oneAPI options -option(HYPRE_ENABLE_ONEMKLSPARSE "Use oneMKL sparse" ON) -option(HYPRE_ENABLE_ONEMKLBLAS "Use oneMKL blas" ON) -option(HYPRE_ENABLE_ONEMKLRAND "Use oneMKL rand" ON) +option(HYPRE_ENABLE_ONEMKLSPARSE "Use oneMKL sparse" ON) +option(HYPRE_ENABLE_ONEMKLBLAS "Use oneMKL blas" ON) +option(HYPRE_ENABLE_ONEMKLRAND "Use oneMKL rand" ON) # Umpire resource management options -option(HYPRE_WITH_UMPIRE "Use Umpire Allocator for device and unified memory" OFF) -option(HYPRE_WITH_UMPIRE_HOST "Use Umpire Allocator for host memory" OFF) -option(HYPRE_WITH_UMPIRE_DEVICE "Use Umpire Allocator for device memory" OFF) -option(HYPRE_WITH_UMPIRE_UM "Use Umpire Allocator for unified memory" OFF) -option(HYPRE_WITH_UMPIRE_PINNED "Use Umpire Allocator for pinned memory" OFF) +option(HYPRE_WITH_UMPIRE "Use Umpire Allocator for device and unified memory" OFF) +option(HYPRE_WITH_UMPIRE_HOST "Use Umpire Allocator for host memory" OFF) +option(HYPRE_WITH_UMPIRE_DEVICE "Use Umpire Allocator for device memory" OFF) +option(HYPRE_WITH_UMPIRE_UM "Use Umpire Allocator for unified memory" OFF) +option(HYPRE_WITH_UMPIRE_PINNED "Use Umpire Allocator for pinned memory" OFF) # TPL options -option(TPL_UMPIRE_LIBRARIES "List of absolute paths to Umpire link libraries [].") -option(TPL_UMPIRE_INCLUDE_DIRS "List of absolute paths to Umpire include directories [].") -option(TPL_SUPERLU_LIBRARIES "List of absolute paths to SuperLU link libraries [].") -option(TPL_SUPERLU_INCLUDE_DIRS "List of absolute paths to SuperLU include directories [].") -option(TPL_DSUPERLU_LIBRARIES "List of absolute paths to SuperLU_Dist link libraries [].") -option(TPL_DSUPERLU_INCLUDE_DIRS "List of absolute paths to SuperLU_Dist include directories [].") -option(TPL_MAGMA_LIBRARIES "List of absolute paths to MAGMA link libraries [].") -option(TPL_MAGMA_INCLUDE_DIRS "List of absolute paths to MAGMA include directories [].") -option(TPL_BLAS_LIBRARIES "Optional list of absolute paths to BLAS libraries, otherwise use FindBLAS to locate [].") -option(TPL_LAPACK_LIBRARIES "Optional list of absolute paths to LAPACK libraries, otherwise use FindLAPACK to locate [].") -option(TPL_FEI_INCLUDE_DIRS "List of absolute paths to FEI include directories [].") +option(TPL_UMPIRE_LIBRARIES "List of absolute paths to Umpire link libraries [].") +option(TPL_UMPIRE_INCLUDE_DIRS "List of absolute paths to Umpire include directories [].") +option(TPL_SUPERLU_LIBRARIES "List of absolute paths to SuperLU link libraries [].") +option(TPL_SUPERLU_INCLUDE_DIRS "List of absolute paths to SuperLU include directories [].") +option(TPL_DSUPERLU_LIBRARIES "List of absolute paths to SuperLU_Dist link libraries [].") +option(TPL_DSUPERLU_INCLUDE_DIRS "List of absolute paths to SuperLU_Dist include directories [].") +option(TPL_MAGMA_LIBRARIES "List of absolute paths to MAGMA link libraries [].") +option(TPL_MAGMA_INCLUDE_DIRS "List of absolute paths to MAGMA include directories [].") +option(TPL_BLAS_LIBRARIES "Optional list of absolute paths to BLAS libraries, otherwise use FindBLAS to locate [].") +option(TPL_LAPACK_LIBRARIES "Optional list of absolute paths to LAPACK libraries, otherwise use FindLAPACK to locate [].") +option(TPL_FEI_INCLUDE_DIRS "List of absolute paths to FEI include directories [].") # Extra flags -set(HYPRE_WITH_EXTRA_CFLAGS "" CACHE STRING "Define extra C compile flags") -set(HYPRE_WITH_EXTRA_CXXFLAGS "" CACHE STRING "Define extra CXX compile flags") -set(HYPRE_SYCL_TARGET "" CACHE STRING "Target SYCL architecture, e.g. 'spir64_gen'.") -set(HYPRE_SYCL_TARGET_BACKEND "" CACHE STRING "Additional SYCL backend options, e.g. '-device 12.1.0,12.4.0'.") +set(HYPRE_WITH_EXTRA_CFLAGS "" CACHE STRING "Define extra C compile flags") +set(HYPRE_WITH_EXTRA_CXXFLAGS "" CACHE STRING "Define extra CXX compile flags") +set(HYPRE_SYCL_TARGET "" CACHE STRING "Target SYCL architecture, e.g. 'spir64_gen'.") +set(HYPRE_SYCL_TARGET_BACKEND "" CACHE STRING "Additional SYCL backend options, e.g. '-device 12.1.0,12.4.0'.") # Set config name values -set_conditional_var(HYPRE_ENABLE_SHARED HYPRE_SHARED) -set_conditional_var(HYPRE_ENABLE_BIGINT HYPRE_BIGINT) -set_conditional_var(HYPRE_ENABLE_MIXEDINT HYPRE_MIXEDINT) -set_conditional_var(HYPRE_ENABLE_SINGLE HYPRE_SINGLE) -set_conditional_var(HYPRE_ENABLE_LONG_DOUBLE HYPRE_LONG_DOUBLE) -set_conditional_var(HYPRE_ENABLE_COMPLEX HYPRE_COMPLEX) -set_conditional_var(CMAKE_BUILD_TYPE STREQUAL "Debug" HYPRE_DEBUG) -set_conditional_var(HYPRE_ENABLE_HYPRE_BLAS HYPRE_USING_HYPRE_BLAS) -set_conditional_var(HYPRE_ENABLE_HYPRE_LAPACK HYPRE_USING_HYPRE_LAPACK) -set_conditional_var(HYPRE_ENABLE_PERSISTENT_COMM HYPRE_USING_PERSISTENT_COMM) -set_conditional_var(HYPRE_WITH_OPENMP HYPRE_USING_OPENMP) -set_conditional_var(HYPRE_ENABLE_HOPSCOTCH HYPRE_HOPSCOTCH) -set_conditional_var(HYPRE_WITH_CUDA HYPRE_USING_CUDA) -set_conditional_var(HYPRE_WITH_HIP HYPRE_USING_HIP) -set_conditional_var(HYPRE_WITH_SYCL HYPRE_USING_SYCL) -set_conditional_var(HYPRE_WITH_MAGMA HYPRE_USING_MAGMA) -set_conditional_var(HYPRE_WITH_CALIPER HYPRE_USING_CALIPER) +set_conditional_option(ENABLE "" SHARED) +set_conditional_option(ENABLE "" BIGINT) +set_conditional_option(ENABLE "" MIXEDINT) +set_conditional_option(ENABLE "" SINGLE) +set_conditional_option(ENABLE "" LONG_DOUBLE) +set_conditional_option(ENABLE "" COMPLEX) +set_conditional_option(ENABLE USING HYPRE_BLAS) +set_conditional_option(ENABLE USING HYPRE_LAPACK) +set_conditional_option(ENABLE "" HOPSCOTCH) +set_conditional_option(WITH USING OPENMP) +set_conditional_option(WITH USING CUDA) +set_conditional_option(WITH USING HIP) +set_conditional_option(WITH USING SYCL) +set_conditional_option(WITH USING MAGMA) +set_conditional_option(WITH USING CALIPER) if (HYPRE_WITH_MPI) set(HYPRE_HAVE_MPI ON CACHE BOOL "" FORCE) @@ -211,6 +194,7 @@ endif () # FEI doesn't currently compile with shared if (HYPRE_SHARED OR HYPRE_BIGINT OR HYPRE_SINGLE OR HYPRE_LONG_DOUBLE) set(HYPRE_USING_FEI OFF CACHE BOOL "" FORCE) + set(HYPRE_ENABLE_FEI OFF CACHE BOOL "" FORCE) endif () if (HYPRE_SEQUENTIAL) @@ -269,7 +253,7 @@ endif (HYPRE_USING_OPENMP) if (HYPRE_WITH_CUDA OR HYPRE_WITH_HIP OR HYPRE_WITH_SYCL) include(HYPRE_SetupGPUToolkit) else () - message(STATUS "No GPU support enabled.") + message(STATUS "GPU support disabled") set(HYPRE_USING_HOST_MEMORY ON CACHE BOOL "" FORCE) endif () @@ -285,7 +269,7 @@ set(BUILD_SHARED_LIBS ${HYPRE_SHARED} CACHE INTERNAL "Build shared libraries" FO # Set output directory for the library set_target_properties(HYPRE PROPERTIES VERSION "${HYPRE_VERSION}" - SOVERSION "${HYPRE_VERSION}" + SOVERSION "${HYPRE_VERSION_MAJOR}" ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" @@ -299,30 +283,12 @@ set(HYPRE_MAIN_HEADERS ) list(APPEND HYPRE_HEADERS ${HYPRE_MAIN_HEADERS}) -# Find FEI, if requested -if (HYPRE_USING_FEI) - enable_language(CXX) - - if (NOT TPL_FEI_INCLUDE_DIRS) - message(FATAL_ERROR "TPL_FEI_INCLUDE_DIRS option should be set for FEI support.") - endif () - - foreach (dir ${TPL_FEI_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "FEI include directory not found: ${dir}") - endif () - target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) - endforeach () - message(STATUS "Enabled support for using FEI.") - set(FEI_FOUND TRUE) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_FEI_INCLUDE_DIRS}) -endif(HYPRE_USING_FEI) - # TPL variables set(HYPRE_DEPENDENCY_DIRS "" CACHE INTERNAL "List of absolute paths to TPL directories") set(HYPRE_NEEDS_CXX OFF CACHE INTERNAL "Flag to indicate if C++ is needed for TPLs") # Setup TPLs +setup_fei() setup_tpl_or_internal(blas) setup_tpl_or_internal(lapack) setup_tpl(superlu) @@ -360,7 +326,7 @@ elseif (HYPRE_WITH_HIP) set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE HIP) elseif (HYPRE_WITH_SYCL) set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE SYCL) -endif () +endif() # Include directories target_include_directories(${PROJECT_NAME} PUBLIC @@ -462,8 +428,8 @@ add_library(HYPRE::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) # Export the package export(PACKAGE ${PROJECT_NAME}) -# Print the status of the options for debugging purposes -print_option_status( +# Set the base options for printing +set(BASE_OPTIONS BUILD_SHARED_LIBS HYPRE_ENABLE_SHARED HYPRE_ENABLE_BIGINT @@ -475,9 +441,57 @@ print_option_status( HYPRE_ENABLE_HYPRE_LAPACK HYPRE_ENABLE_PERSISTENT_COMM HYPRE_ENABLE_FEI + HYPRE_ENABLE_HOPSCOTCH + HYPRE_USING_GPU HYPRE_WITH_MPI HYPRE_WITH_OPENMP - HYPRE_WITH_CUDA - HYPRE_WITH_HIP - HYPRE_WITH_SYCL ) + +# Set the dependency libraries options for printing +set(TPL_OPTIONS + HYPRE_WITH_CALIPER + HYPRE_WITH_UMPIRE + HYPRE_WITH_UMPIRE_HOST + HYPRE_WITH_UMPIRE_DEVICE + HYPRE_WITH_UMPIRE_UM + HYPRE_WITH_UMPIRE_PINNED + HYPRE_WITH_MAGMA + HYPRE_WITH_DSUPERLU +) + +# Set the GPU options for printing +if(HYPRE_WITH_CUDA) + set(GPU_OPTIONS + HYPRE_WITH_CUDA + HYPRE_WITH_GPU_AWARE_MPI + HYPRE_ENABLE_CUDA_STREAMS + HYPRE_ENABLE_CUSPARSE + HYPRE_ENABLE_CUSOLVER + HYPRE_ENABLE_DEVICE_POOL + HYPRE_ENABLE_CUBLAS + HYPRE_ENABLE_CURAND + HYPRE_ENABLE_GPU_PROFILING + ) +elseif(HYPRE_WITH_HIP) + set(GPU_OPTIONS + HYPRE_WITH_HIP + HYPRE_WITH_GPU_AWARE_MPI + HYPRE_ENABLE_ROCBLAS + HYPRE_ENABLE_ROCSPARSE + HYPRE_ENABLE_ROCRAND + HYPRE_ENABLE_ROCSOLVER + HYPRE_ENABLE_GPU_PROFILING + ) +elseif(HYPRE_WITH_SYCL) + set(GPU_OPTIONS + HYPRE_WITH_SYCL + HYPRE_WITH_GPU_AWARE_MPI + HYPRE_ENABLE_ONEMKLSPARSE + HYPRE_ENABLE_ONEMKLBLAS + HYPRE_ENABLE_ONEMKLRAND + HYPRE_ENABLE_GPU_PROFILING + ) +endif() + +# Print the status of the options for debugging purposes +print_option_status(${BASE_OPTIONS} ${GPU_OPTIONS} ${TPL_OPTIONS}) diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index 32a71997a3..dfb4e35d2f 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -# Function to set conditional variables +# Function to set generic variables based on condition function(set_conditional_var condition var_name) if(${condition}) #set(${var_name} ON CACHE INTERNAL "${var_name} set to ON because ${condition} is true") @@ -11,6 +11,46 @@ function(set_conditional_var condition var_name) endif() endfunction() +# Function to set conditional hypre build options +function(set_conditional_option condition_prefix var_prefix var_name) + if(HYPRE_${condition_prefix}_${var_name}) + if(var_prefix STREQUAL "") + set(HYPRE_${var_name} ON CACHE BOOL "" FORCE) + else() + set(HYPRE_${var_prefix}_${var_name} ON CACHE BOOL "" FORCE) + endif() + endif() +endfunction() + +# Function to setup git version info +function(setup_git_version_info HYPRE_GIT_DIR) + set(GIT_VERSION_FOUND FALSE PARENT_SCOPE) + if (EXISTS "${HYPRE_GIT_DIR}") + execute_process(COMMAND git -C ${HYPRE_GIT_DIR} describe --match v* --long --abbrev=9 + OUTPUT_VARIABLE develop_string + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE git_result) + if (git_result EQUAL 0) + set(GIT_VERSION_FOUND TRUE PARENT_SCOPE) + execute_process(COMMAND git -C ${HYPRE_GIT_DIR} describe --match v* --abbrev=0 + OUTPUT_VARIABLE develop_lastag + OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND git -C ${HYPRE_GIT_DIR} rev-list --count ${develop_lastag}..HEAD + OUTPUT_VARIABLE develop_number + OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND git -C ${HYPRE_GIT_DIR} rev-parse --abbrev-ref HEAD + OUTPUT_VARIABLE develop_branch + OUTPUT_STRIP_TRAILING_WHITESPACE) + set(HYPRE_DEVELOP_STRING ${develop_string} PARENT_SCOPE) + set(HYPRE_DEVELOP_NUMBER ${develop_number} PARENT_SCOPE) + set(HYPRE_BRANCH_NAME ${develop_branch} PARENT_SCOPE) + if (develop_branch MATCHES "master") + set(HYPRE_DEVELOP_BRANCH ${develop_branch} PARENT_SCOPE) + endif () + endif() + endif() +endfunction() + # Function to configure MPI target function(configure_mpi_target) find_package(MPI REQUIRED) @@ -43,9 +83,36 @@ function(configure_mpi_target) endif () endfunction() +# Function to get dependency library version +function(get_library_version LIBNAME) + if(TARGET ${LIBNAME}::${LIBNAME}) + get_target_property(LIB_VERSION ${LIBNAME}::${LIBNAME} VERSION) + endif() + if(NOT LIB_VERSION) + if(DEFINED ${LIBNAME}_VERSION) + set(LIB_VERSION "${${LIBNAME}_VERSION}") + elseif(DEFINED ${LIBNAME}_VERSION_STRING) + set(LIB_VERSION "${${LIBNAME}_VERSION_STRING}") + elseif(DEFINED ${LIBNAME}_VERSION_MAJOR AND DEFINED ${LIBNAME}_VERSION_MINOR) + set(LIB_VERSION "${${LIBNAME}_VERSION_MAJOR}.${${LIBNAME}_VERSION_MINOR}") + if(DEFINED ${LIBNAME}_VERSION_PATCH) + set(LIB_VERSION "${LIB_VERSION}.${${LIBNAME}_VERSION_PATCH}") + endif() + endif() + endif() + if(LIB_VERSION) + message(STATUS " ${LIBNAME} version: ${LIB_VERSION}") + else() + message(STATUS " ${LIBNAME} version: unknown") + endif() +endfunction() + # Function to handle TPL (Third-Party Library) setup function(setup_tpl LIBNAME) string(TOUPPER ${LIBNAME} LIBNAME_UPPER) + + # Note we need to check for "USING" instead of "WITH" because + # we want to allow for post-processing of build options via cmake if(HYPRE_USING_${LIBNAME_UPPER}) if(TPL_${LIBNAME_UPPER}_LIBRARIES AND TPL_${LIBNAME_UPPER}_INCLUDE_DIRS) # Use specified TPL libraries and include dirs @@ -71,7 +138,7 @@ function(setup_tpl LIBNAME) # Use find_package find_package(${LIBNAME} REQUIRED CONFIG) if(${LIBNAME}_FOUND) - message(STATUS "Found ${LIBNAME_UPPER}") + message(STATUS "Found ${LIBNAME_UPPER} library") list(APPEND HYPRE_DEPENDENCY_DIRS "${${LIBNAME}_ROOT}") set(HYPRE_DEPENDENCY_DIRS "${HYPRE_DEPENDENCY_DIRS}" CACHE INTERNAL "" FORCE) @@ -84,21 +151,20 @@ function(setup_tpl LIBNAME) elseif(TARGET ${LIBNAME}) target_link_libraries(${PROJECT_NAME} PUBLIC ${LIBNAME}) else() - message(FATAL_ERROR "${LIBNAME} target not found. Please check your ${LIBNAME} installation.") + message(FATAL_ERROR "${LIBNAME} target not found. Please check your ${LIBNAME} installation") endif() else() - message(FATAL_ERROR "${LIBNAME_UPPER} target not found. Please check your ${LIBNAME_UPPER} installation.") + message(FATAL_ERROR "${LIBNAME_UPPER} target not found. Please check your ${LIBNAME_UPPER} installation") endif() - if(DEFINED ${LIBNAME}_VERSION) - message(STATUS " Version: ${${LIBNAME}_VERSION}") - endif() + # Display library info + get_library_version(${LIBNAME}) if(DEFINED ${LIBNAME}_DIR) message(STATUS " Config directory: ${${LIBNAME}_DIR}") endif() endif() - message(STATUS "Enabled support for using ${LIBNAME_UPPER}.") + message(STATUS "Enabled support for using ${LIBNAME_UPPER}") if(${LIBNAME_UPPER} STREQUAL "SUPERLU" OR ${LIBNAME_UPPER} STREQUAL "DSUPERLU" OR ${LIBNAME_UPPER} STREQUAL "UMPIRE") target_link_libraries(${PROJECT_NAME} PUBLIC stdc++) @@ -111,7 +177,7 @@ endfunction() # Function to setup TPL or internal library implementation function(setup_tpl_or_internal LIB_NAME) string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) - + if(HYPRE_USING_HYPRE_${LIB_NAME_UPPER}) # Use internal library add_subdirectory(${LIB_NAME}) @@ -150,6 +216,28 @@ function(setup_tpl_or_internal LIB_NAME) endif() endfunction() +# Function to setup FEI (to be phased out) +function(setup_fei) + if (HYPRE_USING_FEI) + set(HYPRE_NEEDS_CXX TRUE PARENT_SCOPE) + + if (NOT TPL_FEI_INCLUDE_DIRS) + message(FATAL_ERROR "TPL_FEI_INCLUDE_DIRS option should be set for FEI support.") + endif () + + foreach (dir ${TPL_FEI_INCLUDE_DIRS}) + if (NOT EXISTS ${dir}) + message(FATAL_ERROR "FEI include directory not found: ${dir}") + endif () + target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) + endforeach () + + message(STATUS "Enabled support for using FEI.") + set(FEI_FOUND TRUE PARENT_SCOPE) + target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_FEI_INCLUDE_DIRS}) + endif() +endfunction() + # A handy function to add the current source directory to a local # filename. To be used for creating a list of sources. function(convert_filenames_to_full_paths NAMES) @@ -214,6 +302,7 @@ function(add_hypre_executables EXE_SRCS) endforeach(SRC_FILE) endfunction() +# Function to print the status of build options function(print_option_status) # Define column widths set(COLUMN1_WIDTH 40) @@ -226,45 +315,56 @@ function(print_option_status) string(REPEAT "-" ${HEADER2_PAD} SEPARATOR2) set(separator "+${SEPARATOR1}+${SEPARATOR2}+") - # Create header and separator + # Function to print a block of options + function(print_option_block title options) + message(STATUS "") + message(STATUS " ${title}:") + message(STATUS " ${separator}") + message(STATUS " | Option | Status |") + message(STATUS " ${separator}") + + foreach(opt ${options}) + if(${${opt}}) + set(status "ON") + else() + set(status "OFF") + endif() + + string(LENGTH "${opt}" opt_length) + math(EXPR padding "${COLUMN1_WIDTH} - ${opt_length} - 5") + if(${padding} GREATER 0) + string(REPEAT " " ${padding} pad_spaces) + else() + set(pad_spaces "") + endif() + + string(LENGTH "${status}" status_length) + math(EXPR status_padding "${COLUMN2_WIDTH} - ${status_length} - 3") + if(${status_padding} GREATER 0) + string(REPEAT " " ${status_padding} status_pad_spaces) + else() + set(status_pad_spaces "") + endif() + + message(STATUS " | ${opt}${pad_spaces} | ${status}${status_pad_spaces} |") + endforeach() + + message(STATUS " ${separator}") + endfunction() + message(STATUS "") message(STATUS "HYPRE Configuration Summary:") - message(STATUS "${separator}") - message(STATUS "| Option | Status |") - message(STATUS "${separator}") - - # Iterate through each option and display its status - foreach(opt ${ARGN}) - # Determine the status string - if(${${opt}}) - set(status "ON") - else() - set(status "OFF") - endif() - # Calculate padding for the option name - string(LENGTH "${opt}" opt_length) - math(EXPR padding "${COLUMN1_WIDTH} - ${opt_length} - 5") # 5 accounts for "| Option | Status |" - if(${padding} GREATER 0) - string(REPEAT " " ${padding} pad_spaces) - else() - set(pad_spaces "") - endif() + # Print BASE_OPTIONS + print_option_block("Base Options" "${BASE_OPTIONS}") - # Calculate padding for the status - string(LENGTH "${status}" status_length) - math(EXPR status_padding "${COLUMN2_WIDTH} - ${status_length} - 3") # 3 accounts for "| " and space - if(${status_padding} GREATER 0) - string(REPEAT " " ${status_padding} status_pad_spaces) - else() - set(status_pad_spaces "") - endif() + # Print GPU_OPTIONS + if(HYPRE_WITH_CUDA OR HYPRE_WITH_HIP OR HYPRE_WITH_SYCL) + print_option_block("GPU Options" "${GPU_OPTIONS}") + endif() - # Print the formatted row - message(STATUS "| ${opt}${pad_spaces} | ${status}${status_pad_spaces} |") - endforeach() + # Print TPL_OPTIONS + print_option_block("Third-Party Library Options" "${TPL_OPTIONS}") - # Print the footer separator - message(STATUS "${separator}") message(STATUS "") endfunction() \ No newline at end of file diff --git a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake index 326d0e6225..996341cff4 100644 --- a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake @@ -6,7 +6,11 @@ message(STATUS "Enabling CUDA toolkit") # Check for CUDA_PATH, CUDA_HOME or CUDA_DIR -if(DEFINED CUDA_DIR) +if(DEFINED CUDAToolkit_ROOT) + set(CUDA_DIR ${CUDAToolkit_ROOT}) +elseif(DEFINED ENV{CUDAToolkit_ROOT}) + set(CUDA_DIR $ENV{CUDAToolkit_ROOT}) +elseif(DEFINED CUDA_DIR) set(CUDA_DIR ${CUDA_DIR}) elseif(DEFINED ENV{CUDA_DIR}) set(CUDA_DIR $ENV{CUDA_DIR}) @@ -144,20 +148,20 @@ function(find_and_add_cuda_library LIB_NAME HYPRE_ENABLE_VAR) find_package(CUDAToolkit REQUIRED COMPONENTS ${LIB_NAME}) if(TARGET CUDAToolkit::${LIB_NAME}) - message(STATUS "Found ${LIB_NAME} library: ${${LIB_NAME}_LIBRARY}") + message(STATUS "Found ${LIB_NAME_UPPER} library: ${${LIB_NAME}_LIBRARY}") list(APPEND CUDA_LIBS CUDAToolkit::${LIB_NAME}) else() #message(WARNING "CUDAToolkit::${LIB_NAME} target not found. Attempting manual linking.") find_library(${LIB_NAME}_LIBRARY ${LIB_NAME} HINTS ${CUDAToolkit_LIBRARY_DIR}) if(${LIB_NAME}_LIBRARY) - message(STATUS "Found ${LIB_NAME} library: ${${LIB_NAME}_LIBRARY}") + message(STATUS "Found ${LIB_NAME_UPPER} library: ${${LIB_NAME}_LIBRARY}") add_library(CUDAToolkit::${LIB_NAME} UNKNOWN IMPORTED) set_target_properties(CUDAToolkit::${LIB_NAME} PROPERTIES IMPORTED_LOCATION "${${LIB_NAME}_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${CUDAToolkit_INCLUDE_DIRS}") list(APPEND CUDA_LIBS CUDAToolkit::${LIB_NAME}) else() - message(FATAL_ERROR "Could not find ${LIB_NAME} library. Please check your CUDA installation.") + message(FATAL_ERROR "Could not find ${LIB_NAME_UPPER} library. Please check your CUDA installation.") endif() endif() From fcb58e62864b2473ede3314e52d378647c6d4250 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 21 Sep 2024 01:07:16 -0400 Subject: [PATCH 19/55] Simplify build with set_hypre_option --- src/CMakeLists.txt | 219 +++++++------------- src/config/cmake/HYPRE_CMakeUtilities.cmake | 16 +- 2 files changed, 91 insertions(+), 144 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4df999b250..636c36d8bb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -80,71 +80,71 @@ if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) endif () message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") -# Configuration options -option(HYPRE_ENABLE_SHARED "Build a shared library" ${BUILD_SHARED_LIBS}) -option(HYPRE_ENABLE_BIGINT "Use long long int for HYPRE_Int" OFF) -option(HYPRE_ENABLE_MIXEDINT "Use long long int for HYPRE_BigInt, int for HYPRE_INT" OFF) -option(HYPRE_ENABLE_SINGLE "Use float for HYPRE_Real" OFF) -option(HYPRE_ENABLE_LONG_DOUBLE "Use long double for HYPRE_Real" OFF) -option(HYPRE_ENABLE_COMPLEX "Use complex values" OFF) -option(HYPRE_ENABLE_HYPRE_BLAS "Use internal BLAS library" ON) -option(HYPRE_ENABLE_HYPRE_LAPACK "Use internal LAPACK library" ON) -option(HYPRE_ENABLE_PERSISTENT_COMM "Use persistent communication" OFF) -option(HYPRE_ENABLE_FEI "Use FEI" OFF) # TODO: Add this cmake feature -option(HYPRE_WITH_MPI "Compile with MPI" ON) -option(HYPRE_WITH_GPU_AWARE_MPI "Compile with device aware GPU support" OFF) -option(HYPRE_WITH_OPENMP "Use OpenMP" OFF) -option(HYPRE_ENABLE_HOPSCOTCH "Use hopscotch hashing with OpenMP" OFF) -option(HYPRE_WITH_SUPERLU "Use TPL SuperLU" OFF) -option(HYPRE_WITH_DSUPERLU "Use TPL SuperLU_Dist" OFF) -option(HYPRE_WITH_MAGMA "Use TPL MAGMA" OFF) -option(HYPRE_WITH_CALIPER "Use Caliper" OFF) -option(HYPRE_PRINT_ERRORS "Print HYPRE errors" OFF) -option(HYPRE_TIMING "Use HYPRE timing routines" OFF) -option(HYPRE_BUILD_EXAMPLES "Build examples" OFF) -option(HYPRE_BUILD_TESTS "Build tests" OFF) -option(HYPRE_USING_HOST_MEMORY "Use host memory" ON) +# Base configuration options +set_hypre_option(BASE HYPRE_ENABLE_SHARED "Build a shared library" ${BUILD_SHARED_LIBS}) +set_hypre_option(BASE HYPRE_ENABLE_BIGINT "Use long long int for HYPRE_Int" OFF) +set_hypre_option(BASE HYPRE_ENABLE_MIXEDINT "Use long long int for HYPRE_BigInt, int for HYPRE_INT" OFF) +set_hypre_option(BASE HYPRE_ENABLE_SINGLE "Use float for HYPRE_Real" OFF) +set_hypre_option(BASE HYPRE_ENABLE_LONG_DOUBLE "Use long double for HYPRE_Real" OFF) +set_hypre_option(BASE HYPRE_ENABLE_COMPLEX "Use complex values" OFF) +set_hypre_option(BASE HYPRE_ENABLE_INTERNAL_BLAS "Use internal BLAS library" ON) +set_hypre_option(BASE HYPRE_ENABLE_INTERNAL_LAPACK "Use internal LAPACK library" ON) +set_hypre_option(BASE HYPRE_ENABLE_PERSISTENT_COMM "Use persistent communication" OFF) +set_hypre_option(BASE HYPRE_ENABLE_FEI "Use FEI" OFF) # TODO: Add this cmake feature +set_hypre_option(BASE HYPRE_WITH_MPI "Compile with MPI" ON) +set_hypre_option(BASE HYPRE_WITH_GPU_AWARE_MPI "Compile with device aware GPU support" OFF) +set_hypre_option(BASE HYPRE_WITH_OPENMP "Use OpenMP" OFF) +set_hypre_option(BASE HYPRE_ENABLE_HOPSCOTCH "Use hopscotch hashing with OpenMP" OFF) +set_hypre_option(BASE HYPRE_PRINT_ERRORS "Print HYPRE errors" OFF) +set_hypre_option(BASE HYPRE_TIMING "Use HYPRE timing routines" OFF) +set_hypre_option(BASE HYPRE_BUILD_EXAMPLES "Build examples" OFF) +set_hypre_option(BASE HYPRE_BUILD_TESTS "Build tests" OFF) +set_hypre_option(BASE HYPRE_USING_HOST_MEMORY "Use host memory" ON) # GPU options -option(HYPRE_WITH_CUDA "Use CUDA. Require cuda-8.0 or higher" OFF) -option(HYPRE_WITH_HIP "Use HIP" OFF) -option(HYPRE_WITH_SYCL "Use SYCL" OFF) -option(HYPRE_ENABLE_UNIFIED_MEMORY "Use unified memory for allocating the memory" OFF) -option(HYPRE_ENABLE_DEVICE_MALLOC_ASYNC "Use device async malloc" OFF) +set_hypre_option(GPU HYPRE_WITH_CUDA "Use CUDA. Require cuda-8.0 or higher" OFF) +set_hypre_option(GPU HYPRE_WITH_HIP "Use HIP" OFF) +set_hypre_option(GPU HYPRE_WITH_SYCL "Use SYCL" OFF) +set_hypre_option(GPU HYPRE_ENABLE_UNIFIED_MEMORY "Use unified memory for allocating the memory" OFF) +set_hypre_option(GPU HYPRE_ENABLE_DEVICE_MALLOC_ASYNC "Use device async malloc" OFF) # CUDA options -option(HYPRE_ENABLE_CUDA_STREAMS "Use CUDA streams" ON) -option(HYPRE_ENABLE_CUSPARSE "Use cuSPARSE" ON) -option(HYPRE_ENABLE_CUSOLVER "Use cuSOLVER" OFF) -option(HYPRE_ENABLE_DEVICE_POOL "Use device memory pool" OFF) -option(HYPRE_ENABLE_CUBLAS "Use cuBLAS" OFF) -option(HYPRE_ENABLE_CURAND "Use cuRAND" ON) -option(HYPRE_ENABLE_GPU_PROFILING "Use NVTX on CUDA" OFF) +set_hypre_option(CUDA HYPRE_ENABLE_CUDA_STREAMS "Use CUDA streams" ON) +set_hypre_option(CUDA HYPRE_ENABLE_CUSPARSE "Use cuSPARSE" ON) +set_hypre_option(CUDA HYPRE_ENABLE_CUSOLVER "Use cuSOLVER" OFF) +set_hypre_option(CUDA HYPRE_ENABLE_DEVICE_POOL "Use device memory pool" OFF) +set_hypre_option(CUDA HYPRE_ENABLE_CUBLAS "Use cuBLAS" OFF) +set_hypre_option(CUDA HYPRE_ENABLE_CURAND "Use cuRAND" ON) +set_hypre_option(CUDA HYPRE_ENABLE_GPU_PROFILING "Use NVTX on CUDA" OFF) # HIP options -option(HYPRE_ENABLE_ROCBLAS "Use rocBLAS" ON) -option(HYPRE_ENABLE_ROCSPARSE "Use rocSPARSE" ON) -option(HYPRE_ENABLE_ROCRAND "Use rocRAND" ON) -option(HYPRE_ENABLE_ROCSOLVER "Use rocSOLVER" ON) +set_hypre_option(HIP HYPRE_ENABLE_ROCBLAS "Use rocBLAS" ON) +set_hypre_option(HIP HYPRE_ENABLE_ROCSPARSE "Use rocSPARSE" ON) +set_hypre_option(HIP HYPRE_ENABLE_ROCRAND "Use rocRAND" ON) +set_hypre_option(HIP HYPRE_ENABLE_ROCSOLVER "Use rocSOLVER" ON) # oneAPI options -option(HYPRE_ENABLE_ONEMKLSPARSE "Use oneMKL sparse" ON) -option(HYPRE_ENABLE_ONEMKLBLAS "Use oneMKL blas" ON) -option(HYPRE_ENABLE_ONEMKLRAND "Use oneMKL rand" ON) -# Umpire resource management options -option(HYPRE_WITH_UMPIRE "Use Umpire Allocator for device and unified memory" OFF) -option(HYPRE_WITH_UMPIRE_HOST "Use Umpire Allocator for host memory" OFF) -option(HYPRE_WITH_UMPIRE_DEVICE "Use Umpire Allocator for device memory" OFF) -option(HYPRE_WITH_UMPIRE_UM "Use Umpire Allocator for unified memory" OFF) -option(HYPRE_WITH_UMPIRE_PINNED "Use Umpire Allocator for pinned memory" OFF) +set_hypre_option(SYCL HYPRE_ENABLE_ONEMKLSPARSE "Use oneMKL sparse" ON) +set_hypre_option(SYCL HYPRE_ENABLE_ONEMKLBLAS "Use oneMKL blas" ON) +set_hypre_option(SYCL HYPRE_ENABLE_ONEMKLRAND "Use oneMKL rand" ON) # TPL options -option(TPL_UMPIRE_LIBRARIES "List of absolute paths to Umpire link libraries [].") -option(TPL_UMPIRE_INCLUDE_DIRS "List of absolute paths to Umpire include directories [].") -option(TPL_SUPERLU_LIBRARIES "List of absolute paths to SuperLU link libraries [].") -option(TPL_SUPERLU_INCLUDE_DIRS "List of absolute paths to SuperLU include directories [].") -option(TPL_DSUPERLU_LIBRARIES "List of absolute paths to SuperLU_Dist link libraries [].") -option(TPL_DSUPERLU_INCLUDE_DIRS "List of absolute paths to SuperLU_Dist include directories [].") -option(TPL_MAGMA_LIBRARIES "List of absolute paths to MAGMA link libraries [].") -option(TPL_MAGMA_INCLUDE_DIRS "List of absolute paths to MAGMA include directories [].") -option(TPL_BLAS_LIBRARIES "Optional list of absolute paths to BLAS libraries, otherwise use FindBLAS to locate [].") -option(TPL_LAPACK_LIBRARIES "Optional list of absolute paths to LAPACK libraries, otherwise use FindLAPACK to locate [].") -option(TPL_FEI_INCLUDE_DIRS "List of absolute paths to FEI include directories [].") +set_hypre_option(TPL HYPRE_WITH_UMPIRE "Use Umpire Allocator for device and unified memory" OFF) +set_hypre_option(TPL HYPRE_WITH_UMPIRE_HOST "Use Umpire Allocator for host memory" OFF) +set_hypre_option(TPL HYPRE_WITH_UMPIRE_PINNED "Use Umpire Allocator for pinned memory" OFF) +set_hypre_option(TPL HYPRE_WITH_UMPIRE_DEVICE "Use Umpire Allocator for device memory" OFF) +set_hypre_option(TPL HYPRE_WITH_UMPIRE_UM "Use Umpire Allocator for unified memory" OFF) +set_hypre_option(TPL HYPRE_WITH_SUPERLU "Use TPL SuperLU" OFF) +set_hypre_option(TPL HYPRE_WITH_DSUPERLU "Use TPL SuperLU_Dist" OFF) +set_hypre_option(TPL HYPRE_WITH_MAGMA "Use TPL MAGMA" OFF) +set_hypre_option(SKIP TPL_UMPIRE_LIBRARIES "Optional list of absolute paths to Umpire link libraries []." "") +set_hypre_option(SKIP TPL_UMPIRE_INCLUDE_DIRS "Optional list of absolute paths to Umpire include directories []." "") +set_hypre_option(SKIP TPL_SUPERLU_LIBRARIES "Optional list of absolute paths to SuperLU link libraries []." "") +set_hypre_option(SKIP TPL_SUPERLU_INCLUDE_DIRS "Optional list of absolute paths to SuperLU include directories []." "") +set_hypre_option(SKIP TPL_DSUPERLU_LIBRARIES "Optional list of absolute paths to SuperLU_Dist link libraries []." "") +set_hypre_option(SKIP TPL_DSUPERLU_INCLUDE_DIRS "Optional list of absolute paths to SuperLU_Dist include directories []." "") +set_hypre_option(SKIP TPL_MAGMA_LIBRARIES "Optional list of absolute paths to MAGMA link libraries []." "") +set_hypre_option(SKIP TPL_MAGMA_INCLUDE_DIRS "Optional list of absolute paths to MAGMA include directories []." "") +set_hypre_option(SKIP TPL_CALIPER_LIBRARIES "Optional list of absolute paths to Caliper link libraries []." "") +set_hypre_option(SKIP TPL_CALIPER_INCLUDE_DIRS "Optional list of absolute paths to Caliper include directories []." "") +set_hypre_option(SKIP TPL_BLAS_LIBRARIES "Optional list of absolute paths to BLAS libraries, otherwise use FindBLAS to locate []." "") +set_hypre_option(SKIP TPL_LAPACK_LIBRARIES "Optional list of absolute paths to LAPACK libraries, otherwise use FindLAPACK to locate []." "") +set_hypre_option(SKIP TPL_FEI_INCLUDE_DIRS "Optional list of absolute paths to FEI include directories []." "") # Extra flags set(HYPRE_WITH_EXTRA_CFLAGS "" CACHE STRING "Define extra C compile flags") set(HYPRE_WITH_EXTRA_CXXFLAGS "" CACHE STRING "Define extra CXX compile flags") @@ -152,21 +152,21 @@ set(HYPRE_SYCL_TARGET "" CACHE STRING "Target SYCL architectu set(HYPRE_SYCL_TARGET_BACKEND "" CACHE STRING "Additional SYCL backend options, e.g. '-device 12.1.0,12.4.0'.") # Set config name values -set_conditional_option(ENABLE "" SHARED) -set_conditional_option(ENABLE "" BIGINT) -set_conditional_option(ENABLE "" MIXEDINT) -set_conditional_option(ENABLE "" SINGLE) -set_conditional_option(ENABLE "" LONG_DOUBLE) -set_conditional_option(ENABLE "" COMPLEX) -set_conditional_option(ENABLE USING HYPRE_BLAS) -set_conditional_option(ENABLE USING HYPRE_LAPACK) -set_conditional_option(ENABLE "" HOPSCOTCH) -set_conditional_option(WITH USING OPENMP) -set_conditional_option(WITH USING CUDA) -set_conditional_option(WITH USING HIP) -set_conditional_option(WITH USING SYCL) -set_conditional_option(WITH USING MAGMA) -set_conditional_option(WITH USING CALIPER) +set_conditional_hypre_option(ENABLE "" SHARED) +set_conditional_hypre_option(ENABLE "" BIGINT) +set_conditional_hypre_option(ENABLE "" MIXEDINT) +set_conditional_hypre_option(ENABLE "" SINGLE) +set_conditional_hypre_option(ENABLE "" LONG_DOUBLE) +set_conditional_hypre_option(ENABLE "" COMPLEX) +set_conditional_hypre_option(ENABLE USING HYPRE_BLAS) +set_conditional_hypre_option(ENABLE USING HYPRE_LAPACK) +set_conditional_hypre_option(ENABLE "" HOPSCOTCH) +set_conditional_hypre_option(WITH USING OPENMP) +set_conditional_hypre_option(WITH USING CUDA) +set_conditional_hypre_option(WITH USING HIP) +set_conditional_hypre_option(WITH USING SYCL) +set_conditional_hypre_option(WITH USING MAGMA) +set_conditional_hypre_option(WITH USING CALIPER) if (HYPRE_WITH_MPI) set(HYPRE_HAVE_MPI ON CACHE BOOL "" FORCE) @@ -428,70 +428,5 @@ add_library(HYPRE::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) # Export the package export(PACKAGE ${PROJECT_NAME}) -# Set the base options for printing -set(BASE_OPTIONS - BUILD_SHARED_LIBS - HYPRE_ENABLE_SHARED - HYPRE_ENABLE_BIGINT - HYPRE_ENABLE_MIXEDINT - HYPRE_ENABLE_SINGLE - HYPRE_ENABLE_LONG_DOUBLE - HYPRE_ENABLE_COMPLEX - HYPRE_ENABLE_HYPRE_BLAS - HYPRE_ENABLE_HYPRE_LAPACK - HYPRE_ENABLE_PERSISTENT_COMM - HYPRE_ENABLE_FEI - HYPRE_ENABLE_HOPSCOTCH - HYPRE_USING_GPU - HYPRE_WITH_MPI - HYPRE_WITH_OPENMP -) - -# Set the dependency libraries options for printing -set(TPL_OPTIONS - HYPRE_WITH_CALIPER - HYPRE_WITH_UMPIRE - HYPRE_WITH_UMPIRE_HOST - HYPRE_WITH_UMPIRE_DEVICE - HYPRE_WITH_UMPIRE_UM - HYPRE_WITH_UMPIRE_PINNED - HYPRE_WITH_MAGMA - HYPRE_WITH_DSUPERLU -) - -# Set the GPU options for printing -if(HYPRE_WITH_CUDA) - set(GPU_OPTIONS - HYPRE_WITH_CUDA - HYPRE_WITH_GPU_AWARE_MPI - HYPRE_ENABLE_CUDA_STREAMS - HYPRE_ENABLE_CUSPARSE - HYPRE_ENABLE_CUSOLVER - HYPRE_ENABLE_DEVICE_POOL - HYPRE_ENABLE_CUBLAS - HYPRE_ENABLE_CURAND - HYPRE_ENABLE_GPU_PROFILING - ) -elseif(HYPRE_WITH_HIP) - set(GPU_OPTIONS - HYPRE_WITH_HIP - HYPRE_WITH_GPU_AWARE_MPI - HYPRE_ENABLE_ROCBLAS - HYPRE_ENABLE_ROCSPARSE - HYPRE_ENABLE_ROCRAND - HYPRE_ENABLE_ROCSOLVER - HYPRE_ENABLE_GPU_PROFILING - ) -elseif(HYPRE_WITH_SYCL) - set(GPU_OPTIONS - HYPRE_WITH_SYCL - HYPRE_WITH_GPU_AWARE_MPI - HYPRE_ENABLE_ONEMKLSPARSE - HYPRE_ENABLE_ONEMKLBLAS - HYPRE_ENABLE_ONEMKLRAND - HYPRE_ENABLE_GPU_PROFILING - ) -endif() - -# Print the status of the options for debugging purposes -print_option_status(${BASE_OPTIONS} ${GPU_OPTIONS} ${TPL_OPTIONS}) +# Print status of the build options +print_option_status(${BASE_OPTIONS} ${GPU_OPTIONS} ${TPL_OPTIONS}) \ No newline at end of file diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index dfb4e35d2f..6ce127dd7f 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -3,6 +3,18 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Function to set hypre build options +function(set_hypre_option category name description default_value) + option(${name} "${description}" ${default_value}) + if (${category} STREQUAL "CUDA" OR ${category} STREQUAL "HIP" OR ${category} STREQUAL "SYCL") + if (HYPRE_WITH_${category} STREQUAL "ON") + set(GPU_OPTIONS ${GPU_OPTIONS} ${name} PARENT_SCOPE) + endif() + else() + set(${category}_OPTIONS ${${category}_OPTIONS} ${name} PARENT_SCOPE) + endif() +endfunction() + # Function to set generic variables based on condition function(set_conditional_var condition var_name) if(${condition}) @@ -12,7 +24,7 @@ function(set_conditional_var condition var_name) endfunction() # Function to set conditional hypre build options -function(set_conditional_option condition_prefix var_prefix var_name) +function(set_conditional_hypre_option condition_prefix var_prefix var_name) if(HYPRE_${condition_prefix}_${var_name}) if(var_prefix STREQUAL "") set(HYPRE_${var_name} ON CACHE BOOL "" FORCE) @@ -178,7 +190,7 @@ endfunction() function(setup_tpl_or_internal LIB_NAME) string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) - if(HYPRE_USING_HYPRE_${LIB_NAME_UPPER}) + if(HYPRE_USING_INTERNAL_${LIB_NAME_UPPER}) # Use internal library add_subdirectory(${LIB_NAME}) message(STATUS "Using internal ${LIB_NAME_UPPER}") From 14f58d8d4f7000406edb8e783176e0dd13f54f13 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 21 Sep 2024 21:46:58 -0400 Subject: [PATCH 20/55] Add ensure_options_match/differ macros --- src/CMakeLists.txt | 87 +++++++++++--------- src/config/cmake/HYPRE_CMakeUtilities.cmake | 50 ++++++++--- src/config/cmake/HYPRE_SetupGPUToolkit.cmake | 4 - 3 files changed, 84 insertions(+), 57 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 636c36d8bb..28c929f6cd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -62,11 +62,17 @@ set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") # Where binaries can loo set(CMAKE_MODULE_PATH "${HYPRE_SOURCE_DIR}/config/cmake" "${CMAKE_MODULE_PATH}") # Set default installation directory, but provide a means for users to change -set(HYPRE_INSTALL_PREFIX "${PROJECT_SOURCE_DIR}/hypre" CACHE PATH - "Installation directory for HYPRE") -if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - set(CMAKE_INSTALL_PREFIX "${HYPRE_INSTALL_PREFIX}" CACHE INTERNAL "" FORCE) -endif () +if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT OR NOT DEFINED HYPRE_INSTALL_PREFIX) + if (NOT DEFINED HYPRE_INSTALL_PREFIX) + set(CMAKE_INSTALL_PREFIX "${PROJECT_SOURCE_DIR}/hypre" CACHE PATH "Installation directory for HYPRE" FORCE) + else() + set(CMAKE_INSTALL_PREFIX "${HYPRE_INSTALL_PREFIX}" CACHE PATH "Installation directory for HYPRE" FORCE) + endif() +else() + if (NOT DEFINED HYPRE_INSTALL_PREFIX) + set(HYPRE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE PATH "Installation directory for HYPRE" FORCE) + endif() +endif() # Set default compile optimization flag set(HYPRE_BUILD_TYPE "Release" CACHE STRING @@ -80,7 +86,7 @@ if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) endif () message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") -# Base configuration options +# Base boolean configuration options set_hypre_option(BASE HYPRE_ENABLE_SHARED "Build a shared library" ${BUILD_SHARED_LIBS}) set_hypre_option(BASE HYPRE_ENABLE_BIGINT "Use long long int for HYPRE_Int" OFF) set_hypre_option(BASE HYPRE_ENABLE_MIXEDINT "Use long long int for HYPRE_BigInt, int for HYPRE_INT" OFF) @@ -91,12 +97,11 @@ set_hypre_option(BASE HYPRE_ENABLE_INTERNAL_BLAS "Use internal BLAS libra set_hypre_option(BASE HYPRE_ENABLE_INTERNAL_LAPACK "Use internal LAPACK library" ON) set_hypre_option(BASE HYPRE_ENABLE_PERSISTENT_COMM "Use persistent communication" OFF) set_hypre_option(BASE HYPRE_ENABLE_FEI "Use FEI" OFF) # TODO: Add this cmake feature -set_hypre_option(BASE HYPRE_WITH_MPI "Compile with MPI" ON) -set_hypre_option(BASE HYPRE_WITH_GPU_AWARE_MPI "Compile with device aware GPU support" OFF) -set_hypre_option(BASE HYPRE_WITH_OPENMP "Use OpenMP" OFF) set_hypre_option(BASE HYPRE_ENABLE_HOPSCOTCH "Use hopscotch hashing with OpenMP" OFF) +set_hypre_option(BASE HYPRE_WITH_OPENMP "Use OpenMP" OFF) +set_hypre_option(BASE HYPRE_WITH_MPI "Compile with MPI" ON) set_hypre_option(BASE HYPRE_PRINT_ERRORS "Print HYPRE errors" OFF) -set_hypre_option(BASE HYPRE_TIMING "Use HYPRE timing routines" OFF) +set_hypre_option(SKIP HYPRE_TIMING "Use HYPRE timing routines" OFF) set_hypre_option(BASE HYPRE_BUILD_EXAMPLES "Build examples" OFF) set_hypre_option(BASE HYPRE_BUILD_TESTS "Build tests" OFF) set_hypre_option(BASE HYPRE_USING_HOST_MEMORY "Use host memory" ON) @@ -104,6 +109,7 @@ set_hypre_option(BASE HYPRE_USING_HOST_MEMORY "Use host memory" ON) set_hypre_option(GPU HYPRE_WITH_CUDA "Use CUDA. Require cuda-8.0 or higher" OFF) set_hypre_option(GPU HYPRE_WITH_HIP "Use HIP" OFF) set_hypre_option(GPU HYPRE_WITH_SYCL "Use SYCL" OFF) +set_hypre_option(GPU HYPRE_WITH_GPU_AWARE_MPI "Compile with device aware GPU support" OFF) set_hypre_option(GPU HYPRE_ENABLE_UNIFIED_MEMORY "Use unified memory for allocating the memory" OFF) set_hypre_option(GPU HYPRE_ENABLE_DEVICE_MALLOC_ASYNC "Use device async malloc" OFF) # CUDA options @@ -152,21 +158,23 @@ set(HYPRE_SYCL_TARGET "" CACHE STRING "Target SYCL architectu set(HYPRE_SYCL_TARGET_BACKEND "" CACHE STRING "Additional SYCL backend options, e.g. '-device 12.1.0,12.4.0'.") # Set config name values -set_conditional_hypre_option(ENABLE "" SHARED) -set_conditional_hypre_option(ENABLE "" BIGINT) -set_conditional_hypre_option(ENABLE "" MIXEDINT) -set_conditional_hypre_option(ENABLE "" SINGLE) -set_conditional_hypre_option(ENABLE "" LONG_DOUBLE) -set_conditional_hypre_option(ENABLE "" COMPLEX) -set_conditional_hypre_option(ENABLE USING HYPRE_BLAS) -set_conditional_hypre_option(ENABLE USING HYPRE_LAPACK) -set_conditional_hypre_option(ENABLE "" HOPSCOTCH) -set_conditional_hypre_option(WITH USING OPENMP) -set_conditional_hypre_option(WITH USING CUDA) -set_conditional_hypre_option(WITH USING HIP) -set_conditional_hypre_option(WITH USING SYCL) -set_conditional_hypre_option(WITH USING MAGMA) -set_conditional_hypre_option(WITH USING CALIPER) +set_internal_hypre_option(ENABLE "" SHARED) +set_internal_hypre_option(ENABLE "" BIGINT) +set_internal_hypre_option(ENABLE "" MIXEDINT) +set_internal_hypre_option(ENABLE "" SINGLE) +set_internal_hypre_option(ENABLE "" LONG_DOUBLE) +set_internal_hypre_option(ENABLE "" COMPLEX) +set_internal_hypre_option(ENABLE USING HYPRE_BLAS) +set_internal_hypre_option(ENABLE USING HYPRE_LAPACK) +set_internal_hypre_option(ENABLE "" HOPSCOTCH) +set_internal_hypre_option(ENABLE USING GPU_STREAMS) +set_internal_hypre_option(ENABLE USING DEVICE_POOL) +set_internal_hypre_option(WITH USING OPENMP) +set_internal_hypre_option(WITH USING CUDA) +set_internal_hypre_option(WITH USING HIP) +set_internal_hypre_option(WITH USING SYCL) +set_internal_hypre_option(WITH USING MAGMA) +set_internal_hypre_option(WITH USING CALIPER) if (HYPRE_WITH_MPI) set(HYPRE_HAVE_MPI ON CACHE BOOL "" FORCE) @@ -227,16 +235,13 @@ if (HYPRE_WITH_UMPIRE_PINNED) set(HYPRE_USING_UMPIRE_PINNED ON CACHE BOOL "" FORCE) endif () -# Ensure that HYPRE_ENABLE_SHARED and HYPRE_SHARED do not conflict with BUILD_SHARED_LIBS -if(DEFINED CACHE{BUILD_SHARED_LIBS} AND (DEFINED CACHE{HYPRE_ENABLE_SHARED} OR DEFINED CACHE{HYPRE_SHARED})) - if ((HYPRE_ENABLE_SHARED AND NOT BUILD_SHARED_LIBS) OR - (NOT HYPRE_ENABLE_SHARED AND BUILD_SHARED_LIBS)) - message(FATAL_ERROR "Incompatible options: HYPRE_ENABLE_SHARED (${HYPRE_ENABLE_SHARED}) and BUILD_SHARED_LIBS (${BUILD_SHARED_LIBS}) must have the same value.") - elseif ((HYPRE_SHARED AND NOT BUILD_SHARED_LIBS) OR - (NOT HYPRE_SHARED AND BUILD_SHARED_LIBS)) - message(FATAL_ERROR "Incompatible options: HYPRE_SHARED (${HYPRE_SHARED}) and BUILD_SHARED_LIBS (${BUILD_SHARED_LIBS}) must have the same value.") - endif() -endif() +# Check for conflicting options +ensure_options_match(HYPRE_ENABLE_SHARED BUILD_SHARED_LIBS) +ensure_options_match(HYPRE_INSTALL_PREFIX CMAKE_INSTALL_PREFIX) +ensure_options_match(HYPRE_BUILD_TYPE CMAKE_BUILD_TYPE) +ensure_options_differ(HYPRE_WITH_CUDA HYPRE_WITH_HIP) +ensure_options_differ(HYPRE_WITH_CUDA HYPRE_WITH_SYCL) +ensure_options_differ(HYPRE_WITH_HIP HYPRE_WITH_SYCL) # Set MPI compile flags if (HYPRE_WITH_MPI) @@ -249,11 +254,11 @@ if (HYPRE_USING_OPENMP) target_link_libraries(${PROJECT_NAME} PUBLIC OpenMP::OpenMP_C) endif (HYPRE_USING_OPENMP) -# Set GPU compile flags +# Setup GPU build options: CUDA, HIP, SYCL are mutually exclusive if (HYPRE_WITH_CUDA OR HYPRE_WITH_HIP OR HYPRE_WITH_SYCL) include(HYPRE_SetupGPUToolkit) else () - message(STATUS "GPU support disabled") + message(STATUS "GPU support not enabled") set(HYPRE_USING_HOST_MEMORY ON CACHE BOOL "" FORCE) endif () @@ -284,10 +289,10 @@ set(HYPRE_MAIN_HEADERS list(APPEND HYPRE_HEADERS ${HYPRE_MAIN_HEADERS}) # TPL variables -set(HYPRE_DEPENDENCY_DIRS "" CACHE INTERNAL "List of absolute paths to TPL directories") +set(HYPRE_DEPENDENCY_DIRS "" CACHE INTERNAL "List of absolute paths to third-party libraries (TPLs) directories") set(HYPRE_NEEDS_CXX OFF CACHE INTERNAL "Flag to indicate if C++ is needed for TPLs") -# Setup TPLs +# Setup third-party libraries (TPLs) setup_fei() setup_tpl_or_internal(blas) setup_tpl_or_internal(lapack) @@ -297,8 +302,8 @@ setup_tpl(magma) setup_tpl(caliper) setup_tpl(umpire) -# Some TPLs need C++ to be enabled -if(HYPRE_NEEDS_CXX) +# Some TPLs need C++ to be enabled (GPU already enables C++) +if(HYPRE_NEEDS_CXX AND NOT HYPRE_USING_GPU) enable_language(CXX) endif() diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index 6ce127dd7f..b15b8ee550 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -15,21 +15,13 @@ function(set_hypre_option category name description default_value) endif() endfunction() -# Function to set generic variables based on condition -function(set_conditional_var condition var_name) - if(${condition}) - #set(${var_name} ON CACHE INTERNAL "${var_name} set to ON because ${condition} is true") - set(${var_name} ON CACHE BOOL "" FORCE) - endif() -endfunction() - -# Function to set conditional hypre build options -function(set_conditional_hypre_option condition_prefix var_prefix var_name) +# Function to set internal hypre build options +function(set_internal_hypre_option condition_prefix var_prefix var_name) if(HYPRE_${condition_prefix}_${var_name}) if(var_prefix STREQUAL "") - set(HYPRE_${var_name} ON CACHE BOOL "" FORCE) + set(HYPRE_${var_name} ON CACHE INTERNAL "") else() - set(HYPRE_${var_prefix}_${var_name} ON CACHE BOOL "" FORCE) + set(HYPRE_${var_prefix}_${var_name} ON CACHE INTERNAL "") endif() endif() endfunction() @@ -63,6 +55,40 @@ function(setup_git_version_info HYPRE_GIT_DIR) endif() endfunction() +# Function to check if two options have the same value +function(ensure_options_match option1 option2) + if(DEFINED CACHE{${option1}} AND DEFINED CACHE{${option2}}) + #if ((${option1} AND NOT ${option2}) OR (NOT ${option1} AND ${option2})) + if(NOT (${option1} STREQUAL ${option2}) AND NOT (${option1} STREQUAL "OFF" AND ${option2} STREQUAL "OFF")) + # Save the value of the conflicting options + set(saved_value1 "${${option1}}") + set(saved_value2 "${${option2}}") + + # Unset conflicting options + unset(${option1} CACHE) + unset(${option2} CACHE) + message(FATAL_ERROR "Incompatible options: ${option1} (${saved_value1}) and ${option2} (${saved_value2}) must have the same value. Unsetting both options.") + endif() + endif() +endfunction() + +# Function to check if two options have different values +function(ensure_options_differ option1 option2) + if(DEFINED ${option1} AND DEFINED ${option2}) + if(${option1} AND ${${option2}}) + # Save the value of the conflicting options + set(saved_value1 "${${option1}}") + set(saved_value2 "${${option2}}") + + # Unset conflicting options + unset(${option1} CACHE) + unset(${option2} CACHE) + + message(FATAL_ERROR "Error: ${option1} (${saved_value1}) and ${option2} (${saved_value2}) are mutually exclusive. Only one can be set to ON. Unsetting both options.") + endif() + endif() +endfunction() + # Function to configure MPI target function(configure_mpi_target) find_package(MPI REQUIRED) diff --git a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake index 5c0f20c711..7756efdd78 100644 --- a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake @@ -35,10 +35,6 @@ if(NOT HYPRE_WITH_EXTRA_CXXFLAGS STREQUAL "") add_compile_options("$<$:${HYPRE_WITH_EXTRA_CXXFLAGS_LIST}>") endif() -# Set conditional variables -set_conditional_var(HYPRE_ENABLE_GPU_STREAMS HYPRE_USING_GPU_STREAMS) -set_conditional_var(HYPRE_ENABLE_DEVICE_POOL HYPRE_USING_DEVICE_POOL) - if(HYPRE_WITH_CUDA) include(HYPRE_SetupCUDAToolkit) From 657d0d548a06d18a79e9526db1d1fd5b549d0d38 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 23 Sep 2024 20:37:51 -0400 Subject: [PATCH 21/55] Fix target: CUDAToolkit -> CUDA --- src/config/cmake/HYPRE_SetupCUDAToolkit.cmake | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake index 996341cff4..044f84c348 100644 --- a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake @@ -147,19 +147,19 @@ function(find_and_add_cuda_library LIB_NAME HYPRE_ENABLE_VAR) # Use CUDAToolkit to find the component find_package(CUDAToolkit REQUIRED COMPONENTS ${LIB_NAME}) - if(TARGET CUDAToolkit::${LIB_NAME}) - message(STATUS "Found ${LIB_NAME_UPPER} library: ${${LIB_NAME}_LIBRARY}") - list(APPEND CUDA_LIBS CUDAToolkit::${LIB_NAME}) + if(TARGET CUDA::${LIB_NAME}) + message(STATUS "Found ${LIB_NAME_UPPER} library") + list(APPEND CUDA_LIBS CUDA::${LIB_NAME}) else() - #message(WARNING "CUDAToolkit::${LIB_NAME} target not found. Attempting manual linking.") + message(STATUS "CUDA::${LIB_NAME} target not found. Attempting manual linking.") find_library(${LIB_NAME}_LIBRARY ${LIB_NAME} HINTS ${CUDAToolkit_LIBRARY_DIR}) if(${LIB_NAME}_LIBRARY) message(STATUS "Found ${LIB_NAME_UPPER} library: ${${LIB_NAME}_LIBRARY}") - add_library(CUDAToolkit::${LIB_NAME} UNKNOWN IMPORTED) - set_target_properties(CUDAToolkit::${LIB_NAME} PROPERTIES + add_library(CUDA::${LIB_NAME} UNKNOWN IMPORTED) + set_target_properties(CUDA::${LIB_NAME} PROPERTIES IMPORTED_LOCATION "${${LIB_NAME}_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${CUDAToolkit_INCLUDE_DIRS}") - list(APPEND CUDA_LIBS CUDAToolkit::${LIB_NAME}) + list(APPEND CUDA_LIBS CUDA::${LIB_NAME}) else() message(FATAL_ERROR "Could not find ${LIB_NAME_UPPER} library. Please check your CUDA installation.") endif() From 34cd9cddd3a04e278db5202a52909366138446bf Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 23 Sep 2024 20:38:46 -0400 Subject: [PATCH 22/55] Improve dependency search in HYPREConfig.cmake --- src/config/HYPREConfig.cmake.in | 126 ++++++++++++++++++++++++-------- 1 file changed, 96 insertions(+), 30 deletions(-) diff --git a/src/config/HYPREConfig.cmake.in b/src/config/HYPREConfig.cmake.in index 82c80648da..d280438897 100644 --- a/src/config/HYPREConfig.cmake.in +++ b/src/config/HYPREConfig.cmake.in @@ -7,6 +7,18 @@ include(CMakeFindDependencyMacro) +function(find_hypre_dependency name) + string(TOUPPER ${name} UPPER_NAME) + if(HYPRE_WITH_${UPPER_NAME}) + find_dependency(${name} REQUIRED HINTS ${HYPRE_DEPENDENCY_DIRS}) + if(${name}_FOUND) + message(STATUS "Found ${UPPER_NAME}: ${${name}_DIR}") + else() + message(FATAL_ERROR "${UPPER_NAME} not found.") + endif() + endif() +endfunction() + set(HYPRE_DEPENDENCY_DIRS "@HYPRE_DEPENDENCY_DIRS@") set(HYPRE_ENABLE_SHARED @HYPRE_ENABLE_SHARED@) set(HYPRE_ENABLE_BIGINT @HYPRE_ENABLE_BIGINT@) @@ -53,31 +65,14 @@ if (UNIX) list(APPEND TPL_LIBRARIES m) endif() -if(HYPRE_WITH_CALIPER) - find_dependency(caliper REQUIRED HINTS ${HYPRE_DEPENDENCY_DIRS}) - if(caliper_FOUND) - message(STATUS "Caliper found: ${caliper_DIR}") - else() - message(FATAL_ERROR "Caliper not found.") - endif() -endif() - -if(NOT HYPRE_ENABLE_HYPRE_BLAS) +if(NOT HYPRE_ENABLE_INTERNAL_BLAS) find_dependency(BLAS) endif() -if(NOT HYPRE_ENABLE_HYPRE_LAPACK) +if(NOT HYPRE_ENABLE_INTERNAL_LAPACK) find_dependency(LAPACK) endif() -if(HYPRE_USING_DSUPERLU) - list(APPEND TPL_LIBRARIES @TPL_DSUPERLU_LIBRARIES@ stdc++) -endif() - -if(HYPRE_USING_MAGMA) - list(APPEND TPL_LIBRARIES @TPL_MAGMA_LIBRARIES@ stdc++) -endif() - if(HYPRE_WITH_MPI) enable_language(C) find_dependency(MPI @MPI_C_VERSION@ COMPONENTS C) @@ -87,19 +82,90 @@ if(HYPRE_WITH_OPENMP) find_dependency(OpenMP) endif() -if(HYPRE_WITH_CUDA) - find_dependency(CUDAToolkit REQUIRED COMPONENTS cusparse cublas curand) - - # Diagnostic Messages - if(CUDAToolkit_FOUND) - message(STATUS "CUDAToolkit found: ${CUDAToolkit_VERSION}") - if(TARGET CUDAToolkit::cusparse AND TARGET CUDAToolkit::cublas AND TARGET CUDAToolkit::curand) - message(STATUS "Required CUDA components found: cusparse, cublas, curand.") +find_hypre_dependency(caliper) +find_hypre_dependency(dsuperlu) +find_hypre_dependency(magma) +find_hypre_dependency(umpire) + +if(HYPRE_WITH_CUDA OR HYPRE_WITH_HIP OR HYPRE_WITH_SYCL) + set(REQUIRED_GPU_COMPONENTS) + + if(HYPRE_WITH_CUDA) + if(HYPRE_ENABLE_CUSPARSE) + list(APPEND REQUIRED_GPU_COMPONENTS cusparse) + endif() + + if(HYPRE_ENABLE_CUBLAS) + list(APPEND REQUIRED_GPU_COMPONENTS cublas) + endif() + + if(HYPRE_ENABLE_CURAND) + list(APPEND REQUIRED_GPU_COMPONENTS curand) + endif() + elseif(HYPRE_WITH_HIP) + if(HYPRE_ENABLE_ROCSPARSE) + list(APPEND REQUIRED_GPU_COMPONENTS rocsparse) + endif() + + if(HYPRE_ENABLE_ROCBLAS) + list(APPEND REQUIRED_GPU_COMPONENTS rocblas) + endif() + + if(HYPRE_ENABLE_ROCRAND) + list(APPEND REQUIRED_GPU_COMPONENTS rocrand) + endif() + elseif(HYPRE_WITH_SYCL) + if(HYPRE_ENABLE_ONEMKLSPARSE) + list(APPEND REQUIRED_GPU_COMPONENTS onemklsparse) + endif() + + if(HYPRE_ENABLE_ONEMKLBLAS) + list(APPEND REQUIRED_GPU_COMPONENTS onemklblas) + endif() + + if(HYPRE_ENABLE_ONEMKLRAND) + list(APPEND REQUIRED_GPU_COMPONENTS onemklrand) + endif() + endif() + + if(REQUIRED_GPU_COMPONENTS) + if(HYPRE_WITH_CUDA) + message(STATUS "Finding CUDA Toolkit components: ${REQUIRED_GPU_COMPONENTS}") + find_dependency(CUDAToolkit REQUIRED COMPONENTS ${REQUIRED_GPU_COMPONENTS}) + set(GPU_TOOLKIT_FOUND ${CUDAToolkit_FOUND}) + set(GPU_TOOLKIT_VERSION ${CUDAToolkit_VERSION}) + elseif(HYPRE_WITH_HIP) + message(STATUS "Finding ROCm Toolkit components: ${REQUIRED_GPU_COMPONENTS}") + find_dependency(rocm REQUIRED COMPONENTS ${REQUIRED_GPU_COMPONENTS}) + set(GPU_TOOLKIT_FOUND ${rocm_FOUND}) + set(GPU_TOOLKIT_VERSION ${rocm_VERSION}) + elseif(HYPRE_WITH_SYCL) + message(STATUS "Finding Intel DPC++ Toolkit components: ${REQUIRED_GPU_COMPONENTS}") + find_dependency(IntelDPCPP REQUIRED) + set(GPU_TOOLKIT_FOUND ${IntelDPCPP_FOUND}) + set(GPU_TOOLKIT_VERSION ${IntelDPCPP_VERSION}) + endif() + + if(GPU_TOOLKIT_FOUND) + message(STATUS "GPU Toolkit found: ${GPU_TOOLKIT_VERSION}") + set(MISSING_COMPONENTS) + foreach(component ${REQUIRED_GPU_COMPONENTS}) + if(HYPRE_WITH_CUDA AND NOT TARGET CUDA::${component}) + list(APPEND MISSING_COMPONENTS ${component}) + elseif(HYPRE_WITH_HIP AND NOT TARGET roc::${component}) + list(APPEND MISSING_COMPONENTS ${component}) + elseif(HYPRE_WITH_SYCL AND NOT TARGET sycl::${component}) + list(APPEND MISSING_COMPONENTS ${component}) + endif() + endforeach() + if(MISSING_COMPONENTS) + message(FATAL_ERROR "Required GPU components not found: ${MISSING_COMPONENTS}") else() - message(FATAL_ERROR "One or more required CUDA components (cusparse, cublas, curand) not found.") + message(STATUS "Required GPU components found: ${REQUIRED_GPU_COMPONENTS}") endif() - else() - message(FATAL_ERROR "CUDAToolkit not found.") + else() + message(FATAL_ERROR "GPU Toolkit not found.") + endif() endif() endif() From 9212a9318009edce3175ada5d490b42f4627d9c5 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Thu, 26 Sep 2024 21:27:04 -0400 Subject: [PATCH 23/55] Adjustments --- src/CMakeLists.txt | 77 ++++++++++++--------- src/config/cmake/HYPRE_CMakeUtilities.cmake | 3 +- 2 files changed, 48 insertions(+), 32 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 28c929f6cd..8f0cd07694 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -75,21 +75,29 @@ else() endif() # Set default compile optimization flag -set(HYPRE_BUILD_TYPE "Release" CACHE STRING - "Optimization flags: set to Debug, Release, RelWithDebInfo, or MinSizeRel") - -if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - set(CMAKE_BUILD_TYPE "${HYPRE_BUILD_TYPE}" CACHE INTERNAL "" FORCE) - # Set the possible values of build type for cmake-gui - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS - "Debug" "Release" "MinSizeRel" "RelWithDebInfo") -endif () -message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") +if(NOT CMAKE_BUILD_TYPE AND NOT DEFINED HYPRE_BUILD_TYPE) + set(HYPRE_BUILD_TYPE "Release" CACHE STRING + "Optimization flags: set to Debug, Release, RelWithDebInfo, or MinSizeRel") +endif() + +# Set the possible values of build type for cmake-gui +set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS + "Debug" "Release" "MinSizeRel" "RelWithDebInfo") + +# If CMAKE_BUILD_TYPE is not set, use HYPRE_BUILD_TYPE +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "${HYPRE_BUILD_TYPE}" CACHE STRING + "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." + FORCE) +endif() + +# Print the build type +message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") # Base boolean configuration options set_hypre_option(BASE HYPRE_ENABLE_SHARED "Build a shared library" ${BUILD_SHARED_LIBS}) -set_hypre_option(BASE HYPRE_ENABLE_BIGINT "Use long long int for HYPRE_Int" OFF) -set_hypre_option(BASE HYPRE_ENABLE_MIXEDINT "Use long long int for HYPRE_BigInt, int for HYPRE_INT" OFF) +set_hypre_option(BASE HYPRE_ENABLE_BIGINT "Use long long int for HYPRE_Int and HYPRE_BigInt" OFF) +set_hypre_option(BASE HYPRE_ENABLE_MIXEDINT "Use long long int for HYPRE_BigInt, int for HYPRE_Int" OFF) set_hypre_option(BASE HYPRE_ENABLE_SINGLE "Use float for HYPRE_Real" OFF) set_hypre_option(BASE HYPRE_ENABLE_LONG_DOUBLE "Use long double for HYPRE_Real" OFF) set_hypre_option(BASE HYPRE_ENABLE_COMPLEX "Use complex values" OFF) @@ -112,14 +120,14 @@ set_hypre_option(GPU HYPRE_WITH_SYCL "Use SYCL" OFF) set_hypre_option(GPU HYPRE_WITH_GPU_AWARE_MPI "Compile with device aware GPU support" OFF) set_hypre_option(GPU HYPRE_ENABLE_UNIFIED_MEMORY "Use unified memory for allocating the memory" OFF) set_hypre_option(GPU HYPRE_ENABLE_DEVICE_MALLOC_ASYNC "Use device async malloc" OFF) +set_hypre_option(GPU HYPRE_ENABLE_GPU_PROFILING "Use NVTX on CUDA or rocTX on HIP" OFF) # CUDA options set_hypre_option(CUDA HYPRE_ENABLE_CUDA_STREAMS "Use CUDA streams" ON) set_hypre_option(CUDA HYPRE_ENABLE_CUSPARSE "Use cuSPARSE" ON) -set_hypre_option(CUDA HYPRE_ENABLE_CUSOLVER "Use cuSOLVER" OFF) -set_hypre_option(CUDA HYPRE_ENABLE_DEVICE_POOL "Use device memory pool" OFF) -set_hypre_option(CUDA HYPRE_ENABLE_CUBLAS "Use cuBLAS" OFF) +set_hypre_option(CUDA HYPRE_ENABLE_CUSOLVER "Use cuSOLVER" ON) +set_hypre_option(CUDA HYPRE_ENABLE_CUBLAS "Use cuBLAS" ON) set_hypre_option(CUDA HYPRE_ENABLE_CURAND "Use cuRAND" ON) -set_hypre_option(CUDA HYPRE_ENABLE_GPU_PROFILING "Use NVTX on CUDA" OFF) +set_hypre_option(CUDA HYPRE_ENABLE_DEVICE_POOL "Use CUB's device memory pool" OFF) # HIP options set_hypre_option(HIP HYPRE_ENABLE_ROCBLAS "Use rocBLAS" ON) set_hypre_option(HIP HYPRE_ENABLE_ROCSPARSE "Use rocSPARSE" ON) @@ -138,19 +146,20 @@ set_hypre_option(TPL HYPRE_WITH_UMPIRE_UM "Use Umpire Allocator fo set_hypre_option(TPL HYPRE_WITH_SUPERLU "Use TPL SuperLU" OFF) set_hypre_option(TPL HYPRE_WITH_DSUPERLU "Use TPL SuperLU_Dist" OFF) set_hypre_option(TPL HYPRE_WITH_MAGMA "Use TPL MAGMA" OFF) -set_hypre_option(SKIP TPL_UMPIRE_LIBRARIES "Optional list of absolute paths to Umpire link libraries []." "") -set_hypre_option(SKIP TPL_UMPIRE_INCLUDE_DIRS "Optional list of absolute paths to Umpire include directories []." "") -set_hypre_option(SKIP TPL_SUPERLU_LIBRARIES "Optional list of absolute paths to SuperLU link libraries []." "") -set_hypre_option(SKIP TPL_SUPERLU_INCLUDE_DIRS "Optional list of absolute paths to SuperLU include directories []." "") -set_hypre_option(SKIP TPL_DSUPERLU_LIBRARIES "Optional list of absolute paths to SuperLU_Dist link libraries []." "") -set_hypre_option(SKIP TPL_DSUPERLU_INCLUDE_DIRS "Optional list of absolute paths to SuperLU_Dist include directories []." "") -set_hypre_option(SKIP TPL_MAGMA_LIBRARIES "Optional list of absolute paths to MAGMA link libraries []." "") -set_hypre_option(SKIP TPL_MAGMA_INCLUDE_DIRS "Optional list of absolute paths to MAGMA include directories []." "") -set_hypre_option(SKIP TPL_CALIPER_LIBRARIES "Optional list of absolute paths to Caliper link libraries []." "") -set_hypre_option(SKIP TPL_CALIPER_INCLUDE_DIRS "Optional list of absolute paths to Caliper include directories []." "") -set_hypre_option(SKIP TPL_BLAS_LIBRARIES "Optional list of absolute paths to BLAS libraries, otherwise use FindBLAS to locate []." "") -set_hypre_option(SKIP TPL_LAPACK_LIBRARIES "Optional list of absolute paths to LAPACK libraries, otherwise use FindLAPACK to locate []." "") -set_hypre_option(SKIP TPL_FEI_INCLUDE_DIRS "Optional list of absolute paths to FEI include directories []." "") +set_hypre_option(TPL HYPRE_WITH_CALIPER "Use TPL Caliper" OFF) +set_hypre_option(SKIP TPL_UMPIRE_LIBRARIES "Absolute paths to Umpire link libraries [Optional]." "") +set_hypre_option(SKIP TPL_UMPIRE_INCLUDE_DIRS "Absolute paths to Umpire include directories [Optional]." "") +set_hypre_option(SKIP TPL_SUPERLU_LIBRARIES "Absolute paths to SuperLU link libraries [Optional]." "") +set_hypre_option(SKIP TPL_SUPERLU_INCLUDE_DIRS "Absolute paths to SuperLU include directories [Optional]." "") +set_hypre_option(SKIP TPL_DSUPERLU_LIBRARIES "Absolute paths to SuperLU_Dist link libraries [Optional]." "") +set_hypre_option(SKIP TPL_DSUPERLU_INCLUDE_DIRS "Absolute paths to SuperLU_Dist include directories [Optional]." "") +set_hypre_option(SKIP TPL_MAGMA_LIBRARIES "Absolute paths to MAGMA link libraries [Optional]." "") +set_hypre_option(SKIP TPL_MAGMA_INCLUDE_DIRS "Absolute paths to MAGMA include directories [Optional]." "") +set_hypre_option(SKIP TPL_CALIPER_LIBRARIES "Absolute paths to Caliper link libraries [Optional]." "") +set_hypre_option(SKIP TPL_CALIPER_INCLUDE_DIRS "Absolute paths to Caliper include directories [Optional]." "") +set_hypre_option(SKIP TPL_BLAS_LIBRARIES "Absolute paths to BLAS link libraries [Optional]." "") +set_hypre_option(SKIP TPL_LAPACK_LIBRARIES "Absolute paths to LAPACK link libraries [Optional]." "") +set_hypre_option(SKIP TPL_FEI_INCLUDE_DIRS "Absolute paths to FEI include directories [Optional]." "") # Extra flags set(HYPRE_WITH_EXTRA_CFLAGS "" CACHE STRING "Define extra C compile flags") set(HYPRE_WITH_EXTRA_CXXFLAGS "" CACHE STRING "Define extra CXX compile flags") @@ -216,21 +225,25 @@ if (HYPRE_WITH_UMPIRE) endif () if (HYPRE_WITH_UMPIRE_HOST) + set(HYPRE_WITH_UMPIRE ON CACHE BOOL "" FORCE) set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) set(HYPRE_USING_UMPIRE_HOST ON CACHE BOOL "" FORCE) endif () if (HYPRE_WITH_UMPIRE_DEVICE) + set(HYPRE_WITH_UMPIRE ON CACHE BOOL "" FORCE) set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) set(HYPRE_USING_UMPIRE_DEVICE ON CACHE BOOL "" FORCE) endif () if (HYPRE_WITH_UMPIRE_UM) + set(HYPRE_WITH_UMPIRE ON CACHE BOOL "" FORCE) set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) set(HYPRE_USING_UMPIRE_UM ON CACHE BOOL "" FORCE) endif () if (HYPRE_WITH_UMPIRE_PINNED) + set(HYPRE_WITH_UMPIRE ON CACHE BOOL "" FORCE) set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) set(HYPRE_USING_UMPIRE_PINNED ON CACHE BOOL "" FORCE) endif () @@ -241,7 +254,9 @@ ensure_options_match(HYPRE_INSTALL_PREFIX CMAKE_INSTALL_PREFIX) ensure_options_match(HYPRE_BUILD_TYPE CMAKE_BUILD_TYPE) ensure_options_differ(HYPRE_WITH_CUDA HYPRE_WITH_HIP) ensure_options_differ(HYPRE_WITH_CUDA HYPRE_WITH_SYCL) +ensure_options_differ(HYPRE_WITH_CUDA HYPRE_BIGINT) ensure_options_differ(HYPRE_WITH_HIP HYPRE_WITH_SYCL) +ensure_options_differ(HYPRE_WITH_HIP HYPRE_BIGINT) # Set MPI compile flags if (HYPRE_WITH_MPI) @@ -406,12 +421,12 @@ write_basic_package_version_file( configure_package_config_file( "${CMAKE_CURRENT_SOURCE_DIR}/config/HYPREConfig.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" - INSTALL_DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake + INSTALL_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" ) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfigVersion.cmake" - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE" ) # Install the CMake export file install(EXPORT HYPRETargets diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index b15b8ee550..0937cfe35d 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -176,7 +176,6 @@ function(setup_tpl LIBNAME) # Use find_package find_package(${LIBNAME} REQUIRED CONFIG) if(${LIBNAME}_FOUND) - message(STATUS "Found ${LIBNAME_UPPER} library") list(APPEND HYPRE_DEPENDENCY_DIRS "${${LIBNAME}_ROOT}") set(HYPRE_DEPENDENCY_DIRS "${HYPRE_DEPENDENCY_DIRS}" CACHE INTERNAL "" FORCE) @@ -186,8 +185,10 @@ function(setup_tpl LIBNAME) if(TARGET ${LIBNAME}::${LIBNAME}) target_link_libraries(${PROJECT_NAME} PUBLIC ${LIBNAME}::${LIBNAME}) + message(STATUS "Found ${LIBNAME} target: ${LIBNAME}::${LIBNAME}") elseif(TARGET ${LIBNAME}) target_link_libraries(${PROJECT_NAME} PUBLIC ${LIBNAME}) + message(STATUS "Found ${LIBNAME} target: ${LIBNAME}") else() message(FATAL_ERROR "${LIBNAME} target not found. Please check your ${LIBNAME} installation") endif() From f60f359d24d5a6bbf969c9b9f0ebb06df426e27f Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 16 Oct 2024 19:35:00 -0400 Subject: [PATCH 24/55] Update SOVERSION for consistency with ABI approach in hypre --- src/CMakeLists.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0164ff1cf4..001d27c263 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -57,7 +57,9 @@ set(CMAKE_SKIP_BUILD_RPATH FALSE) # Includes rpath in th set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) # Use different paths for build and install set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) # Use link-time paths set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") # Where binaries can look for shared libraries -set(BUILD_SHARED_LIBS "Build using shared libraries" OFF) # Build static library by default +if(NOT BUILD_SHARED_LIBS) + set(BUILD_SHARED_LIBS OFF) # Build static library by default +endif() # Set cmake module path set(CMAKE_MODULE_PATH "${HYPRE_SOURCE_DIR}/config/cmake" "${CMAKE_MODULE_PATH}") @@ -284,13 +286,10 @@ if (NOT HYPRE_WITH_EXTRA_CFLAGS STREQUAL "") add_compile_options("$<$:${HYPRE_WITH_EXTRA_CFLAGS_LIST}>") endif () -# Set library build type (must appear before add_library calls) -set(BUILD_SHARED_LIBS ${HYPRE_SHARED} CACHE INTERNAL "Build shared libraries" FORCE) - # Set output directory for the library set_target_properties(HYPRE PROPERTIES VERSION "${HYPRE_VERSION}" - SOVERSION "${HYPRE_VERSION_MAJOR}" + SOVERSION "${HYPRE_VERSION_MAJOR}.${HYPRE_VERSION_MINOR}" ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" From 35a8c91aa97c5c3f41b8828f6dfc2423f7816ff9 Mon Sep 17 00:00:00 2001 From: Christoph Junghans Date: Thu, 17 Oct 2024 09:21:35 -0600 Subject: [PATCH 25/55] cmake: update soversion (#1164) --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 001d27c263..b2f40fc262 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -289,7 +289,7 @@ endif () # Set output directory for the library set_target_properties(HYPRE PROPERTIES VERSION "${HYPRE_VERSION}" - SOVERSION "${HYPRE_VERSION_MAJOR}.${HYPRE_VERSION_MINOR}" + SOVERSION "${HYPRE_VERSION_MAJOR}${HYPRE_VERSION_MINOR}" ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" From d32a7dcea99699ba6aa494959a76e4c81b8161e9 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Fri, 8 Nov 2024 10:34:36 -0500 Subject: [PATCH 26/55] Remove redundant options + use cache internal --- src/CMakeLists.txt | 114 +++++++++----------- src/config/cmake/HYPRE_CMakeUtilities.cmake | 2 +- 2 files changed, 49 insertions(+), 67 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b2f40fc262..5a1781847b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -52,53 +52,35 @@ if (${HYPRE_SOURCE_DIR} STREQUAL ${HYPRE_BINARY_DIR}) message(FATAL_ERROR "In-place build not allowed! Please use a separate build directory. See the Users Manual or INSTALL file for details.") endif () +# Set default installation directory, but provide a means for users to change +set(CMAKE_INSTALL_PREFIX "${HYPRE_SOURCE_DIR}/hypre" CACHE PATH "Installation directory for HYPRE" FORCE) + # Ensure RPATH/RUNPATH is set properly during the build -set(CMAKE_SKIP_BUILD_RPATH FALSE) # Includes rpath in the binaries being built -set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) # Use different paths for build and install -set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) # Use link-time paths -set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") # Where binaries can look for shared libraries -if(NOT BUILD_SHARED_LIBS) - set(BUILD_SHARED_LIBS OFF) # Build static library by default -endif() +set(CMAKE_SKIP_BUILD_RPATH FALSE CACHE BOOL "Include rpath in the binaries being built") +set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE CACHE BOOL "Use different paths for build and install") +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE BOOL "Use link-time paths for install rpath") +set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib" CACHE STRING "Where binaries look for shared libraries") + +# Build static library by default +option(BUILD_SHARED_LIBS "Build shared libraries" OFF) # Set cmake module path set(CMAKE_MODULE_PATH "${HYPRE_SOURCE_DIR}/config/cmake" "${CMAKE_MODULE_PATH}") -# Set default installation directory, but provide a means for users to change -if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT OR NOT DEFINED HYPRE_INSTALL_PREFIX) - if (NOT DEFINED HYPRE_INSTALL_PREFIX) - set(CMAKE_INSTALL_PREFIX "${PROJECT_SOURCE_DIR}/hypre" CACHE PATH "Installation directory for HYPRE" FORCE) - else() - set(CMAKE_INSTALL_PREFIX "${HYPRE_INSTALL_PREFIX}" CACHE PATH "Installation directory for HYPRE" FORCE) - endif() -else() - if (NOT DEFINED HYPRE_INSTALL_PREFIX) - set(HYPRE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" CACHE PATH "Installation directory for HYPRE" FORCE) - endif() -endif() - -# Set default compile optimization flag -if(NOT CMAKE_BUILD_TYPE AND NOT DEFINED HYPRE_BUILD_TYPE) - set(HYPRE_BUILD_TYPE "Release" CACHE STRING - "Optimization flags: set to Debug, Release, RelWithDebInfo, or MinSizeRel") +# Set CMAKE_BUILD_TYPE to default to Release if not already specified +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type options: Debug, Release, RelWithDebInfo, or MinSizeRel." FORCE) endif() -# Set the possible values of build type for cmake-gui -set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS - "Debug" "Release" "MinSizeRel" "RelWithDebInfo") +# Set the possible values of build type for ccmake/cmake-gui +set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") -# If CMAKE_BUILD_TYPE is not set, use HYPRE_BUILD_TYPE -if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE "${HYPRE_BUILD_TYPE}" CACHE STRING - "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." - FORCE) -endif() - -# Print the build type +# Print the configuration message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") +message(STATUS "Shared library: ${BUILD_SHARED_LIBS}") +message(STATUS "Installation directory: ${CMAKE_INSTALL_PREFIX}") # Base boolean configuration options -set_hypre_option(BASE HYPRE_ENABLE_SHARED "Build a shared library" ${BUILD_SHARED_LIBS}) set_hypre_option(BASE HYPRE_ENABLE_BIGINT "Use long long int for HYPRE_Int and HYPRE_BigInt" OFF) set_hypre_option(BASE HYPRE_ENABLE_MIXEDINT "Use long long int for HYPRE_BigInt, int for HYPRE_Int" OFF) set_hypre_option(BASE HYPRE_ENABLE_SINGLE "Use float for HYPRE_Real" OFF) @@ -170,7 +152,6 @@ set(HYPRE_SYCL_TARGET "" CACHE STRING "Target SYCL architectu set(HYPRE_SYCL_TARGET_BACKEND "" CACHE STRING "Additional SYCL backend options, e.g. '-device 12.1.0,12.4.0'.") # Set config name values -set_internal_hypre_option(ENABLE "" SHARED) set_internal_hypre_option(ENABLE "" BIGINT) set_internal_hypre_option(ENABLE "" MIXEDINT) set_internal_hypre_option(ENABLE "" SINGLE) @@ -188,73 +169,74 @@ set_internal_hypre_option(WITH USING SYCL) set_internal_hypre_option(WITH USING MAGMA) set_internal_hypre_option(WITH USING CALIPER) +if (BUILD_SHARED_LIBS) + set(HYPRE_SHARED ON CACHE INTERNAL "") +endif () + if (HYPRE_WITH_MPI) - set(HYPRE_HAVE_MPI ON CACHE BOOL "" FORCE) - set(HYPRE_SEQUENTIAL OFF CACHE BOOL "" FORCE) + set(HYPRE_HAVE_MPI ON CACHE INTERNAL "") + set(HYPRE_SEQUENTIAL OFF CACHE INTERNAL "") else () - set(HYPRE_SEQUENTIAL ON CACHE BOOL "" FORCE) + set(HYPRE_SEQUENTIAL ON CACHE INTERNAL "") endif () if (HYPRE_WITH_SUPERLU) - set(HYPRE_USING_SUPERLU ON CACHE BOOL "" FORCE) + set(HYPRE_USING_SUPERLU ON CACHE INTERNAL "") add_compile_definitions(HAVE_SUPERLU) endif () if (HYPRE_WITH_DSUPERLU) - set(HYPRE_USING_DSUPERLU ON CACHE BOOL "" FORCE) - set(HYPRE_USING_HYPRE_BLAS OFF CACHE BOOL "" FORCE) - set(HYPRE_USING_HYPRE_LAPACK OFF CACHE BOOL "" FORCE) + set(HYPRE_USING_DSUPERLU ON CACHE INTERNAL "") + set(HYPRE_USING_HYPRE_BLAS OFF CACHE INTERNAL "") + set(HYPRE_USING_HYPRE_LAPACK OFF CACHE INTERNAL "") endif () if (HYPRE_ENABLE_FEI) - set(HYPRE_USING_FEI ON CACHE BOOL "" FORCE) + set(HYPRE_USING_FEI ON CACHE INTERNAL "") message(WARNING "CMake support for FEI is not complete!") endif () # FEI doesn't currently compile with shared if (HYPRE_SHARED OR HYPRE_BIGINT OR HYPRE_SINGLE OR HYPRE_LONG_DOUBLE) - set(HYPRE_USING_FEI OFF CACHE BOOL "" FORCE) - set(HYPRE_ENABLE_FEI OFF CACHE BOOL "" FORCE) + set(HYPRE_USING_FEI OFF CACHE INTERNAL "") + set(HYPRE_ENABLE_FEI OFF CACHE INTERNAL "") endif () if (HYPRE_SEQUENTIAL) - set(HYPRE_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + set(HYPRE_BUILD_EXAMPLES OFF CACHE INTERNAL "") endif () if (HYPRE_WITH_UMPIRE) - set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_DEVICE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_UM ON CACHE BOOL "" FORCE) + set(HYPRE_USING_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_DEVICE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_UM ON CACHE INTERNAL "") endif () if (HYPRE_WITH_UMPIRE_HOST) - set(HYPRE_WITH_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_HOST ON CACHE BOOL "" FORCE) + set(HYPRE_WITH_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_HOST ON CACHE INTERNAL "") endif () if (HYPRE_WITH_UMPIRE_DEVICE) - set(HYPRE_WITH_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_DEVICE ON CACHE BOOL "" FORCE) + set(HYPRE_WITH_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_DEVICE ON CACHE INTERNAL "") endif () if (HYPRE_WITH_UMPIRE_UM) - set(HYPRE_WITH_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_UM ON CACHE BOOL "" FORCE) + set(HYPRE_WITH_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_UM ON CACHE INTERNAL "") endif () if (HYPRE_WITH_UMPIRE_PINNED) - set(HYPRE_WITH_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_PINNED ON CACHE BOOL "" FORCE) + set(HYPRE_WITH_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_PINNED ON CACHE INTERNAL "") endif () # Check for conflicting options -ensure_options_match(HYPRE_ENABLE_SHARED BUILD_SHARED_LIBS) -ensure_options_match(HYPRE_INSTALL_PREFIX CMAKE_INSTALL_PREFIX) -ensure_options_match(HYPRE_BUILD_TYPE CMAKE_BUILD_TYPE) ensure_options_differ(HYPRE_WITH_CUDA HYPRE_WITH_HIP) ensure_options_differ(HYPRE_WITH_CUDA HYPRE_WITH_SYCL) ensure_options_differ(HYPRE_WITH_CUDA HYPRE_BIGINT) diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index df29008523..caf7227d62 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -241,7 +241,7 @@ function(setup_tpl_or_internal LIB_NAME) # Use external library if(TPL_${LIB_NAME_UPPER}_LIBRARIES) # Use specified TPL libraries - message(STATUS "Enabled support for using ${LIB_NAME_UPPER}.") + message(STATUS "Enabled support for using external ${LIB_NAME_UPPER}.") foreach(lib ${TPL_${LIB_NAME_UPPER}_LIBRARIES}) if(EXISTS ${lib}) message(STATUS "${LIB_NAME_UPPER} library found: ${lib}") From de4e0a8337a4694b27c407f733f878214f1f001e Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Sat, 9 Nov 2024 00:04:52 -0500 Subject: [PATCH 27/55] Update CMake documentation --- src/docs/usr-manual/ch-intro.rst | 25 +- src/docs/usr-manual/ch-misc.rst | 863 ++++++++++++++++++------------- 2 files changed, 507 insertions(+), 381 deletions(-) diff --git a/src/docs/usr-manual/ch-intro.rst b/src/docs/usr-manual/ch-intro.rst index cb55a1b4bb..ec88935585 100644 --- a/src/docs/usr-manual/ch-intro.rst +++ b/src/docs/usr-manual/ch-intro.rst @@ -122,19 +122,24 @@ How to get started Installing hypre ------------------------------------------------------------------------------ -As previously noted, on most systems hypre can be built by simply typing -``configure`` followed by ``make`` in the top-level source directory. -Alternatively, the CMake system [CMakeWeb]_ can be used, and is the best -approach for building hypre on Windows systems in particular. For more detailed -instructions, read the ``INSTALL`` file provided with the hypre distribution or -the :ref:`ch-General` section of this manual. Note the following requirements: +As previously noted, on most systems hypre can be built by typing +``./configure`` followed by ``make`` in the top-level source directory. +Alternatively, `CMake `_ can be used and is the recommended +method for building hypre on Windows. For more detailed instructions, refer to +the `INSTALL `_ file +provided in the hypre distribution or see the :ref:`ch-General` section of this manual. -* To run in parallel, hypre requires an installation of MPI. +.. note:: -* Configuration of hypre with threads requires an implementation of OpenMP. - Currently, only a subset of hypre is threaded. + * To run in parallel, hypre requires an installation of MPI. -* The hypre library currently does not directly support complex-valued systems. + * Configuration of hypre with threads requires an implementation of OpenMP. + Currently, only a subset of hypre is threaded. + + * To run on GPUs, hypre must be configured with support for the appropriate GPU toolkit. + Available options are CUDA for NVIDIA GPUs, HIP for AMD GPUs, and SYCL for Intel GPUs. + + * The hypre library currently does not directly support complex-valued systems. .. _choosing-interface: diff --git a/src/docs/usr-manual/ch-misc.rst b/src/docs/usr-manual/ch-misc.rst index c79362c85c..689f0fc430 100644 --- a/src/docs/usr-manual/ch-misc.rst +++ b/src/docs/usr-manual/ch-misc.rst @@ -10,42 +10,57 @@ General Information ****************************************************************************** +.. _getting-source: Getting the Source Code ============================================================================== -The most recent hypre distribution is available at -`https://github.com/hypre-space/hypre/tags`_ along with previous distribution -versions. +In this and the following sections, we discuss how to obtain and build hypre. We provide +instructions for two build systems: autotools (configure & make) and CMake. While autotools +is traditionally used on Unix-like systems (Linux, macOS, etc.), CMake provides +cross-platform support and is required for Windows builds. Both systems are actively +maintained and supported. There are two ways to obtain hypre: + +1. **Clone from GitHub (recommended)**:: + + git clone https://github.com/hypre-space/hypre.git + cd hypre/src + +2. **Download a Release** + + Download the latest release from our `GitHub releases page `_. + After extracting the archive, you'll find the source code in the ``src`` directory:: + + tar -xvf hypre-x.y.z.tar.gz + cd hypre-x.y.z/src + + where ``x.y.z`` represents the version number (e.g., 2.29.0). Building the Library ============================================================================== -In this and the following several sections, we discuss the steps to install and -use hypre. First, we focus on the primary method targeting Unix-like operating -systems, such as Linux, AIX, and Mac OS X. Then in `CMake instructions`_, we -explain an alternative approach using the CMake build system [CMakeWeb]_, which -is the best approach for building hypre on Windows systems in particular. +After obtaining the source code (see :ref:`getting-source`), there are two main ways to build hypre: -After unpacking the hypre tar file, the source code will be in the ``src`` -sub-directory of a directory named hypre-VERSION, where VERSION is the current -version number (e.g., hypre-2.29.0). +1. Using autotools (Configure & Make) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Move to the ``src`` sub-directory to build hypre for the host platform. The -simplest method is to configure, compile and install the libraries in -``./hypre/lib`` and ``./hypre/include`` directories, which is accomplished by: +The simplest method is to use the traditional configure and make: .. code-block:: bash - ./configure - make + cd src # Move to the source directory + ./configure # Configure the build system + make -j 4 # Use threads for a faster parallel build + make install # (Optional) Install hypre on a user-specified path via --prefix= -NOTE: when executing on an IBM platform ``configure`` must be executed under the -nopoe script (``./nopoe ./configure