diff --git a/.gitignore b/.gitignore index 4b1d9c762d..1962e925fd 100644 --- a/.gitignore +++ b/.gitignore @@ -20,10 +20,11 @@ HYPRE_config.h autom4te.cache src/TAGS hypre/ -cmbuild/ install/ AUTOTEST/*.dir +out/ .vscode +.vs ############### # Documentation @@ -42,17 +43,32 @@ src/docs/usr-manual/_build/ # Executables ############### src/test/ams_driver +src/test/ams_driver.exe src/test/ij +src/test/ij.exe src/test/ij_assembly +src/test/ij_assembly.exe src/test/ij_mv +src/test/ij_mv.exe src/test/ij_mm +src/test/ij_mm.exe src/test/maxwell_unscaled +src/test/maxwell_unscaled.exe src/test/sstruct +src/test/sstruct.exe src/test/sstruct_fac +src/test/sstruct_fac.exe src/test/struct +src/test/struct.exe src/test/struct_migrate +src/test/struct_migrate.exe src/test/zboxloop +src/test/zboxloop.exe src/examples/ex[0-9] -src/examples/ex1[0-6] +src/examples/ex[0-9].exe +src/examples/ex1[0-8] +src/examples/ex1[0-8].exe src/examples/ex5f +src/examples/ex5f.exe src/examples/ex12f +src/examples/ex12f.exe diff --git a/AUTOTEST/check-double.sh b/AUTOTEST/check-double.sh index 54aac333f3..86123092a1 100755 --- a/AUTOTEST/check-double.sh +++ b/AUTOTEST/check-double.sh @@ -32,7 +32,6 @@ shift cd $src_dir find . -type f -print | egrep '[.]*[.](c|cc|cpp|cxx|C|h|hpp|hxx|H)$' | - egrep -v '/cmbuild' | egrep -v '/docs' | egrep -v '/examples' | egrep -v '/FEI_mv' | diff --git a/AUTOTEST/check-int.sh b/AUTOTEST/check-int.sh index 153e6274a9..2d3b249351 100755 --- a/AUTOTEST/check-int.sh +++ b/AUTOTEST/check-int.sh @@ -32,7 +32,6 @@ shift cd $src_dir find . -type f -print | egrep '[.]*[.](c|cc|cpp|cxx|C|h|hpp|hxx|H)$' | - egrep -v '/cmbuild' | egrep -v '/docs' | egrep -v '/examples' | egrep -v '/FEI_mv' | diff --git a/AUTOTEST/check-license.sh b/AUTOTEST/check-license.sh index b48a8be3d7..bc2e9e8738 100755 --- a/AUTOTEST/check-license.sh +++ b/AUTOTEST/check-license.sh @@ -62,7 +62,6 @@ cat > check-license.remove < configuration options -mo: make options -ro: call the run script with these options -eo: call the examples script with these options -h|-help prints this usage information and exits - This script uses cmake to configure and compile the source in {src_dir}, then + This script uses cmake to configure and compile the source in {root_dir}/src, then optionally runs driver and example tests. - Example usage: $0 ../src -co -DCMAKE_BUILD_TYPE=Debug -ro: -ij + Example usage: $0 .. -co -DCMAKE_BUILD_TYPE=Debug -ro: -ij EOF exit ;; esac -# Set src_dir -src_dir=`cd $1; pwd` +# Set root_dir +root_dir=`cd $1; pwd` shift # Parse the rest of the command line @@ -68,38 +68,33 @@ test_dir=`pwd` output_dir=`pwd`/$testname.dir rm -fr $output_dir mkdir -p $output_dir -cd $src_dir -src_dir=`pwd` +cd $root_dir +root_dir=`pwd` -# Clean up the cmbuild directories (do it from src_dir as a precaution) -cd $src_dir -rm -fr `echo cmbuild/* | sed 's/[^ ]*README.txt//g'` +# Clean up the build directories (do it from root_dir as a precaution) +cd $root_dir +rm -fr build/* # Clean up the previous install -cd $src_dir -rm -fr hypre +cd $root_dir +rm -fr src/hypre # Configure -cd $src_dir/cmbuild -eval cmake $copts .. +cd $root_dir/build +eval cmake $copts ../src make $mopts install -# Make -cd $src_dir/cmbuild/test -make $mopts -mv -f $drivers ../../test - cd $test_dir # Run if [ -n "$rset" ]; then - ./test.sh run.sh $src_dir $ropts + ./test.sh run.sh $root_dir/src $ropts mv -f run.??? $output_dir fi # Examples if [ -n "$eset" ]; then - ./test.sh examples.sh $src_dir $eopts + ./test.sh examples.sh $root_dir/src $eopts mv -f examples.??? $output_dir fi @@ -110,7 +105,7 @@ do done # Clean up -cd $src_dir -rm -fr `echo cmbuild/* | sed 's/[^ ]*README.txt//g'` -rm -fr hypre -( cd $src_dir/test; rm -f $drivers; ./cleantest.sh ) +cd $root_dir +rm -fr build/* +rm -fr src/hypre +( cd $root_dir/src/test; rm -f $drivers; ./cleantest.sh ) diff --git a/AUTOTEST/machine-lassen.sh b/AUTOTEST/machine-lassen.sh index ff2f5c8f95..b778e4ba7e 100755 --- a/AUTOTEST/machine-lassen.sh +++ b/AUTOTEST/machine-lassen.sh @@ -33,6 +33,7 @@ output_dir=`pwd`/$testname.dir rm -fr $output_dir mkdir -p $output_dir src_dir=`cd $1; pwd` +root_dir=`cd $src_dir/..; pwd` shift # Basic build and run tests @@ -126,24 +127,24 @@ ro="-struct -rt -mpibind -save ${save}" ##################################### ## CUDA + CMake build (only) tests ## ##################################### -module -q load cmake/3.16.8 -module list cmake/3.16.8 |& grep "None found" +module -q load cmake/3.22 +module list cmake/3.22 |& grep "None found" mo="-j" # CUDA with UM + CMake -co="-DCMAKE_C_COMPILER=$(which xlc) -DCMAKE_CXX_COMPILER=$(which xlc++) -DCMAKE_CUDA_COMPILER=$(which nvcc) -DMPI_C_COMPILER=$(which mpicc) -DMPI_CXX_COMPILER=$(which mpicxx) -DHYPRE_WITH_CUDA=ON -DHYPRE_ENABLE_UNIFIED_MEMORY=ON -DCMAKE_BUILD_TYPE=Debug -DHYPRE_ENABLE_PERSISTENT_COMM=ON -DHYPRE_ENABLE_DEVICE_POOL=ON -DHYPRE_WITH_EXTRA_CFLAGS="\'"-qsuppress=1500-029"\'" -DHYPRE_WITH_EXTRA_CXXFLAGS="\'"-qsuppress=1500-029"\'" -DHYPRE_CUDA_SM=70" -./test.sh cmake.sh $src_dir -co: $co -mo: $mo +co="-DCMAKE_C_COMPILER=$(which xlc) -DCMAKE_CXX_COMPILER=$(which xlc++) -DCMAKE_CUDA_COMPILER=$(which nvcc) -DMPI_C_COMPILER=$(which mpicc) -DMPI_CXX_COMPILER=$(which mpicxx) -DHYPRE_ENABLE_CUDA=ON -DHYPRE_ENABLE_UNIFIED_MEMORY=ON -DCMAKE_BUILD_TYPE=Debug -DHYPRE_ENABLE_PERSISTENT_COMM=ON -DHYPRE_ENABLE_DEVICE_POOL=ON -DHYPRE_WITH_EXTRA_CFLAGS="\'"-qsuppress=1500-029"\'" -DHYPRE_WITH_EXTRA_CXXFLAGS="\'"-qsuppress=1500-029"\'" -DCMAKE_CUDA_ARCHITECTURES=70" +./test.sh cmake.sh $root_dir -co: $co -mo: $mo ./renametest.sh cmake $output_dir/cmake-cuda-um-ij # CUDA with UM [shared library] + CMake -co="-DCMAKE_C_COMPILER=$(which xlc) -DCMAKE_CXX_COMPILER=$(which xlc++) -DCMAKE_CUDA_COMPILER=$(which nvcc) -DMPI_C_COMPILER=$(which mpicc) -DMPI_CXX_COMPILER=$(which mpicxx) -DHYPRE_WITH_CUDA=ON -DHYPRE_ENABLE_UNIFIED_MEMORY=ON -DCMAKE_BUILD_TYPE=Debug -DHYPRE_WITH_OPENMP=ON -DHYPRE_ENABLE_HOPSCOTCH=ON -DHYPRE_ENABLE_SHARED=ON -DHYPRE_WITH_EXTRA_CFLAGS="\'"-qsuppress=1500-029"\'" -DHYPRE_WITH_EXTRA_CXXFLAGS="\'"-qsuppress=1500-029 "\'" -DHYPRE_CUDA_SM=70" -./test.sh cmake.sh $src_dir -co: $co -mo: $mo +co="-DCMAKE_C_COMPILER=$(which xlc) -DCMAKE_CXX_COMPILER=$(which xlc++) -DCMAKE_CUDA_COMPILER=$(which nvcc) -DMPI_C_COMPILER=$(which mpicc) -DMPI_CXX_COMPILER=$(which mpicxx) -DHYPRE_ENABLE_CUDA=ON -DHYPRE_ENABLE_UNIFIED_MEMORY=ON -DCMAKE_BUILD_TYPE=Debug -DHYPRE_ENABLE_OPENMP=ON -DHYPRE_ENABLE_HOPSCOTCH=ON -DBUILD_SHARED_LIBS=ON -DHYPRE_WITH_EXTRA_CFLAGS="\'"-qsuppress=1500-029"\'" -DHYPRE_WITH_EXTRA_CXXFLAGS="\'"-qsuppress=1500-029 "\'" -DCMAKE_CUDA_ARCHITECTURES=70" +./test.sh cmake.sh $root_dir -co: $co -mo: $mo ./renametest.sh cmake $output_dir/cmake-cuda-um-shared # CUDA w.o UM + CMake -co="-DCMAKE_C_COMPILER=$(which xlc) -DCMAKE_CXX_COMPILER=$(which xlc++) -DCMAKE_CUDA_COMPILER=$(which nvcc) -DMPI_C_COMPILER=$(which mpicc) -DMPI_CXX_COMPILER=$(which mpicxx) -DHYPRE_WITH_CUDA=ON -DCMAKE_BUILD_TYPE=Debug -DHYPRE_WITH_EXTRA_CFLAGS="\'"-qsuppress=1500-029"\'" -DHYPRE_WITH_EXTRA_CXXFLAGS="\'"-qsuppress=1500-029"\'" -DHYPRE_CUDA_SM=70" -./test.sh cmake.sh $src_dir -co: $co -mo: $mo +co="-DCMAKE_C_COMPILER=$(which xlc) -DCMAKE_CXX_COMPILER=$(which xlc++) -DCMAKE_CUDA_COMPILER=$(which nvcc) -DMPI_C_COMPILER=$(which mpicc) -DMPI_CXX_COMPILER=$(which mpicxx) -DHYPRE_ENABLE_CUDA=ON -DCMAKE_BUILD_TYPE=Debug -DHYPRE_WITH_EXTRA_CFLAGS="\'"-qsuppress=1500-029"\'" -DHYPRE_WITH_EXTRA_CXXFLAGS="\'"-qsuppress=1500-029"\'" -DCMAKE_CUDA_ARCHITECTURES=70" +./test.sh cmake.sh $root_dir -co: $co -mo: $mo ./renametest.sh cmake $output_dir/cmake-cuda-nonum-struct ####################### @@ -169,4 +170,3 @@ for errfile in $( find $output_dir ! -size 0 -name "*.err" ) do echo $errfile >&2 done - diff --git a/AUTOTEST/machine-tioga.sh b/AUTOTEST/machine-tioga.sh index 988d344fda..84b8776eaf 100755 --- a/AUTOTEST/machine-tioga.sh +++ b/AUTOTEST/machine-tioga.sh @@ -33,6 +33,7 @@ output_dir=`pwd`/$testname.dir rm -fr $output_dir mkdir -p $output_dir src_dir=`cd $1; pwd` +root_dir=`cd $src_dir/..; pwd` shift # Basic build and run tests @@ -68,6 +69,27 @@ ro="-ij-noilu -ams -struct -sstruct -rt -D HYPRE_NO_SAVED" ./test.sh basic.sh $src_dir -co: $co -mo: $mo -ro: $ro ./renametest.sh basic $output_dir/basic-hip-cpu +#################################### +## HIP + CMake build (only) tests ## +#################################### + +module -q load cmake/3.24.2 + +# HIP without UM + CMake (no run) +co="-DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC -DMPI_C_COMPILER=cc -DMPI_CXX_COMPILER=CC -DHYPRE_ENABLE_HIP=ON -DCMAKE_HIP_ARCHITECTURES=gfx90a -DCMAKE_BUILD_TYPE=Debug -DHYPRE_BUILD_TESTS=ON" +./test.sh cmake.sh $root_dir -co: $co -mo: $mo +./renametest.sh cmake $output_dir/cmake-hip-nonum + +# HIP with UM + Shared library + CMake (no run) +co="-DBUILD_SHARED_LIBS=ON -DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC -DMPI_C_COMPILER=cc -DMPI_CXX_COMPILER=CC -DHYPRE_ENABLE_HIP=ON -DCMAKE_HIP_ARCHITECTURES=gfx90a -DHYPRE_ENABLE_UNIFIED_MEMORY=ON -DCMAKE_BUILD_TYPE=Debug -DHYPRE_BUILD_TESTS=ON" +./test.sh cmake.sh $root_dir -co: $co -mo: $mo +./renametest.sh cmake $output_dir/cmake-hip-um-shared + +# CPU + CMake (no run) +co="-DCMAKE_C_COMPILER=cc -DCMAKE_CXX_COMPILER=CC -DMPI_C_COMPILER=cc -DMPI_CXX_COMPILER=CC -DCMAKE_BUILD_TYPE=Debug -DHYPRE_BUILD_TESTS=ON" +./test.sh cmake.sh $root_dir -co: $co -mo: $mo +./renametest.sh cmake $output_dir/cmake-cpu + # Echo to stderr all nonempty error files in $output_dir for errfile in $( find $output_dir ! -size 0 -name "*.err" ) do diff --git a/AUTOTEST/machine-tux.sh b/AUTOTEST/machine-tux.sh index d171907ba9..ed08800e69 100755 --- a/AUTOTEST/machine-tux.sh +++ b/AUTOTEST/machine-tux.sh @@ -34,6 +34,7 @@ output_dir=`pwd`/$testname.dir rm -fr $output_dir mkdir -p $output_dir src_dir=`cd $1; pwd` +root_dir=`cd $src_dir/..; pwd` shift # Organizing the tests from "fast" to "slow" @@ -153,31 +154,31 @@ ro="-ams -ij -sstruct -struct -lobpcg" eo="" co="" -./test.sh cmake.sh $src_dir -co: $co -mo: $mo +./test.sh cmake.sh $root_dir -co: $co -mo: $mo ./renametest.sh cmake $output_dir/cmake-default co="-DHYPRE_SEQUENTIAL=ON" -./test.sh cmake.sh $src_dir -co: $co -mo: $mo +./test.sh cmake.sh $root_dir -co: $co -mo: $mo ./renametest.sh cmake $output_dir/cmake-sequential co="-DHYPRE_SHARED=ON" -./test.sh cmake.sh $src_dir -co: $co -mo: $mo +./test.sh cmake.sh $root_dir -co: $co -mo: $mo ./renametest.sh cmake $output_dir/cmake-shared co="-DHYPRE_SINGLE=ON" -./test.sh cmake.sh $src_dir -co: $co -mo: $mo -ro: -single +./test.sh cmake.sh $root_dir -co: $co -mo: $mo -ro: -single ./renametest.sh cmake $output_dir/cmake-single co="-DHYPRE_LONG_DOUBLE=ON" -./test.sh cmake.sh $src_dir -co: $co -mo: $mo -ro: -longdouble +./test.sh cmake.sh $root_dir -co: $co -mo: $mo -ro: -longdouble ./renametest.sh cmake $output_dir/cmake-longdouble co="-DCMAKE_BUILD_TYPE=Debug" -./test.sh cmake.sh $src_dir -co: $co -mo: $mo -ro: $ro +./test.sh cmake.sh $root_dir -co: $co -mo: $mo -ro: $ro ./renametest.sh cmake $output_dir/cmake-debug co="-DHYPRE_BIGINT=ON" -./test.sh cmake.sh $src_dir -co: $co -mo: $mo -ro: $ro +./test.sh cmake.sh $root_dir -co: $co -mo: $mo -ro: $ro ./renametest.sh cmake $output_dir/cmake-bigint # cmake build doesn't currently support maxdim diff --git a/INSTALL.md b/INSTALL.md index d19410515e..bb13edb6a2 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -57,18 +57,18 @@ platforms) or Visual Studio (Windows). Here are the basic steps: 2. To build the library, run CMake on the top-level HYPRE source directory to generate files appropriate for the native build system. To prevent writing over the Makefiles in HYPRE's configure/make system above, only out-of-source - builds are currently allowed with CMake. The directory `src/cmbuild` - is provided in the release for convenience, but - alternative build directories may be created by the user: + builds are currently allowed with CMake. The directory `build` is provided in + the release for convenience, but alternative build directories may be created + by the user: - - Unix: From the `src/cmbuild` directory, type `cmake ..`. + - Unix: From the `build` directory, type `cmake ../src`. - - Windows: Set the source and build directories to `src` and `src/cmbuild`, - then click on 'Configure' following by 'Generate'. + - Windows: Set the source and build directories to `src` and `build`, then + click on 'Configure' following by 'Generate'. 3. To complete the build of the library, compile with the native build system: - - Unix: From the `src/cmbuild` directory, type `make install`. + - Unix: From the `build` directory, type `make install`. - Windows Visual Studio: Open the 'hypre' VS solution file generated by CMake and build the 'ALL_BUILD' target, then the 'INSTALL' target. @@ -78,7 +78,7 @@ Optional Features: Various configuration options can be set from within CMake. Change these as appropriate, then reconfigure/generate: -- Unix: From the `src/cmbuild` directory, type `ccmake ..`, change options, then +- Unix: From the `build` directory, type `ccmake ../src`, change options, then type `c` then `q`. - Windows VS: Change options, then click on 'Configure' then 'Generate'. diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 0000000000..5e7d2734cf --- /dev/null +++ b/build/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c65faf06c7..e1beeabf6c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,13 +3,20 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -cmake_minimum_required(VERSION 3.13...3.16) +cmake_minimum_required(VERSION 3.21) +message(STATUS "CMake Executable: ${CMAKE_COMMAND}") +message(STATUS "CMake Version: ${CMAKE_VERSION}") -if (${CMAKE_VERSION} VERSION_LESS 3.16) - cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) -else () - cmake_policy(VERSION 3.16) -endif () +# Include hypre's CMake utilities +include("${CMAKE_CURRENT_SOURCE_DIR}/config/cmake/HYPRE_CMakeUtilities.cmake") + +# Set cmake policies +if(POLICY CMP0146) + cmake_policy(SET CMP0146 NEW) # Ensure CUDA toolkit is found +endif() +if(POLICY CMP0104) + cmake_policy(SET CMP0104 NEW) # CUDA_ARCHITECTURES is allowed to be set in cache +endif() # The version number. set(HYPRE_VERSION 2.32.0) @@ -19,642 +26,421 @@ set(HYPRE_TIME 00:00:00) set(HYPRE_BUGS https://github.com/hypre-space/hypre/issues) set(HYPRE_SRCDIR "${PROJECT_SOURCE_DIR}") +# Display the hypre version +setup_git_version_info("${CMAKE_CURRENT_SOURCE_DIR}/../.git") +if (GIT_VERSION_FOUND) + message(STATUS "Hypre Version: ${HYPRE_DEVELOP_STRING} (${HYPRE_BRANCH_NAME})") +else() + message(STATUS "NOTE: Could not find .git directory") + message(STATUS "Hypre Version: ${HYPRE_VERSION}") +endif() + +# Set the project name set(PROJECT_NAME HYPRE) project(${PROJECT_NAME} VERSION ${HYPRE_VERSION} LANGUAGES C) +# Create the HYPRE library object +add_library(${PROJECT_NAME}) + # We use C99 by default, but users are free to specify any newer standard version -set(CMAKE_C_STANDARD 99) +if (NOT CMAKE_C_STANDARD) + set(CMAKE_C_STANDARD 99 CACHE STRING "Standard to use for the C compiler") + set(CMAKE_C_STANDARD_REQUIRED ON) +endif () +# Raise an error if the source and binary directories are the same if (${HYPRE_SOURCE_DIR} STREQUAL ${HYPRE_BINARY_DIR}) message(FATAL_ERROR "In-place build not allowed! Please use a separate build directory. See the Users Manual or INSTALL file for details.") endif () -if (EXISTS ${HYPRE_SOURCE_DIR}/../.git) - execute_process(COMMAND git -C ${HYPRE_SOURCE_DIR} describe --match v* --long --abbrev=9 - OUTPUT_VARIABLE develop_string - OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND git -C ${HYPRE_SOURCE_DIR} describe --match v* --abbrev=0 - OUTPUT_VARIABLE develop_lastag - OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND git -C ${HYPRE_SOURCE_DIR} rev-list --count ${develop_lastag}..HEAD - OUTPUT_VARIABLE develop_number - OUTPUT_STRIP_TRAILING_WHITESPACE) - execute_process(COMMAND git -C ${HYPRE_SOURCE_DIR} rev-parse --abbrev-ref HEAD - OUTPUT_VARIABLE develop_branch - OUTPUT_STRIP_TRAILING_WHITESPACE) - if (${develop_string} MATCHES ".*") - set(HYPRE_DEVELOP_STRING ${develop_string}) - set(HYPRE_DEVELOP_NUMBER ${develop_number}) - set(HYPRE_BRANCH_NAME ${develop_branch}) - if (develop_branch MATCHES "master") - set(HYPRE_DEVELOP_BRANCH ${develop_branch}) - else () - message(STATUS "NOTE: On branch ${develop_branch}, not the main development branch") - endif () - else () - message(STATUS "NOTE: Could not describe development branch") - endif () -else () - message(STATUS "NOTE: Could not find .git directory") +# Set default installation directory, but provide a means for users to change +if (NOT CMAKE_INSTALL_PREFIX OR CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(CMAKE_INSTALL_PREFIX "${HYPRE_SOURCE_DIR}/hypre" CACHE PATH "Installation directory for HYPRE" FORCE) endif () +# Ensure RPATH/RUNPATH is set properly during the build +set(CMAKE_SKIP_BUILD_RPATH FALSE CACHE BOOL "Include rpath in the binaries being built") +set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE CACHE BOOL "Use different paths for build and install") +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE BOOL "Use link-time paths for install rpath") +set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib" CACHE STRING "Where binaries look for shared libraries") + +# Build static library by default +option(BUILD_SHARED_LIBS "Build shared libraries" OFF) + # Set cmake module path set(CMAKE_MODULE_PATH "${HYPRE_SOURCE_DIR}/config/cmake" "${CMAKE_MODULE_PATH}") -include(HYPRE_CMakeUtilities) -# Set default installation directory, but provide a means for users to change -set(HYPRE_INSTALL_PREFIX "${PROJECT_SOURCE_DIR}/hypre" CACHE PATH - "Installation directory for HYPRE") -if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - set(CMAKE_INSTALL_PREFIX "${HYPRE_INSTALL_PREFIX}" CACHE INTERNAL "" FORCE) -endif () +# Set CMAKE_BUILD_TYPE to default to Release if not already specified +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type options: Debug, Release, RelWithDebInfo, or MinSizeRel." FORCE) +endif() -# Set default compile optimization flag -set(HYPRE_BUILD_TYPE "Release" CACHE STRING - "Optimization flags: set to Debug, Release, RelWithDebInfo, or MinSizeRel") +# Set the possible values of build type for ccmake/cmake-gui +set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") + +# Set debug flags based on compiler +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + if(CMAKE_C_COMPILER_ID MATCHES "Intel") + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O0 -Wall -Wremarks> + $<$: + -O0 -Wall -Wremarks + -Wno-unused-lambda-capture + -Wno-unused-local-typedef + -Wno-unused-variable + -Wno-unused-parameter + -Wno-unused-function + >) + + elseif(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O0 -Wall> + $<$:-O0 -Wall> + $<$:-O0 -Xcompiler=-Wall> + $<$:-O0 -Wall>) + + elseif(CMAKE_C_COMPILER_ID MATCHES "MSVC") + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:/Od /Zi /W4> + $<$:/Od /Zi /W4>) + + elseif(CMAKE_C_COMPILER_ID MATCHES "PGI|NVHPC") + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O0 -Minform=warn> + $<$:-O0 -Minform=warn> + $<$:-O0 -g -Minform=warn>) + + elseif(CMAKE_C_COMPILER_ID MATCHES "XL|XLClang") + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O0-qinfo=all> + $<$:-O0 -qinfo=all>) + + else() + message(WARNING "Unknown C compiler '${CMAKE_C_COMPILER_ID}', using default debug flags") + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O0> + $<$:-O0>) + endif() +endif() -if (NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - set(CMAKE_BUILD_TYPE "${HYPRE_BUILD_TYPE}" CACHE INTERNAL "" FORCE) - # Set the possible values of build type for cmake-gui - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS - "Debug" "Release" "MinSizeRel" "RelWithDebInfo") -endif () +# Print the configuration +message(STATUS "Build type: ${CMAKE_BUILD_TYPE}") +message(STATUS "Shared library: ${BUILD_SHARED_LIBS}") +message(STATUS "Installation directory: ${CMAKE_INSTALL_PREFIX}") +message(STATUS "Using C standard: ${CMAKE_C_STANDARD}") + +# Base boolean configuration options +set_hypre_option(BASE HYPRE_ENABLE_BIGINT "Use long long int for HYPRE_Int and HYPRE_BigInt" OFF) +set_hypre_option(BASE HYPRE_ENABLE_MIXEDINT "Use long long int for HYPRE_BigInt, int for HYPRE_Int" OFF) +set_hypre_option(BASE HYPRE_ENABLE_SINGLE "Use float for HYPRE_Real" OFF) +set_hypre_option(BASE HYPRE_ENABLE_LONG_DOUBLE "Use long double for HYPRE_Real" OFF) +set_hypre_option(BASE HYPRE_ENABLE_COMPLEX "Use complex values" OFF) +set_hypre_option(BASE HYPRE_ENABLE_HYPRE_BLAS "Use internal BLAS library" ON) +set_hypre_option(BASE HYPRE_ENABLE_HYPRE_LAPACK "Use internal LAPACK library" ON) +set_hypre_option(BASE HYPRE_ENABLE_PERSISTENT_COMM "Use persistent communication" OFF) +set_hypre_option(BASE HYPRE_ENABLE_FEI "Use FEI" OFF) # TODO: Add this cmake feature +set_hypre_option(BASE HYPRE_ENABLE_HOPSCOTCH "Use hopscotch hashing with OpenMP" OFF) +set_hypre_option(BASE HYPRE_ENABLE_OPENMP "Use OpenMP" OFF) +set_hypre_option(BASE HYPRE_ENABLE_MPI "Use MPI" ON) +set_hypre_option(BASE HYPRE_ENABLE_LTO "Use Link-Time Optimization (LTO)" OFF) +set_hypre_option(BASE HYPRE_ENABLE_PRINT_ERRORS "Print HYPRE errors" OFF) +set_hypre_option(SKIP HYPRE_ENABLE_TIMING "Use HYPRE timing routines" OFF) +set_hypre_option(SKIP HYPRE_ENABLE_STRICT_CHECKING "Use strict error checking (Sequential only)" OFF) +set_hypre_option(BASE HYPRE_ENABLE_TEST_USING_HOST "Execute tests on host (CPU)" OFF) +set_hypre_option(BASE HYPRE_ENABLE_HOST_MEMORY "Use host memory" ON) +set_hypre_option(BASE HYPRE_BUILD_EXAMPLES "Build examples" OFF) +set_hypre_option(BASE HYPRE_BUILD_TESTS "Build tests" OFF) -# Configuration options -option(HYPRE_ENABLE_SHARED "Build a shared library" OFF) -option(HYPRE_ENABLE_BIGINT "Use long long int for HYPRE_Int" OFF) -option(HYPRE_ENABLE_MIXEDINT "Use long long int for HYPRE_BigInt, int for HYPRE_INT" OFF) -option(HYPRE_ENABLE_SINGLE "Use float for HYPRE_Real" OFF) -option(HYPRE_ENABLE_LONG_DOUBLE "Use long double for HYPRE_Real" OFF) -option(HYPRE_ENABLE_COMPLEX "Use complex values" OFF) -option(HYPRE_ENABLE_HYPRE_BLAS "Use internal BLAS library" ON) -option(HYPRE_ENABLE_HYPRE_LAPACK "Use internal LAPACK library" ON) -option(HYPRE_ENABLE_PERSISTENT_COMM "Use persistent communication" OFF) -option(HYPRE_ENABLE_FEI "Use FEI" OFF) # TODO: Add this cmake feature -option(HYPRE_WITH_MPI "Compile with MPI" ON) -option(HYPRE_WITH_GPU_AWARE_MPI "Compile with device aware GPU support" OFF) -option(HYPRE_WITH_OPENMP "Use OpenMP" OFF) -option(HYPRE_ENABLE_HOPSCOTCH "Use hopscotch hashing with OpenMP" OFF) -option(HYPRE_WITH_SUPERLU "Use TPL SuperLU" OFF) -option(HYPRE_WITH_DSUPERLU "Use TPL SuperLU_Dist" OFF) -option(HYPRE_WITH_MAGMA "Use TPL MAGMA" OFF) -option(HYPRE_WITH_CALIPER "Use Caliper" OFF) -option(HYPRE_PRINT_ERRORS "Print HYPRE errors" OFF) -option(HYPRE_TIMING "Use HYPRE timing routines" OFF) -option(HYPRE_BUILD_EXAMPLES "Build examples" OFF) -option(HYPRE_BUILD_TESTS "Build tests" OFF) -option(HYPRE_USING_HOST_MEMORY "Use host memory" ON) -set(HYPRE_WITH_EXTRA_CFLAGS "" CACHE STRING "Define extra C compile flags") -set(HYPRE_WITH_EXTRA_CXXFLAGS "" CACHE STRING "Define extra CXX compile flags") # GPU options -option(HYPRE_WITH_CUDA "Use CUDA. Require cuda-8.0 or higher" OFF) -option(HYPRE_WITH_SYCL "Use SYCL" OFF) -option(HYPRE_ENABLE_UNIFIED_MEMORY "Use unified memory for allocating the memory" OFF) -option(HYPRE_ENABLE_DEVICE_MALLOC_ASYNC "Use device async malloc" OFF) +set_hypre_option(GPU HYPRE_ENABLE_CUDA "Use CUDA. Require cuda-10.0 or higher" OFF) +set_hypre_option(GPU HYPRE_ENABLE_HIP "Use HIP" OFF) +set_hypre_option(GPU HYPRE_ENABLE_SYCL "Use SYCL" OFF) +set_hypre_option(GPU HYPRE_ENABLE_GPU_AWARE_MPI "Use device aware MPI support" OFF) +set_hypre_option(GPU HYPRE_ENABLE_UNIFIED_MEMORY "Use unified memory for allocating the memory" OFF) +set_hypre_option(GPU HYPRE_ENABLE_DEVICE_MALLOC_ASYNC "Use device async malloc" OFF) +set_hypre_option(GPU HYPRE_ENABLE_THRUST_ASYNC "Use Thrust par_nosync policy" OFF) +set_hypre_option(GPU HYPRE_ENABLE_GPU_PROFILING "Use NVTX on CUDA or rocTX on HIP" OFF) # CUDA options -option(HYPRE_ENABLE_CUDA_STREAMS "Use CUDA streams" ON) -option(HYPRE_ENABLE_CUSPARSE "Use cuSPARSE" ON) -option(HYPRE_ENABLE_CUSOLVER "Use cuSOLVER" OFF) -option(HYPRE_ENABLE_DEVICE_POOL "Use device memory pool" OFF) -option(HYPRE_ENABLE_CUBLAS "Use cuBLAS" OFF) -option(HYPRE_ENABLE_CURAND "Use cuRAND" ON) -option(HYPRE_ENABLE_GPU_PROFILING "Use NVTX on CUDA" OFF) -set(HYPRE_CUDA_SM "70" CACHE STRING "Target CUDA architecture.") +set_hypre_option(CUDA HYPRE_ENABLE_CUDA_STREAMS "Use CUDA streams" ON) +set_hypre_option(CUDA HYPRE_ENABLE_CUSPARSE "Use cuSPARSE" ON) +set_hypre_option(CUDA HYPRE_ENABLE_CUSOLVER "Use cuSOLVER" ON) +set_hypre_option(CUDA HYPRE_ENABLE_CUBLAS "Use cuBLAS" ON) +set_hypre_option(CUDA HYPRE_ENABLE_CURAND "Use cuRAND" ON) +set_hypre_option(CUDA HYPRE_ENABLE_DEVICE_POOL "Use CUB's device memory pool" OFF) +# HIP options +set_hypre_option(HIP HYPRE_ENABLE_ROCBLAS "Use rocBLAS" ON) +set_hypre_option(HIP HYPRE_ENABLE_ROCSPARSE "Use rocSPARSE" ON) +set_hypre_option(HIP HYPRE_ENABLE_ROCRAND "Use rocRAND" ON) +set_hypre_option(HIP HYPRE_ENABLE_ROCSOLVER "Use rocSOLVER" ON) # oneAPI options -option(HYPRE_ENABLE_ONEMKLSPARSE "Use oneMKL sparse" ON) -option(HYPRE_ENABLE_ONEMKLBLAS "Use oneMKL blas" ON) -option(HYPRE_ENABLE_ONEMKLRAND "Use oneMKL rand" ON) -set(HYPRE_SYCL_TARGET "" CACHE STRING "Target SYCL architecture, e.g. 'spir64_gen'.") -set(HYPRE_SYCL_TARGET_BACKEND "" CACHE STRING "Additional SYCL backend options, e.g. '-device 12.1.0,12.4.0'.") -# Umpire resource management options -option(HYPRE_WITH_UMPIRE "Use Umpire Allocator for device and unified memory" OFF) -option(HYPRE_WITH_UMPIRE_HOST "Use Umpire Allocator for host memory" OFF) -option(HYPRE_WITH_UMPIRE_DEVICE "Use Umpire Allocator for device memory" OFF) -option(HYPRE_WITH_UMPIRE_UM "Use Umpire Allocator for unified memory" OFF) -option(HYPRE_WITH_UMPIRE_PINNED "Use Umpire Allocator for pinned memory" OFF) -option(TPL_UMPIRE_LIBRARIES "List of absolute paths to Umpire link libraries [].") -option(TPL_UMPIRE_INCLUDE_DIRS "List of absolute paths to Umpire include directories [].") - -option(TPL_SUPERLU_LIBRARIES "List of absolute paths to SuperLU link libraries [].") -option(TPL_SUPERLU_INCLUDE_DIRS "List of absolute paths to SuperLU include directories [].") -option(TPL_DSUPERLU_LIBRARIES "List of absolute paths to SuperLU_Dist link libraries [].") -option(TPL_DSUPERLU_INCLUDE_DIRS "List of absolute paths to SuperLU_Dist include directories [].") -option(TPL_MAGMA_LIBRARIES "List of absolute paths to MAGMA link libraries [].") -option(TPL_MAGMA_INCLUDE_DIRS "List of absolute paths to MAGMA include directories [].") -option(TPL_BLAS_LIBRARIES "Optional list of absolute paths to BLAS libraries, otherwise use FindBLAS to locate [].") -option(TPL_LAPACK_LIBRARIES "Optional list of absolute paths to LAPACK libraries, otherwise use FindLAPACK to locate [].") -option(TPL_FEI_INCLUDE_DIRS "List of absolute paths to FEI include directories [].") +set_hypre_option(SYCL HYPRE_ENABLE_ONEMKLSPARSE "Use oneMKL sparse" ON) +set_hypre_option(SYCL HYPRE_ENABLE_ONEMKLBLAS "Use oneMKL blas" ON) +set_hypre_option(SYCL HYPRE_ENABLE_ONEMKLRAND "Use oneMKL rand" ON) +# TPL options +set_hypre_option(TPL HYPRE_ENABLE_UMPIRE "Use Umpire Allocator for device and unified memory" OFF) +set_hypre_option(TPL HYPRE_ENABLE_UMPIRE_HOST "Use Umpire Allocator for host memory" OFF) +set_hypre_option(TPL HYPRE_ENABLE_UMPIRE_PINNED "Use Umpire Allocator for pinned memory" OFF) +set_hypre_option(TPL HYPRE_ENABLE_UMPIRE_DEVICE "Use Umpire Allocator for device memory" OFF) +set_hypre_option(TPL HYPRE_ENABLE_UMPIRE_UM "Use Umpire Allocator for unified memory" OFF) +set_hypre_option(TPL HYPRE_ENABLE_SUPERLU "Use TPL SuperLU" OFF) +set_hypre_option(TPL HYPRE_ENABLE_DSUPERLU "Use TPL SuperLU_Dist" OFF) +set_hypre_option(TPL HYPRE_ENABLE_MAGMA "Use TPL MAGMA" OFF) +set_hypre_option(TPL HYPRE_ENABLE_CALIPER "Use TPL Caliper" OFF) +set_hypre_option(SKIP TPL_UMPIRE_LIBRARIES "Absolute paths to Umpire link libraries [Optional]." "") +set_hypre_option(SKIP TPL_UMPIRE_INCLUDE_DIRS "Absolute paths to Umpire include directories [Optional]." "") +set_hypre_option(SKIP TPL_SUPERLU_LIBRARIES "Absolute paths to SuperLU link libraries [Optional]." "") +set_hypre_option(SKIP TPL_SUPERLU_INCLUDE_DIRS "Absolute paths to SuperLU include directories [Optional]." "") +set_hypre_option(SKIP TPL_DSUPERLU_LIBRARIES "Absolute paths to SuperLU_Dist link libraries [Optional]." "") +set_hypre_option(SKIP TPL_DSUPERLU_INCLUDE_DIRS "Absolute paths to SuperLU_Dist include directories [Optional]." "") +set_hypre_option(SKIP TPL_MAGMA_LIBRARIES "Absolute paths to MAGMA link libraries [Optional]." "") +set_hypre_option(SKIP TPL_MAGMA_INCLUDE_DIRS "Absolute paths to MAGMA include directories [Optional]." "") +set_hypre_option(SKIP TPL_CALIPER_LIBRARIES "Absolute paths to Caliper link libraries [Optional]." "") +set_hypre_option(SKIP TPL_CALIPER_INCLUDE_DIRS "Absolute paths to Caliper include directories [Optional]." "") +set_hypre_option(SKIP TPL_BLAS_LIBRARIES "Absolute paths to BLAS link libraries [Optional]." "") +set_hypre_option(SKIP TPL_LAPACK_LIBRARIES "Absolute paths to LAPACK link libraries [Optional]." "") +set_hypre_option(SKIP TPL_FEI_INCLUDE_DIRS "Absolute paths to FEI include directories [Optional]." "") + +# Extra flags +set(HYPRE_WITH_EXTRA_CFLAGS "" CACHE STRING "Define extra C compile flags") +set(HYPRE_WITH_EXTRA_CXXFLAGS "" CACHE STRING "Define extra CXX compile flags") +set(HYPRE_SYCL_TARGET "" CACHE STRING "Target SYCL architecture, e.g. 'spir64_gen'.") +set(HYPRE_SYCL_TARGET_BACKEND "" CACHE STRING "Additional SYCL backend options, e.g. '-device 12.1.0,12.4.0'.") + +# Set possible values for the Fortran name mangling schemes +set(HYPRE_ENABLE_FMANGLE "UNSPECIFIED" CACHE STRING "Set the Fortran name mangling scheme") +set(HYPRE_ENABLE_FMANGLE_BLAS "UNSPECIFIED" CACHE STRING "Set the Fortran BLAS name mangling scheme") +set(HYPRE_ENABLE_FMANGLE_LAPACK "UNSPECIFIED" CACHE STRING "Set the Fortran LAPACK name mangling scheme") +foreach(VARNAME IN ITEMS FMANGLE FMANGLE_BLAS FMANGLE_LAPACK) + set_property(CACHE HYPRE_ENABLE_${VARNAME} PROPERTY STRINGS UNSPECIFIED NONE ONE_UNDERSCORE TWO_UNDERSCORES CAPS PRE_POST_UNDERSCORE 0 1 2 3 4 5) +endforeach() + +# Set internal hypre variable for each Fortran mangling scheme +process_fmangling_scheme(FMANGLE "name") +process_fmangling_scheme(FMANGLE_BLAS "BLAS") +process_fmangling_scheme(FMANGLE_LAPACK "LAPACK") # Set config name values -if (HYPRE_ENABLE_SHARED) - set(HYPRE_SHARED ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_BIGINT) - set(HYPRE_BIGINT ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_MIXEDINT) - set(HYPRE_MIXEDINT ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_SINGLE) - set(HYPRE_SINGLE ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_LONG_DOUBLE) - set(HYPRE_LONG_DOUBLE ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_COMPLEX) - set(HYPRE_COMPLEX ON CACHE BOOL "" FORCE) -endif () - -if (CMAKE_BUILD_TYPE STREQUAL "Debug") - set(HYPRE_DEBUG ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_HYPRE_BLAS) - set(HYPRE_USING_HYPRE_BLAS ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_ENABLE_HYPRE_LAPACK) - set(HYPRE_USING_HYPRE_LAPACK ON CACHE BOOL "" FORCE) -endif () +set_internal_hypre_option("" BIGINT) +set_internal_hypre_option("" MIXEDINT) +set_internal_hypre_option("" SINGLE) +set_internal_hypre_option("" LONG_DOUBLE) +set_internal_hypre_option("" COMPLEX) +set_internal_hypre_option("" TEST_USING_HOST) +set_internal_hypre_option(USING HYPRE_BLAS) +set_internal_hypre_option(USING HYPRE_LAPACK) +set_internal_hypre_option(USING HOPSCOTCH) +set_internal_hypre_option(USING GPU_AWARE_MPI) +set_internal_hypre_option(USING GPU_STREAMS) +set_internal_hypre_option(USING DEVICE_POOL) +set_internal_hypre_option(USING OPENMP) +set_internal_hypre_option(USING CUDA) +set_internal_hypre_option(USING HIP) +set_internal_hypre_option(USING SYCL) +set_internal_hypre_option(USING MAGMA) +set_internal_hypre_option(USING CALIPER) + +# Check for conflicting options +ensure_options_differ(HYPRE_ENABLE_SINGLE HYPRE_ENABLE_LONG_DOUBLE) +ensure_options_differ(HYPRE_ENABLE_CUDA HYPRE_ENABLE_HIP) +ensure_options_differ(HYPRE_ENABLE_CUDA HYPRE_ENABLE_SYCL) +ensure_options_differ(HYPRE_ENABLE_CUDA HYPRE_ENABLE_BIGINT) +ensure_options_differ(HYPRE_ENABLE_CUDA HYPRE_ENABLE_LONG_DOUBLE) +ensure_options_differ(HYPRE_ENABLE_SYCL HYPRE_ENABLE_HIP ) +ensure_options_differ(HYPRE_ENABLE_SYCL HYPRE_ENABLE_BIGINT) +ensure_options_differ(HYPRE_ENABLE_SYCL HYPRE_ENABLE_LONG_DOUBLE) +ensure_options_differ(HYPRE_ENABLE_HIP HYPRE_ENABLE_BIGINT) +ensure_options_differ(HYPRE_ENABLE_HIP HYPRE_ENABLE_LONG_DOUBLE) +ensure_options_differ(HYPRE_ENABLE_LTO HYPRE_ENABLE_SYCL) # TODO: allow LTO with SYCL + +# Enable CTest support if tests are being built +if (HYPRE_BUILD_TESTS) + include(CTest) + enable_testing() +endif() -if (HYPRE_ENABLE_PERSISTENT_COMM) - set(HYPRE_USING_PERSISTENT_COMM ON CACHE BOOL "" FORCE) +if (BUILD_SHARED_LIBS) + set(HYPRE_SHARED ON CACHE INTERNAL "") endif () -if (HYPRE_WITH_MPI) - set(HYPRE_HAVE_MPI ON CACHE BOOL "" FORCE) - set(HYPRE_SEQUENTIAL OFF CACHE BOOL "" FORCE) +if (HYPRE_ENABLE_MPI) + set(HYPRE_HAVE_MPI ON CACHE INTERNAL "") + set(HYPRE_SEQUENTIAL OFF CACHE INTERNAL "") else () - set(HYPRE_SEQUENTIAL ON CACHE BOOL "" FORCE) -endif () - -if (HYPRE_WITH_OPENMP) - set(HYPRE_USING_OPENMP ON CACHE BOOL "" FORCE) -endif () + set(HYPRE_SEQUENTIAL ON CACHE INTERNAL "") +endif () + +# Add strict checking option +if (HYPRE_ENABLE_STRICT_CHECKING) + set(HYPRE_ENABLE_MPI OFF CACHE INTERNAL "") + set(HYPRE_HAVE_MPI OFF CACHE INTERNAL "") + set(HYPRE_SEQUENTIAL ON CACHE INTERNAL "") + set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "" FORCE) + if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O0 -Wall -Wextra -pedantic -Wfloat-conversion -fsingle-precision-constant -Werror> + $<$:-Wall -Wshadow -fno-implicit-templates -Woverloaded-virtual -ansi -pedantic>) + + elseif(CMAKE_C_COMPILER_ID MATCHES "Intel") + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O0 -Wall -Werror> + $<$:-O0 -Wall -Werror>) + + elseif(CMAKE_C_COMPILER_ID MATCHES "XL|XLClang") + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O0 -qinfo=all> + $<$:-O0 -qinfo=all>) + + else() + message(WARNING "Unknown C compiler '${CMAKE_C_COMPILER_ID}', using default debug flags") + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:-O0> + $<$:-O0>) + endif() -if (HYPRE_ENABLE_HOPSCOTCH) - set(HYPRE_HOPSCOTCH ON CACHE BOOL "" FORCE) -endif () + message(STATUS "Enabled strict error checking") +endif() -if (HYPRE_WITH_SUPERLU) - set(HYPRE_USING_SUPERLU ON CACHE BOOL "" FORCE) +if (HYPRE_ENABLE_SUPERLU) + set(HYPRE_USING_SUPERLU ON CACHE INTERNAL "") add_compile_definitions(HAVE_SUPERLU) endif () -if (HYPRE_WITH_DSUPERLU) - set(HYPRE_USING_DSUPERLU ON CACHE BOOL "" FORCE) - set(HYPRE_USING_HYPRE_BLAS OFF CACHE BOOL "" FORCE) - set(HYPRE_USING_HYPRE_LAPACK OFF CACHE BOOL "" FORCE) -endif () - -if (HYPRE_WITH_MAGMA) - set(HYPRE_USING_MAGMA ON CACHE BOOL "" FORCE) +if (HYPRE_ENABLE_DSUPERLU) + set(HYPRE_USING_DSUPERLU ON CACHE INTERNAL "") + set(HYPRE_USING_HYPRE_BLAS OFF CACHE INTERNAL "") + set(HYPRE_USING_HYPRE_LAPACK OFF CACHE INTERNAL "") endif () if (HYPRE_ENABLE_FEI) - set(HYPRE_USING_FEI ON CACHE BOOL "" FORCE) + set(HYPRE_USING_FEI ON CACHE INTERNAL "") message(WARNING "CMake support for FEI is not complete!") endif () -if (HYPRE_WITH_CALIPER) - set(HYPRE_USING_CALIPER ON CACHE BOOL "" FORCE) -endif () - +# FEI doesn't currently compile with shared if (HYPRE_SHARED OR HYPRE_BIGINT OR HYPRE_SINGLE OR HYPRE_LONG_DOUBLE) - # FEI doesn't currently compile with shared - set(HYPRE_USING_FEI OFF CACHE BOOL "" FORCE) + set(HYPRE_USING_FEI OFF CACHE INTERNAL "") + set(HYPRE_ENABLE_FEI OFF CACHE BOOL "Add FEI support" FORCE) endif () if (HYPRE_SEQUENTIAL) - set(HYPRE_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + set(HYPRE_BUILD_EXAMPLES OFF CACHE BOOL "Build examples" FORCE) endif () -if (HYPRE_WITH_UMPIRE) - set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_DEVICE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_UM ON CACHE BOOL "" FORCE) +if (HYPRE_ENABLE_UMPIRE) + set(HYPRE_USING_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_DEVICE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_UM ON CACHE INTERNAL "") endif () -if (HYPRE_WITH_UMPIRE_HOST) - set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_HOST ON CACHE BOOL "" FORCE) +if (HYPRE_ENABLE_UMPIRE_HOST) + set(HYPRE_ENABLE_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_HOST ON CACHE INTERNAL "") endif () -if (HYPRE_WITH_UMPIRE_DEVICE) - set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_DEVICE ON CACHE BOOL "" FORCE) +if (HYPRE_ENABLE_UMPIRE_DEVICE) + set(HYPRE_ENABLE_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_DEVICE ON CACHE INTERNAL "") endif () -if (HYPRE_WITH_UMPIRE_UM) - set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_UM ON CACHE BOOL "" FORCE) +if (HYPRE_ENABLE_UMPIRE_UM) + set(HYPRE_ENABLE_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_UM ON CACHE INTERNAL "") endif () -if (HYPRE_WITH_UMPIRE_PINNED) - set(HYPRE_USING_UMPIRE ON CACHE BOOL "" FORCE) - set(HYPRE_USING_UMPIRE_PINNED ON CACHE BOOL "" FORCE) +if (HYPRE_ENABLE_UMPIRE_PINNED) + set(HYPRE_ENABLE_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE ON CACHE INTERNAL "") + set(HYPRE_USING_UMPIRE_PINNED ON CACHE INTERNAL "") endif () -# CUDA -if (HYPRE_WITH_CUDA) - enable_language(CXX) - message(STATUS "Enabled support for CXX.") - - # Enforce C++11 - if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 11) - set(CMAKE_CXX_STANDARD 11) - endif () - set(CMAKE_CXX_STANDARD_REQUIRED ON) - - message(STATUS "Using CXX standard: c++${CMAKE_CXX_STANDARD}") - - # Add any extra CXX compiler flags HYPRE_WITH_EXTRA_CXXFLAGS - if (NOT HYPRE_WITH_EXTRA_CXXFLAGS STREQUAL "") - string(REPLACE " " ";" HYPRE_WITH_EXTRA_CXXFLAGS "${HYPRE_WITH_EXTRA_CXXFLAGS}") - add_compile_options("$<$:${HYPRE_WITH_EXTRA_CXXFLAGS}>") - endif () - - # Check if CUDA is available, then enable it - include(CheckLanguage) - check_language(CUDA) - - # Use ${CMAKE_CXX_COMPILER} as the cuda host compiler. - if (NOT CMAKE_CUDA_HOST_COMPILER) - set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) - endif () - - if (CMAKE_CUDA_COMPILER) - - enable_language(CUDA) - message(STATUS "Enabled support for CUDA.") - - if (NOT CMAKE_CUDA_STANDARD OR CMAKE_CUDA_STANDARD EQUAL 98) - set(CMAKE_CUDA_STANDARD 11) - endif () - - set(CMAKE_CUDA_STANDARD_REQUIRED ON CACHE BOOL "" FORCE) - - if (HYPRE_ENABLE_DEVICE_MALLOC_ASYNC) - set(HYPRE_USING_DEVICE_MALLOC_ASYNC ON CACHE BOOL "" FORCE) - endif () - - set(HYPRE_USING_CUDA ON CACHE BOOL "" FORCE) - set(HYPRE_USING_GPU ON CACHE BOOL "" FORCE) - - if (HYPRE_ENABLE_UNIFIED_MEMORY) - set(HYPRE_USING_UNIFIED_MEMORY ON CACHE BOOL "" FORCE) - else () - set(HYPRE_USING_DEVICE_MEMORY ON CACHE BOOL "" FORCE) - endif () - - # Check if examples are enabled, but not unified memory - if (HYPRE_BUILD_EXAMPLES AND NOT HYPRE_ENABLE_UNIFIED_MEMORY) - message(WARNING "Running the examples on GPUs requires Unified Memory! - Examples will not be built!") - set(HYPRE_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) - endif () - - if (CMAKE_VERSION VERSION_LESS 3.18.0) - add_compile_options("$<$:-arch=sm_${HYPRE_CUDA_SM}>") - else () - set(CMAKE_CUDA_ARCHITECTURES "${HYPRE_CUDA_SM}") - endif () - message(STATUS "Using CUDA architecture: ${HYPRE_CUDA_SM}") - - add_compile_options("$<$:-expt-extended-lambda>") - - set(HYPRE_USING_HOST_MEMORY OFF CACHE BOOL "" FORCE) - - if (HYPRE_ENABLE_CUDA_STREAMS) - set(HYPRE_USING_CUDA_STREAMS ON CACHE BOOL "" FORCE) - endif (HYPRE_ENABLE_CUDA_STREAMS) - - if (HYPRE_ENABLE_DEVICE_POOL) - set(HYPRE_USING_DEVICE_POOL ON CACHE BOOL "" FORCE) - endif (HYPRE_ENABLE_DEVICE_POOL) - - # TODO Eventually should require cmake>=3.17 - # and use cmake's FindCUDAToolkit. Now collect - # CUDA optional libraries. - include(HYPRE_SetupCUDAToolkit) - else () - message(WARNING "No CUDA support!") - set(HYPRE_USING_HOST_MEMORY ON CACHE BOOL "" FORCE) - endif (CMAKE_CUDA_COMPILER) -endif (HYPRE_WITH_CUDA) - -# SYCL -if (HYPRE_WITH_SYCL) - enable_language(CXX) - message(STATUS "Enabled support for CXX.") - - # Enforce C++17 - if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 17) - set(CMAKE_CXX_STANDARD 17) - endif () - set(CMAKE_CXX_STANDARD_REQUIRED ON) - - message(STATUS "Using CXX standard: c++${CMAKE_CXX_STANDARD}") - - # Set CXX compiler to dpcpp - # WM: note that dpcpp is deprecated, but for now oneMKL looks for the dpcpp compiler to configure things appropriately for the sycl backend - set(CMAKE_CXX_COMPILER "dpcpp") - - # Set linker to dpcpp - set(CMAKE_LINKER "dpcpp") - set(CMAKE_CXX_LINK_EXECUTABLE " -o ") - set(CMAKE_CXX_LINKER_WRAPPER_FLAG " ") - set(CMAKE_CXX_LINKER_WRAPPER_FLAG_SEP " ") +# Set MPI compile flags +if (HYPRE_ENABLE_MPI) + configure_mpi_target() +endif () - # Add any extra CXX compiler flags HYPRE_WITH_EXTRA_CXXFLAGS - if (NOT HYPRE_WITH_EXTRA_CXXFLAGS STREQUAL "") - string(REPLACE " " ";" HYPRE_WITH_EXTRA_CXXFLAGS "${HYPRE_WITH_EXTRA_CXXFLAGS}") - add_compile_options("$<$:${HYPRE_WITH_EXTRA_CXXFLAGS}>") - endif () +# Set OpenMP compile flags +if (HYPRE_ENABLE_OPENMP) + find_package(OpenMP REQUIRED) + target_link_libraries(${PROJECT_NAME} PUBLIC OpenMP::OpenMP_C) +endif () - set(HYPRE_USING_SYCL ON CACHE BOOL "" FORCE) - set(HYPRE_USING_GPU ON CACHE BOOL "" FORCE) +# Add LTO-related flags to hypre if requested +if (HYPRE_ENABLE_LTO) + # Additional checks before enabling LTO + include(CheckIPOSupported) + check_ipo_supported(RESULT LTO_SUPPORTED OUTPUT LTO_ERROR) - if (HYPRE_ENABLE_UNIFIED_MEMORY) - set(HYPRE_USING_UNIFIED_MEMORY ON CACHE BOOL "" FORCE) + if (LTO_SUPPORTED) + message(STATUS "Link-Time Optimization (LTO) enabled for Hypre") else () - set(HYPRE_USING_DEVICE_MEMORY ON CACHE BOOL "" FORCE) - endif () - - # Check if examples are enabled, but not unified memory - if (HYPRE_BUILD_EXAMPLES AND NOT HYPRE_ENABLE_UNIFIED_MEMORY) - message(WARNING "Running the examples on GPUs requires Unified Memory! - Examples will not be built!") - set(HYPRE_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + message(WARNING "Link-Time Optimization (LTO) requested but not supported: ${LTO_ERROR}. Turning it off...") + set(HYPRE_ENABLE_LTO OFF) endif () - add_compile_options("$<$:-fsycl>") - add_compile_options("$<$:-fsycl-unnamed-lambda>") - add_link_options("-fsycl") - add_link_options("-fsycl-device-code-split=per_kernel") - add_link_options("-Wl,--no-relax") - - if (HYPRE_SYCL_TARGET) - add_link_options("SHELL:-fsycl-targets=${HYPRE_SYCL_TARGET}") - endif () - if (HYPRE_SYCL_TARGET_BACKEND) - add_link_options("SHELL:-Xsycl-target-backend '${HYPRE_SYCL_TARGET_BACKEND}'") - endif () - - set(HYPRE_USING_HOST_MEMORY OFF CACHE BOOL "" FORCE) - - if (HYPRE_ENABLE_CUDA_STREAMS) - set(HYPRE_USING_CUDA_STREAMS ON CACHE BOOL "" FORCE) - endif (HYPRE_ENABLE_CUDA_STREAMS) - -endif (HYPRE_WITH_SYCL) - -# Add any extra C compiler flags HYPRE_WITH_EXTRA_CFLAGS -if (NOT HYPRE_WITH_EXTRA_CFLAGS STREQUAL "") - string(REPLACE " " ";" HYPRE_WITH_EXTRA_CFLAGS "${HYPRE_WITH_EXTRA_CFLAGS}") - add_compile_options("$<$:${HYPRE_WITH_EXTRA_CFLAGS}>") + set_target_properties(${PROJECT_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) +else () + set_target_properties(${PROJECT_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION FALSE) + message(STATUS "Link-Time Optimization (LTO) disabled") endif () -# Set library build type (must appear before add_library calls) -if (HYPRE_SHARED) - set(BUILD_SHARED_LIBS ON CACHE INTERNAL "" FORCE) +# Setup GPU build options: CUDA, HIP, SYCL are mutually exclusive +if (HYPRE_ENABLE_CUDA OR HYPRE_ENABLE_HIP OR HYPRE_ENABLE_SYCL) + include(HYPRE_SetupGPUToolkit) else () - set(BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE) + message(STATUS "GPU support not enabled") + set(HYPRE_USING_HOST_MEMORY ON CACHE INTERNAL "") endif () -# Create the HYPRE library object -add_library(${PROJECT_NAME}) +# Add any extra C compiler flags HYPRE_WITH_EXTRA_CFLAGS +if (NOT HYPRE_WITH_EXTRA_CFLAGS STREQUAL "") + string(REPLACE " " ";" HYPRE_WITH_EXTRA_CFLAGS_LIST ${HYPRE_WITH_EXTRA_CFLAGS}) + target_compile_options(${PROJECT_NAME} PRIVATE + $<$:${HYPRE_WITH_EXTRA_CFLAGS_LIST}>) +endif () -# Headers and sources -set(HYPRE_HEADERS "") +# Set output directory for the library +set_target_properties(${PROJECT_NAME} PROPERTIES + VERSION "${HYPRE_VERSION}" + SOVERSION "${HYPRE_VERSION_MAJOR}${HYPRE_VERSION_MINOR}" + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin" +) -# Headers and sources: . +# Set headers set(HYPRE_MAIN_HEADERS ${CMAKE_CURRENT_BINARY_DIR}/HYPRE_config.h HYPREf.h HYPRE.h - ) - -set(HYPRE_HEADERS ${HYPRE_HEADERS} ${HYPRE_MAIN_HEADERS}) - -# Headers and sources: blas -if (HYPRE_USING_HYPRE_BLAS) - add_subdirectory(blas) -else () - # Use TPL_BLAS_LIBRARIES if set. - if (TPL_BLAS_LIBRARIES) - message(STATUS "Using TPL_BLAS_LIBRARIES='${TPL_BLAS_LIBRARIES}'") - target_link_libraries(${PROJECT_NAME} PUBLIC "${TPL_BLAS_LIBRARIES}") - else () - # Find system blas - find_package(BLAS REQUIRED) - target_link_libraries(${PROJECT_NAME} PUBLIC "${BLAS_LIBRARIES}") - endif () - target_compile_definitions(${PROJECT_NAME} PUBLIC "USE_VENDOR_BLAS") -endif () - -# Headers and sources: lapack -if (HYPRE_USING_HYPRE_LAPACK) - add_subdirectory(lapack) -else () - # Use TPL_LAPACK_LIBRARIES if set. - if (TPL_LAPACK_LIBRARIES) - message(STATUS "Using TPL_LAPACK_LIBRARIES='${TPL_LAPACK_LIBRARIES}'") - target_link_libraries(${PROJECT_NAME} PUBLIC "${TPL_LAPACK_LIBRARIES}") - else () - # Find system lapack - find_package(LAPACK REQUIRED) - target_link_libraries(${PROJECT_NAME} PUBLIC "${LAPACK_LIBRARIES}") - endif () -endif () - -# Find SUPERLU, if requested -if (HYPRE_USING_SUPERLU) - if (NOT TPL_SUPERLU_LIBRARIES) - message(FATAL_ERROR "TPL_SUPERLU_LIBRARIES option should be set for SuperLU support.") - endif () - - if (NOT TPL_SUPERLU_INCLUDE_DIRS) - message(FATAL_ERROR "TPL_SUPERLU_INCLUDE_DIRS option be set for SuperLU support.") - endif () - - foreach (dir ${TPL_SUPERLU_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "SuperLU include directory not found: ${dir}") - endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") - endforeach () - message(STATUS "Enabled support for using SUPERLU.") - set(SUPERLU_FOUND TRUE) - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_SUPERLU_LIBRARIES} stdc++) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_SUPERLU_INCLUDE_DIRS}) -endif (HYPRE_USING_SUPERLU) - -if (SUPERLU_FOUND) - set(HYPRE_USING_SUPERLU TRUE) -endif () - -# Find DSUPERLU, if requested -if (HYPRE_USING_DSUPERLU) - if (NOT TPL_DSUPERLU_LIBRARIES) - message(FATAL_ERROR "TPL_DSUPERLU_LIBRARIES option should be set for SuperLU_Dist support.") - endif () - - if (NOT TPL_DSUPERLU_INCLUDE_DIRS) - message(FATAL_ERROR "TPL_DSUPERLU_INCLUDE_DIRS option be set for SuperLU_Dist support.") - endif () - - foreach (dir ${TPL_DSUPERLU_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "SuperLU_Dist include directory not found: ${dir}") - endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") - endforeach () - message(STATUS "Enabled support for using DSUPERLU.") - set(DSUPERLU_FOUND TRUE) - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_DSUPERLU_LIBRARIES} stdc++) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_DSUPERLU_INCLUDE_DIRS}) -endif (HYPRE_USING_DSUPERLU) - -if (DSUPERLU_FOUND) - set(HYPRE_USING_DSUPERLU TRUE) -endif () - -# TODO: Check for case if MAGMA without CUDA/HIP -# Find MAGMA, if requested -if (HYPRE_USING_MAGMA) - if (NOT TPL_MAGMA_LIBRARIES) - message(FATAL_ERROR "TPL_MAGMA_LIBRARIES option should be set for MAGMA support.") - endif () - - if (NOT TPL_MAGMA_INCLUDE_DIRS) - message(FATAL_ERROR "TPL_MAGMA_INCLUDE_DIRS option be set for MAGMA support.") - endif () - - foreach (dir ${TPL_MAGMA_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "MAGMA include directory not found: ${dir}") - endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") - endforeach () - message(STATUS "Enabled support for using MAGMA.") - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_MAGMA_LIBRARIES} stdc++) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_MAGMA_INCLUDE_DIRS}) -endif (HYPRE_USING_MAGMA) - -# Find FEI, if requested -if (HYPRE_USING_FEI) +) +list(APPEND HYPRE_HEADERS ${HYPRE_MAIN_HEADERS}) + +# TPL variables +set(HYPRE_DEPENDENCY_DIRS "" CACHE INTERNAL "List of absolute paths to third-party libraries (TPLs) directories") +set(HYPRE_NEEDS_CXX OFF CACHE INTERNAL "Flag to indicate if C++ is needed for TPLs") + +# Setup third-party libraries (TPLs) +setup_fei() +setup_tpl_or_internal(blas) +setup_tpl_or_internal(lapack) +setup_tpl(superlu) +setup_tpl(dsuperlu) +setup_tpl(magma) +setup_tpl(caliper) +setup_tpl(umpire) + +# Some TPLs need C++ to be enabled (GPU already enables C++ by default) +if(HYPRE_NEEDS_CXX AND NOT HYPRE_USING_GPU) enable_language(CXX) - - if (NOT TPL_FEI_INCLUDE_DIRS) - message(FATAL_ERROR "TPL_FEI_INCLUDE_DIRS option should be set for FEI support.") - endif () - - foreach (dir ${TPL_FEI_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "FEI include directory not found: ${dir}") - endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") - set(CMAKE_CXX_FLAGS "-I${dir} ${CMAKE_CXX_FLAGS}") - endforeach () - message(STATUS "Enabled support for using FEI.") - set(FEI_FOUND TRUE) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_FEI_INCLUDE_DIRS}) -endif(HYPRE_USING_FEI) - -if (HYPRE_USING_CUDA) - target_link_libraries(${PROJECT_NAME} PUBLIC "${EXPORT_INTERFACE_CUDA_LIBS}") - if (HYPRE_HAVE_MPI) - target_include_directories(${PROJECT_NAME} PUBLIC - ${MPI_CXX_INCLUDE_DIRS}) - endif () -endif () - -if (HYPRE_USING_SYCL) - target_include_directories(${PROJECT_NAME} PUBLIC $ENV{DPLROOT}/include) - if (HYPRE_ENABLE_ONEMKLSPARSE) - set(HYPRE_USING_ONEMKLSPARSE ON CACHE BOOL "" FORCE) - endif() - if (HYPRE_ENABLE_ONEMKLBLAS) - set(HYPRE_USING_ONEMKLBLAS ON CACHE BOOL "" FORCE) - endif() - if (HYPRE_ENABLE_ONEMKLRAND) - set(HYPRE_USING_ONEMKLRAND ON CACHE BOOL "" FORCE) - endif() - if (HYPRE_USING_ONEMKLSPARSE OR HYPRE_USING_ONEMKLBLAS OR HYPRE_USING_ONEMKLRAND) - set(MKL_LINK static) - set(MKL_THREADING sequential) - find_package(MKL CONFIG REQUIRED HINTS "$ENV{MKLROOT}/lib/cmake/mkl") - target_compile_options(${PROJECT_NAME} PUBLIC $) - target_include_directories(${PROJECT_NAME} PUBLIC $) - target_link_libraries(${PROJECT_NAME} PUBLIC $) - endif() endif() -if (HYPRE_USING_CALIPER) - if (NOT TPL_CALIPER_LIBRARIES OR NOT TPL_CALIPER_INCLUDE_DIRS) - message(FATAL_ERROR "Both TPL_CALIPER_LIBRARIES and TPL_CALIPER_INCLUDE_DIRS options must be set for Caliper support.") - endif () - - foreach (dir ${TPL_CALIPER_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "Caliper include directory not found: ${dir}") - endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") - endforeach () - message(STATUS "Enabled support for using Caliper.") - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_CALIPER_LIBRARIES}) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_CALIPER_INCLUDE_DIRS}) - -endif() - -# Find Umpire, if requested -if (HYPRE_USING_UMPIRE) - if (NOT TPL_UMPIRE_LIBRARIES OR NOT TPL_UMPIRE_INCLUDE_DIRS) - message(FATAL_ERROR "Both TPL_UMPIRE_LIBRARIES and TPL_UMPIRE_INCLUDE_DIRS options must be set for Umpire support.") - endif () - - foreach (dir ${TPL_UMPIRE_INCLUDE_DIRS}) - if (NOT EXISTS ${dir}) - message(FATAL_ERROR "Umpire include directory not found: ${dir}") - endif () - set(CMAKE_C_FLAGS "-I${dir} ${CMAKE_C_FLAGS}") - endforeach () - message(STATUS "Enabled support for using Umpire.") - target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_UMPIRE_LIBRARIES}) - target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_UMPIRE_INCLUDE_DIRS}) -endif () - -# Set MPI compile flags -if (NOT HYPRE_SEQUENTIAL) - find_program(MPIEXEC_EXECUTABLE NAMES mpiexec mpirun) - find_package(MPI REQUIRED) - target_link_libraries(${PROJECT_NAME} PUBLIC MPI::MPI_C) - set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} MPI::MPI_C) - - # Check if MPI supports the MPI_Comm_f2c function - include(CheckCSourceCompiles) - check_c_source_compiles(" - #include - int main() { - MPI_Comm c = MPI_Comm_f2c(0); - return 0; - } - " HYPRE_HAVE_MPI_COMM_F2C) -endif (NOT HYPRE_SEQUENTIAL) - -# Set OpenMP compile flags -if (HYPRE_USING_OPENMP) - find_package(OpenMP REQUIRED) - target_link_libraries(${PROJECT_NAME} PUBLIC OpenMP::OpenMP_C) -endif (HYPRE_USING_OPENMP) +# Print the directories used to hint the user about the active TPLs +message(STATUS "Dependency directories: ${HYPRE_DEPENDENCY_DIRS}") # Configure a header file to pass CMake settings to the source code configure_file( @@ -670,23 +456,25 @@ foreach (DIR IN LISTS HYPRE_DIRS) $) endforeach () -# BINARY must be first in order to get the correct HYPRE_config.h file -target_include_directories(${PROJECT_NAME} PUBLIC - $ - $ - $ - $ - $ - ) - -if (HYPRE_USING_CUDA) +# Set the language for GPU sources (has to be done after the subdirectories are added) +if (HYPRE_ENABLE_CUDA) set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE CUDA) -endif () - -if (HYPRE_USING_SYCL) +elseif (HYPRE_ENABLE_HIP) + set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE HIP) +elseif (HYPRE_ENABLE_SYCL) set_source_files_properties(${HYPRE_GPU_SOURCES} PROPERTIES LANGUAGE CXX) -endif () +endif() +# Include directories +target_include_directories(${PROJECT_NAME} PUBLIC + $ + $ + $ + $ + $ +) + +# Set file properties for MSVC if (MSVC) target_compile_definitions(${PROJECT_NAME} PRIVATE _CRT_SECURE_NO_WARNINGS) if (MSVC_VERSION LESS 1928) # Visual Studio 2019 version 16.8 claims full C11 support @@ -714,6 +502,7 @@ if (MSVC) endif() endif () +# Build FEI, if requested (to be phased out) if (HYPRE_USING_FEI) add_subdirectory(FEI_mv) endif () @@ -728,46 +517,63 @@ if (HYPRE_BUILD_TESTS) add_subdirectory(test) endif () +# Add targets for tags, distclean, and uninstall +add_hypre_target_tags() +add_hypre_target_distclean() +add_hypre_target_uninstall() + +# Installation include(GNUInstallDirs) install(TARGETS ${PROJECT_NAME} EXPORT HYPRETargets - LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" - ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" - RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" - INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) install(FILES ${HYPRE_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") +# Export the package include(CMakePackageConfigHelpers) write_basic_package_version_file( - HYPREConfigVersion.cmake + "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfigVersion.cmake" VERSION ${PACKAGE_VERSION} COMPATIBILITY SameMajorVersion - ) - +) +configure_package_config_file( + "${CMAKE_CURRENT_SOURCE_DIR}/config/HYPREConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" + INSTALL_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" +) +install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfigVersion.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE" +) +# Install the CMake export file install(EXPORT HYPRETargets FILE HYPRETargets.cmake NAMESPACE HYPRE:: DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE" - ) - -configure_package_config_file( - config/HYPREConfig.cmake.in HYPREConfig.cmake - INSTALL_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" - ) -install( - FILES - "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfig.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/HYPREConfigVersion.cmake" - DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/HYPRE" - ) +) +# Export from build tree export(EXPORT HYPRETargets FILE "${CMAKE_CURRENT_BINARY_DIR}/HYPRETargets.cmake" NAMESPACE HYPRE:: - ) +) # Declare an alias so that consumers can depend on HYPRE::HYPRE target # also when using HYPRE via add_directory or FetchContent add_library(HYPRE::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) +# Ensure the C standard is set on the target +set_target_properties(${PROJECT_NAME} PROPERTIES + C_STANDARD ${CMAKE_C_STANDARD} + C_STANDARD_REQUIRED ON) + +# Export the package export(PACKAGE ${PROJECT_NAME}) + +# Print status of the build options +print_option_status(${BASE_OPTIONS} ${GPU_OPTIONS} ${TPL_OPTIONS}) diff --git a/src/IJ_mv/CMakeLists.txt b/src/IJ_mv/CMakeLists.txt index 6291d5186d..7449654aec 100644 --- a/src/IJ_mv/CMakeLists.txt +++ b/src/IJ_mv/CMakeLists.txt @@ -34,7 +34,7 @@ target_sources(${PROJECT_NAME} ${HDRS} ) -if (HYPRE_USING_CUDA OR HYPRE_USING_SYCL) +if (HYPRE_USING_GPU) set(GPU_SRCS IJMatrix_parcsr_device.c IJVector_parcsr_device.c diff --git a/src/IJ_mv/IJ_matrix.h b/src/IJ_mv/IJ_matrix.h index c258992f46..c2de468c5c 100644 --- a/src/IJ_mv/IJ_matrix.h +++ b/src/IJ_mv/IJ_matrix.h @@ -64,7 +64,7 @@ typedef struct hypre_IJMatrix_struct #define hypre_IJMatrixOMPFlag(matrix) ((matrix) -> omp_flag) #define hypre_IJMatrixPrintLevel(matrix) ((matrix) -> print_level) -static inline HYPRE_MemoryLocation +static inline HYPRE_MAYBE_UNUSED_FUNC HYPRE_MemoryLocation hypre_IJMatrixMemoryLocation(hypre_IJMatrix *matrix) { if ( hypre_IJMatrixObject(matrix) && hypre_IJMatrixObjectType(matrix) == HYPRE_PARCSR) diff --git a/src/IJ_mv/IJ_vector.h b/src/IJ_mv/IJ_vector.h index ebcc114576..760d9eda7c 100644 --- a/src/IJ_mv/IJ_vector.h +++ b/src/IJ_mv/IJ_vector.h @@ -48,7 +48,7 @@ typedef struct hypre_IJVector_struct #define hypre_IJVectorGlobalNumRows(vector) ((vector) -> global_num_rows) #define hypre_IJVectorPrintLevel(vector) ((vector) -> print_level) -static inline HYPRE_MemoryLocation +static inline HYPRE_MAYBE_UNUSED_FUNC HYPRE_MemoryLocation hypre_IJVectorMemoryLocation(hypre_IJVector *vector) { if ( hypre_IJVectorObject(vector) && hypre_IJVectorObjectType(vector) == HYPRE_PARCSR) diff --git a/src/IJ_mv/_hypre_IJ_mv.h b/src/IJ_mv/_hypre_IJ_mv.h index a90dd340f2..29991146b8 100644 --- a/src/IJ_mv/_hypre_IJ_mv.h +++ b/src/IJ_mv/_hypre_IJ_mv.h @@ -273,7 +273,7 @@ typedef struct hypre_IJMatrix_struct #define hypre_IJMatrixOMPFlag(matrix) ((matrix) -> omp_flag) #define hypre_IJMatrixPrintLevel(matrix) ((matrix) -> print_level) -static inline HYPRE_MemoryLocation +static inline HYPRE_MAYBE_UNUSED_FUNC HYPRE_MemoryLocation hypre_IJMatrixMemoryLocation(hypre_IJMatrix *matrix) { if ( hypre_IJMatrixObject(matrix) && hypre_IJMatrixObjectType(matrix) == HYPRE_PARCSR) @@ -351,7 +351,7 @@ typedef struct hypre_IJVector_struct #define hypre_IJVectorGlobalNumRows(vector) ((vector) -> global_num_rows) #define hypre_IJVectorPrintLevel(vector) ((vector) -> print_level) -static inline HYPRE_MemoryLocation +static inline HYPRE_MAYBE_UNUSED_FUNC HYPRE_MemoryLocation hypre_IJVectorMemoryLocation(hypre_IJVector *vector) { if ( hypre_IJVectorObject(vector) && hypre_IJVectorObjectType(vector) == HYPRE_PARCSR) diff --git a/src/cmbuild/README.txt b/src/cmbuild/README.txt deleted file mode 100644 index cd487267e4..0000000000 --- a/src/cmbuild/README.txt +++ /dev/null @@ -1 +0,0 @@ -This directory is provided for convenience to do CMake builds... diff --git a/src/config/HYPREConfig.cmake.in b/src/config/HYPREConfig.cmake.in index 717ba85c10..a27a5024d2 100644 --- a/src/config/HYPREConfig.cmake.in +++ b/src/config/HYPREConfig.cmake.in @@ -7,7 +7,19 @@ include(CMakeFindDependencyMacro) -set(HYPRE_ENABLE_SHARED @HYPRE_ENABLE_SHARED@) +function(find_hypre_dependency name) + string(TOUPPER ${name} UPPER_NAME) + if(HYPRE_ENABLE_${UPPER_NAME}) + find_dependency(${name} REQUIRED HINTS ${HYPRE_DEPENDENCY_DIRS}) + if(${name}_FOUND) + message(STATUS "Found ${UPPER_NAME}: ${${name}_DIR}") + else() + message(FATAL_ERROR "${UPPER_NAME} not found.") + endif() + endif() +endfunction() + +set(HYPRE_DEPENDENCY_DIRS "@HYPRE_DEPENDENCY_DIRS@") set(HYPRE_ENABLE_BIGINT @HYPRE_ENABLE_BIGINT@) set(HYPRE_ENABLE_MIXEDINT @HYPRE_ENABLE_MIXEDINT@) set(HYPRE_ENABLE_SINGLE @HYPRE_ENABLE_SINGLE@) @@ -17,38 +29,50 @@ set(HYPRE_ENABLE_HYPRE_BLAS @HYPRE_ENABLE_HYPRE_BLAS@) set(HYPRE_ENABLE_HYPRE_LAPACK @HYPRE_ENABLE_HYPRE_LAPACK@) set(HYPRE_ENABLE_PERSISTENT_COMM @HYPRE_ENABLE_PERSISTENT_COMM@) set(HYPRE_ENABLE_FEI @HYPRE_ENABLE_FEI@) -set(HYPRE_WITH_MPI @HYPRE_WITH_MPI@) -set(HYPRE_WITH_OPENMP @HYPRE_WITH_OPENMP@) -set(HYPRE_WITH_HOPSCOTCH @HYPRE_WITH_HOPSCOTCH@) -set(HYPRE_USING_DSUPERLU @HYPRE_USING_DSUPERLU@) -set(HYPRE_USING_MAGMA @HYPRE_USING_MAGMA@) -set(HYPRE_WITH_CALIPER @HYPRE_WITH_CALIPER@) -set(HYPRE_PRINT_ERRORS @HYPRE_PRINT_ERRORS@) -set(HYPRE_TIMING @HYPRE_TIMING@) -set(HYPRE_BUILD_EXAMPLES @HYPRE_BUILD_EXAMPLES@) -set(HYPRE_BUILD_TESTS @HYPRE_BUILD_TESTS@) -set(HYPRE_USING_HOST_MEMORY @HYPRE_USING_HOST_MEMORY@) -set(HYPRE_WITH_CUDA @HYPRE_WITH_CUDA@) -set(HYPRE_WITH_SYCL @HYPRE_WITH_SYCL@) +set(HYPRE_ENABLE_MPI @HYPRE_ENABLE_MPI@) +set(HYPRE_ENABLE_OPENMP @HYPRE_ENABLE_OPENMP@) +set(HYPRE_ENABLE_HOPSCOTCH @HYPRE_ENABLE_HOPSCOTCH@) +set(HYPRE_ENABLE_PRINT_ERRORS @HYPRE_ENABLE_PRINT_ERRORS@) +set(HYPRE_ENABLE_TIMING @HYPRE_ENABLE_TIMING@) +set(HYPRE_ENABLE_HOST_MEMORY @HYPRE_ENABLE_HOST_MEMORY@) +set(HYPRE_ENABLE_TEST_USING_HOST @HYPRE_ENABLE_TEST_USING_HOST@) +set(HYPRE_ENABLE_CUDA @HYPRE_ENABLE_CUDA@) +set(HYPRE_ENABLE_HIP @HYPRE_ENABLE_HIP@) +set(HYPRE_ENABLE_SYCL @HYPRE_ENABLE_SYCL@) set(HYPRE_ENABLE_UNIFIED_MEMORY @HYPRE_ENABLE_UNIFIED_MEMORY@) set(HYPRE_ENABLE_CUDA_STREAMS @HYPRE_ENABLE_CUDA_STREAMS@) set(HYPRE_ENABLE_CUSPARSE @HYPRE_ENABLE_CUSPARSE@) set(HYPRE_ENABLE_DEVICE_POOL @HYPRE_ENABLE_DEVICE_POOL@) +set(HYPRE_ENABLE_DEVICE_MALLOC_ASYNC @HYPRE_ENABLE_DEVICE_MALLOC_ASYNC@) +set(HYPRE_ENABLE_THRUST_ASYNC @HYPRE_ENABLE_THRUST_ASYNC@) set(HYPRE_ENABLE_CUBLAS @HYPRE_ENABLE_CUBLAS@) set(HYPRE_ENABLE_CURAND @HYPRE_ENABLE_CURAND@) +set(HYPRE_ENABLE_GPU_AWARE_MPI @HYPRE_ENABLE_GPU_AWARE_MPI@) set(HYPRE_ENABLE_GPU_PROFILING @HYPRE_ENABLE_GPU_PROFILING@) +set(HYPRE_ENABLE_ROCBLAS @HYPRE_ENABLE_ROCBLAS@) +set(HYPRE_ENABLE_ROCRAND @HYPRE_ENABLE_ROCRAND@) +set(HYPRE_ENABLE_ROCSOLVER @HYPRE_ENABLE_ROCSOLVER@) set(HYPRE_ENABLE_ONEMKLSPARSE @HYPRE_ENABLE_ONEMKLSPARSE@) set(HYPRE_ENABLE_ONEMKLBLAS @HYPRE_ENABLE_ONEMKLBLAS@) set(HYPRE_ENABLE_ONEMKLRAND @HYPRE_ENABLE_ONEMKLRAND@) -set(HYPRE_WITH_UMPIRE @HYPRE_WITH_UMPIRE@) -set(HYPRE_WITH_UMPIRE_HOST @HYPRE_WITH_UMPIRE_HOST@) -set(HYPRE_WITH_UMPIRE_DEVICE @HYPRE_WITH_UMPIRE_DEVICE@) -set(HYPRE_WITH_UMPIRE_UM @HYPRE_WITH_UMPIRE_UM@) -set(HYPRE_WITH_UMPIRE_PINNED @HYPRE_WITH_UMPIRE_PINNED@) +set(HYPRE_BUILD_EXAMPLES @HYPRE_BUILD_EXAMPLES@) +set(HYPRE_BUILD_TESTS @HYPRE_BUILD_TESTS@) +set(HYPRE_ENABLE_DSUPERLU @HYPRE_ENABLE_DSUPERLU@) +set(HYPRE_ENABLE_MAGMA @HYPRE_ENABLE_MAGMA@) +set(HYPRE_ENABLE_CALIPER @HYPRE_ENABLE_CALIPER@) +set(HYPRE_ENABLE_UMPIRE @HYPRE_ENABLE_UMPIRE@) +set(HYPRE_ENABLE_UMPIRE_HOST @HYPRE_ENABLE_UMPIRE_HOST@) +set(HYPRE_ENABLE_UMPIRE_DEVICE @HYPRE_ENABLE_UMPIRE_DEVICE@) +set(HYPRE_ENABLE_UMPIRE_UM @HYPRE_ENABLE_UMPIRE_UM@) +set(HYPRE_ENABLE_UMPIRE_PINNED @HYPRE_ENABLE_UMPIRE_PINNED@) set(HYPRE_HAVE_MPI_COMM_F2C @HYPRE_HAVE_MPI_COMM_F2C@) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") +if (UNIX) + list(APPEND TPL_LIBRARIES m) +endif() + if(NOT HYPRE_ENABLE_HYPRE_BLAS) find_dependency(BLAS) endif() @@ -57,27 +81,99 @@ if(NOT HYPRE_ENABLE_HYPRE_LAPACK) find_dependency(LAPACK) endif() -if(HYPRE_USING_DSUPERLU) - list(APPEND TPL_LIBRARIES @TPL_DSUPERLU_LIBRARIES@ stdc++) -endif() - -if(HYPRE_USING_MAGMA) - list(APPEND TPL_LIBRARIES @TPL_MAGMA_LIBRARIES@ stdc++) -endif() - -if(HYPRE_WITH_MPI) +if(HYPRE_ENABLE_MPI) enable_language(C) - find_dependency(MPI @MPI_C_VERSION@ EXACT COMPONENTS C) + find_dependency(MPI @MPI_C_VERSION@ COMPONENTS C) endif() -if(HYPRE_WITH_OPENMP) +if(HYPRE_ENABLE_OPENMP) find_dependency(OpenMP) endif() -if(HYPRE_WITH_CUDA) - find_dependency(CUDA REQUIRED) - if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.17) - find_dependency(CUDAToolkit REQUIRED) +find_hypre_dependency(caliper) +find_hypre_dependency(dsuperlu) +find_hypre_dependency(magma) +find_hypre_dependency(umpire) + +if(HYPRE_ENABLE_CUDA OR HYPRE_ENABLE_HIP OR HYPRE_ENABLE_SYCL) + set(REQUIRED_GPU_COMPONENTS) + + if(HYPRE_ENABLE_CUDA) + if(HYPRE_ENABLE_CUSPARSE) + list(APPEND REQUIRED_GPU_COMPONENTS cusparse) + endif() + + if(HYPRE_ENABLE_CUBLAS) + list(APPEND REQUIRED_GPU_COMPONENTS cublas) + endif() + + if(HYPRE_ENABLE_CURAND) + list(APPEND REQUIRED_GPU_COMPONENTS curand) + endif() + elseif(HYPRE_ENABLE_HIP) + if(HYPRE_ENABLE_ROCSPARSE) + list(APPEND REQUIRED_GPU_COMPONENTS rocsparse) + endif() + + if(HYPRE_ENABLE_ROCBLAS) + list(APPEND REQUIRED_GPU_COMPONENTS rocblas) + endif() + + if(HYPRE_ENABLE_ROCRAND) + list(APPEND REQUIRED_GPU_COMPONENTS rocrand) + endif() + elseif(HYPRE_ENABLE_SYCL) + if(HYPRE_ENABLE_ONEMKLSPARSE) + list(APPEND REQUIRED_GPU_COMPONENTS onemklsparse) + endif() + + if(HYPRE_ENABLE_ONEMKLBLAS) + list(APPEND REQUIRED_GPU_COMPONENTS onemklblas) + endif() + + if(HYPRE_ENABLE_ONEMKLRAND) + list(APPEND REQUIRED_GPU_COMPONENTS onemklrand) + endif() + endif() + + if(REQUIRED_GPU_COMPONENTS) + if(HYPRE_ENABLE_CUDA) + message(STATUS "Finding CUDA Toolkit components: ${REQUIRED_GPU_COMPONENTS}") + find_dependency(CUDAToolkit REQUIRED COMPONENTS ${REQUIRED_GPU_COMPONENTS}) + set(GPU_TOOLKIT_FOUND ${CUDAToolkit_FOUND}) + set(GPU_TOOLKIT_VERSION ${CUDAToolkit_VERSION}) + elseif(HYPRE_ENABLE_HIP) + message(STATUS "Finding ROCm Toolkit components: ${REQUIRED_GPU_COMPONENTS}") + find_dependency(rocm REQUIRED COMPONENTS ${REQUIRED_GPU_COMPONENTS}) + set(GPU_TOOLKIT_FOUND ${rocm_FOUND}) + set(GPU_TOOLKIT_VERSION ${rocm_VERSION}) + elseif(HYPRE_ENABLE_SYCL) + message(STATUS "Finding Intel DPC++ Toolkit components: ${REQUIRED_GPU_COMPONENTS}") + find_dependency(IntelDPCPP REQUIRED) + set(GPU_TOOLKIT_FOUND ${IntelDPCPP_FOUND}) + set(GPU_TOOLKIT_VERSION ${IntelDPCPP_VERSION}) + endif() + + if(GPU_TOOLKIT_FOUND) + message(STATUS "GPU Toolkit found: ${GPU_TOOLKIT_VERSION}") + set(MISSING_COMPONENTS) + foreach(component ${REQUIRED_GPU_COMPONENTS}) + if(HYPRE_ENABLE_CUDA AND NOT TARGET CUDA::${component}) + list(APPEND MISSING_COMPONENTS ${component}) + elseif(HYPRE_ENABLE_HIP AND NOT TARGET roc::${component}) + list(APPEND MISSING_COMPONENTS ${component}) + elseif(HYPRE_ENABLE_SYCL AND NOT TARGET sycl::${component}) + list(APPEND MISSING_COMPONENTS ${component}) + endif() + endforeach() + if(MISSING_COMPONENTS) + message(FATAL_ERROR "Required GPU components not found: ${MISSING_COMPONENTS}") + else() + message(STATUS "Required GPU components found: ${REQUIRED_GPU_COMPONENTS}") + endif() + else() + message(FATAL_ERROR "GPU Toolkit not found.") + endif() endif() endif() diff --git a/src/config/HYPRE_config.h.cmake.in b/src/config/HYPRE_config.h.cmake.in index 9f4278dac0..6aa15301d5 100644 --- a/src/config/HYPRE_config.h.cmake.in +++ b/src/config/HYPRE_config.h.cmake.in @@ -35,8 +35,11 @@ /* Use cudaMallocAsync */ #cmakedefine HYPRE_USING_DEVICE_MALLOC_ASYNC 1 +/* Use Thrust par_nosync policy */ +#cmakedefine HYPRE_ENABLE_THRUST_ASYNC 1 + /* Use GPU-aware MPI */ -#cmakedefine HYPRE_WITH_GPU_AWARE_MPI 1 +#cmakedefine HYPRE_USING_GPU_AWARE_MPI 1 /* Debug mode */ #cmakedefine HYPRE_DEBUG 1 @@ -48,7 +51,7 @@ #cmakedefine HYPRE_USING_PERSISTENT_COMM 1 /* Use hopscotch hashing */ -#cmakedefine HYPRE_HOPSCOTCH 1 +#cmakedefine HYPRE_USING_HOPSCOTCH 1 /* Compile without MPI */ #cmakedefine HYPRE_SEQUENTIAL 1 @@ -92,6 +95,21 @@ /* Use if executing on device with SYCL */ #cmakedefine HYPRE_USING_SYCL 1 +/* Use rocBLAS */ +#cmakedefine HYPRE_USING_ROCBLAS 1 + +/* Use rocRAND */ +#cmakedefine HYPRE_USING_ROCRAND 1 + +/* Use rocSPARSE */ +#cmakedefine HYPRE_USING_ROCSPARSE 1 + +/* Use rocSOLVER */ +#cmakedefine HYPRE_USING_ROCSOLVER 1 + +/* Use rocTX profiling */ +#cmakedefine HYPRE_USING_ROCTX 1 + /* Use cuBLAS */ #cmakedefine HYPRE_USING_CUBLAS 1 @@ -125,6 +143,9 @@ /* Use if executing on GPU device */ #cmakedefine HYPRE_USING_GPU 1 +/* Use if executing hypre tests on host (CPU) */ +#cmakedefine HYPRE_TEST_USING_HOST 1 + /* Use HIP */ #cmakedefine HYPRE_USING_HIP 1 @@ -149,9 +170,6 @@ /* Use SuperLU */ #cmakedefine HAVE_SUPERLU 1 -/* Use MAGMA */ -#cmakedefine HYPRE_USING_MAGMA 1 - /* Use MPI */ #cmakedefine HYPRE_HAVE_MPI 1 @@ -165,13 +183,13 @@ * 3 = two underscores * 4 = caps, no underscores * 5 = one underscore before and after */ -#define HYPRE_FMANGLE 0 +#define HYPRE_FMANGLE @HYPRE_FMANGLE@ /* Define as in HYPRE_FMANGLE to set the BLAS name mangling scheme */ -#define HYPRE_FMANGLE_BLAS 0 +#define HYPRE_FMANGLE_BLAS @HYPRE_FMANGLE_BLAS@ /* Define as in HYPRE_FMANGLE to set the LAPACK name mangling scheme */ -#define HYPRE_FMANGLE_LAPACK 0 +#define HYPRE_FMANGLE_LAPACK @HYPRE_FMANGLE_LAPACK@ /* Define to a macro mangling the given C identifier (in lower and upper * case), which must not contain underscores, for linking with Fortran. */ diff --git a/src/config/HYPRE_config.h.in b/src/config/HYPRE_config.h.in index 5596be29ff..103bb00d5d 100644 --- a/src/config/HYPRE_config.h.in +++ b/src/config/HYPRE_config.h.in @@ -97,8 +97,8 @@ /* Define to 1 if the routine MPI_Comm_f2c is found */ #undef HYPRE_HAVE_MPI_COMM_F2C -/* Define to 1 if hopscotch hashing */ -#undef HYPRE_HOPSCOTCH +/* Define to 1 if using hopscotch hashing */ +#undef HYPRE_USING_HOPSCOTCH /* Define to 1 for HP platforms */ #undef HYPRE_HPPA @@ -290,7 +290,7 @@ #undef HYPRE_USING_UNIFIED_MEMORY /* Define to 1 if using GPU aware MPI */ -#undef HYPRE_WITH_GPU_AWARE_MPI +#undef HYPRE_USING_GPU_AWARE_MPI /* Define to the address where bug reports for this package should be sent. */ #undef PACKAGE_BUGREPORT diff --git a/src/config/astyle-apply.sh b/src/config/astyle-apply.sh index 813e96e0f5..ef2a559ef0 100755 --- a/src/config/astyle-apply.sh +++ b/src/config/astyle-apply.sh @@ -49,7 +49,6 @@ fi # Generate list of source files to indent find . -type f -print | egrep '[.]*[.](c|cc|cpp|cxx|C|h|hpp|hxx|H)$' | - egrep -v '/cmbuild' | egrep -v '/docs' | egrep -v '/FEI_mv' | egrep -v '/blas' | diff --git a/src/config/cmake/HYPRE_CMakeUtilities.cmake b/src/config/cmake/HYPRE_CMakeUtilities.cmake index a68acd1473..ab20986538 100644 --- a/src/config/cmake/HYPRE_CMakeUtilities.cmake +++ b/src/config/cmake/HYPRE_CMakeUtilities.cmake @@ -3,6 +3,386 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +# Function to set hypre build options +function(set_hypre_option category name description default_value) + option(${name} "${description}" ${default_value}) + if (${category} STREQUAL "CUDA" OR ${category} STREQUAL "HIP" OR ${category} STREQUAL "SYCL") + if (HYPRE_ENABLE_${category} STREQUAL "ON") + set(GPU_OPTIONS ${GPU_OPTIONS} ${name} PARENT_SCOPE) + endif() + else() + set(${category}_OPTIONS ${${category}_OPTIONS} ${name} PARENT_SCOPE) + endif() +endfunction() + +# Function to set internal hypre build options +function(set_internal_hypre_option var_prefix var_name) + if(HYPRE_ENABLE_${var_name}) + if(var_prefix STREQUAL "") + set(HYPRE_${var_name} ON CACHE INTERNAL "") + else() + set(HYPRE_${var_prefix}_${var_name} ON CACHE INTERNAL "") + endif() + endif() +endfunction() + +# Function to setup git version info +function(setup_git_version_info HYPRE_GIT_DIR) + set(GIT_VERSION_FOUND FALSE PARENT_SCOPE) + if (EXISTS "${HYPRE_GIT_DIR}") + execute_process(COMMAND git -C ${HYPRE_GIT_DIR} describe --match v* --long --abbrev=9 + OUTPUT_VARIABLE develop_string + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE git_result) + if (git_result EQUAL 0) + set(GIT_VERSION_FOUND TRUE PARENT_SCOPE) + execute_process(COMMAND git -C ${HYPRE_GIT_DIR} describe --match v* --abbrev=0 + OUTPUT_VARIABLE develop_lastag + OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND git -C ${HYPRE_GIT_DIR} rev-list --count ${develop_lastag}..HEAD + OUTPUT_VARIABLE develop_number + OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND git -C ${HYPRE_GIT_DIR} rev-parse --abbrev-ref HEAD + OUTPUT_VARIABLE develop_branch + OUTPUT_STRIP_TRAILING_WHITESPACE) + set(HYPRE_DEVELOP_STRING ${develop_string} PARENT_SCOPE) + set(HYPRE_DEVELOP_NUMBER ${develop_number} PARENT_SCOPE) + set(HYPRE_BRANCH_NAME ${develop_branch} PARENT_SCOPE) + if (develop_branch MATCHES "master") + set(HYPRE_DEVELOP_BRANCH ${develop_branch} PARENT_SCOPE) + endif () + endif() + endif() +endfunction() + +# Function to check if two options have the same value +function(ensure_options_match option1 option2) + if(DEFINED CACHE{${option1}} AND DEFINED CACHE{${option2}}) + #if ((${option1} AND NOT ${option2}) OR (NOT ${option1} AND ${option2})) + if(NOT (${option1} STREQUAL ${option2}) AND NOT (${option1} STREQUAL "OFF" AND ${option2} STREQUAL "OFF")) + # Save the value of the conflicting options + set(saved_value1 "${${option1}}") + set(saved_value2 "${${option2}}") + + # Unset conflicting options + unset(${option1} CACHE) + unset(${option2} CACHE) + message(FATAL_ERROR "Incompatible options: ${option1} (${saved_value1}) and ${option2} (${saved_value2}) must have the same value. Unsetting both options.") + endif() + endif() +endfunction() + +# Function to check if two options have different values +function(ensure_options_differ option1 option2) + if(DEFINED ${option1} AND DEFINED ${option2}) + if(${option1} AND ${${option2}}) + # Save the value of the conflicting options + set(saved_value1 "${${option1}}") + set(saved_value2 "${${option2}}") + + # Unset conflicting options + unset(${option1} CACHE) + unset(${option2} CACHE) + + message(FATAL_ERROR "Error: ${option1} (${saved_value1}) and ${option2} (${saved_value2}) are mutually exclusive. Only one can be set to ON. Unsetting both options.") + endif() + endif() +endfunction() + +# Helper function to process Fortran mangling scheme +function(process_fmangling_scheme varname description) + set(mangling_map + UNSPECIFIED 0 + NONE 1 + ONE_UNDERSCORE 2 + TWO_UNDERSCORES 3 + CAPS 4 + PRE_POST_UNDERSCORE 5 + ) + list(LENGTH mangling_map map_length) + math(EXPR last_index "${map_length} - 1") + + # Check if varname is a numeric value + if (HYPRE_ENABLE_${varname} MATCHES "^[0-9]+$") + foreach(i RANGE 0 ${last_index} 2) + math(EXPR next_index "${i} + 1") + list(GET mangling_map ${next_index} value) + if (HYPRE_ENABLE_${varname} STREQUAL ${value}) + list(GET mangling_map ${i} key) + message(STATUS "HYPRE_ENABLE_${varname} corresponds to Fortran ${description} mangling scheme: ${key}") + set(HYPRE_${varname} ${value} CACHE INTERNAL "Set the Fortran ${description} mangling scheme") + return() + endif() + endforeach() + endif() + + # Check if varname matches any string key + foreach(i RANGE 0 ${last_index} 2) + list(GET mangling_map ${i} key) + math(EXPR next_index "${i} + 1") + list(GET mangling_map ${next_index} value) + if (HYPRE_ENABLE_${varname} MATCHES ${key}) + if (NOT HYPRE_ENABLE_${varname} MATCHES "UNSPECIFIED") + message(STATUS "Using Fortran ${description} mangling scheme: ${key}") + endif() + set(HYPRE_${varname} ${value} CACHE INTERNAL "Set the Fortran ${description} mangling scheme") + return() + endif() + endforeach() + + # Default case + message(STATUS "Unknown value for HYPRE_ENABLE_${varname}. Defaulting to UNSPECIFIED (0)") + set(HYPRE_${varname} 0 CACHE INTERNAL "Set the Fortran ${description} mangling scheme") +endfunction() + +# Function to configure MPI target +function(configure_mpi_target) + find_package(MPI REQUIRED) + target_link_libraries(${PROJECT_NAME} PUBLIC MPI::MPI_C) + + # Determine the correct MPI include directory + if(MPI_CXX_INCLUDE_DIR) + set(MPI_INCLUDE_DIR ${MPI_CXX_INCLUDE_DIR}) + elseif(MPI_CXX_INCLUDE_PATH) + set(MPI_INCLUDE_DIR ${MPI_CXX_INCLUDE_PATH}) + elseif(MPI_CXX_COMPILER_INCLUDE_DIR) + set(MPI_INCLUDE_DIR ${MPI_CXX_COMPILER_INCLUDE_DIR}) + elseif(MPI_C_COMPILER_INCLUDE_DIR) + set(MPI_INCLUDE_DIR ${MPI_C_COMPILER_INCLUDE_DIR}) + elseif(MPI_C_INCLUDE_DIR) + set(MPI_INCLUDE_DIR ${MPI_C_INCLUDE_DIR}) + elseif(MPI_C_INCLUDE_PATH) + set(MPI_INCLUDE_DIR ${MPI_C_INCLUDE_PATH}) + elseif(MPI_INCLUDE_PATH) + set(MPI_INCLUDE_DIR ${MPI_INCLUDE_PATH}) + elseif(MPICH_DIR) + set(MPI_INCLUDE_DIR ${MPICH_DIR}/include) + elseif(DEFINED ENV{MPICH_DIR}) + set(MPI_INCLUDE_DIR $ENV{MPICH_DIR}/include) + else() + message(WARNING "MPI include directory not found. Please specify -DMPI_INCLUDE_DIR or the compilation may fail.") + endif() + + if (HYPRE_ENABLE_CUDA OR HYPRE_ENABLE_HIP OR HYPRE_ENABLE_SYCL) + message(STATUS "Adding MPI include directory: ${MPI_INCLUDE_DIR}") + target_include_directories(${PROJECT_NAME} PUBLIC ${MPI_INCLUDE_DIR}) + endif () + + set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} MPI::MPI_C) + + # Check if MPI supports the MPI_Comm_f2c function + include(CheckCSourceCompiles) + check_c_source_compiles(" + #include + int main() { + MPI_Comm c = MPI_Comm_f2c(0); + return 0; + } + " HYPRE_HAVE_MPI_COMM_F2C) + + # Define a pattern for LTO-related flags (compiler-specific) + set(LTO_FLAG_PATTERNS ".*lto.*" ".*ipo.*" ".*-fthinlto.*" ".*fat-lto-objects.*") + + # Remove LTO-related flags from MPI target properties if applicable + foreach (mpi_target MPI::MPI_C MPI::MPI_CXX) + if (TARGET ${mpi_target}) + get_target_property(target_options ${mpi_target} INTERFACE_COMPILE_OPTIONS) + if (target_options) + #message(STATUS "target_options: ${target_options}") + set(original_options "${target_options}") # Save for comparison + list(APPEND target_options) # Ensure it's treated as a list + + # Remove matching flags + set(removed_flags "") + list(APPEND removed_flags) + foreach (pattern IN LISTS LTO_FLAG_PATTERNS) + foreach (flag IN LISTS target_options) + if("${flag}" MATCHES "${pattern}") + list(REMOVE_ITEM target_options "${flag}") + list(APPEND removed_flags "${flag}") + endif() + endforeach() + endforeach() + #message(STATUS "removed_flags: ${removed_flags}") + list(LENGTH removed_flags removed_flags_length) + if (removed_flags_length GREATER 0) + set(target_options "${target_options}" CACHE STRING "Updated ${target_options} without LTO-related flags" FORCE) + set_target_properties(${mpi_target} PROPERTIES INTERFACE_COMPILE_OPTIONS "${target_options}") + message(STATUS "Removed LTO-related flags from ${mpi_target}: ${removed_flags}") + endif() + endif() + endif() + endforeach() +endfunction() + +# Function to get dependency library version +function(get_library_version LIBNAME) + if(TARGET ${LIBNAME}::${LIBNAME}) + get_target_property(LIB_VERSION ${LIBNAME}::${LIBNAME} VERSION) + endif() + if(NOT LIB_VERSION) + if(DEFINED ${LIBNAME}_VERSION) + set(LIB_VERSION "${${LIBNAME}_VERSION}") + elseif(DEFINED ${LIBNAME}_VERSION_STRING) + set(LIB_VERSION "${${LIBNAME}_VERSION_STRING}") + elseif(DEFINED ${LIBNAME}_VERSION_MAJOR AND DEFINED ${LIBNAME}_VERSION_MINOR) + set(LIB_VERSION "${${LIBNAME}_VERSION_MAJOR}.${${LIBNAME}_VERSION_MINOR}") + if(DEFINED ${LIBNAME}_VERSION_PATCH) + set(LIB_VERSION "${LIB_VERSION}.${${LIBNAME}_VERSION_PATCH}") + endif() + endif() + endif() + if(LIB_VERSION) + message(STATUS " ${LIBNAME} version: ${LIB_VERSION}") + else() + message(STATUS " ${LIBNAME} version: unknown") + endif() +endfunction() + +# Macro to extract language flags +macro(get_language_flags in_var out_var lang_type) + string(REGEX MATCHALL "\\$<\\$:([^>]*)>" matches "${in_var}") + if(matches) + string(REGEX REPLACE "\\$<\\$:" "" temp "${matches}") + string(REGEX REPLACE ">" "" temp "${temp}") + set(${out_var} ${temp}) + else() + set(${out_var} "") + endif() +endmacro() + +# Function to handle TPL (Third-Party Library) setup +function(setup_tpl LIBNAME) + string(TOUPPER ${LIBNAME} LIBNAME_UPPER) + + # Note we need to check for "USING" instead of "WITH" because + # we want to allow for post-processing of build options via cmake + if(HYPRE_USING_${LIBNAME_UPPER}) + if(TPL_${LIBNAME_UPPER}_LIBRARIES AND TPL_${LIBNAME_UPPER}_INCLUDE_DIRS) + # Use specified TPL libraries and include dirs + foreach(dir ${TPL_${LIBNAME_UPPER}_INCLUDE_DIRS}) + if(NOT EXISTS ${dir}) + message(FATAL_ERROR "${LIBNAME_UPPER} include directory not found: ${dir}") + endif() + endforeach() + + foreach(lib ${TPL_${LIBNAME_UPPER}_LIBRARIES}) + if(EXISTS ${lib}) + message(STATUS "${LIBNAME_UPPER} library found: ${lib}") + get_filename_component(LIB_DIR "${lib}" DIRECTORY) + set_property(TARGET ${PROJECT_NAME} APPEND PROPERTY INSTALL_RPATH "${LIB_DIR}") + else() + message(WARNING "${LIBNAME_UPPER} library not found at specified path: ${lib}") + endif() + endforeach() + + target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_${LIBNAME_UPPER}_LIBRARIES}) + target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_${LIBNAME_UPPER}_INCLUDE_DIRS}) + else() + # Use find_package + find_package(${LIBNAME} REQUIRED CONFIG) + if(${LIBNAME}_FOUND) + list(APPEND HYPRE_DEPENDENCY_DIRS "${${LIBNAME}_ROOT}") + set(HYPRE_DEPENDENCY_DIRS "${HYPRE_DEPENDENCY_DIRS}" CACHE INTERNAL "" FORCE) + + if(${LIBNAME} STREQUAL "caliper") + set(HYPRE_NEEDS_CXX TRUE PARENT_SCOPE) + endif() + + if(TARGET ${LIBNAME}::${LIBNAME}) + target_link_libraries(${PROJECT_NAME} PUBLIC ${LIBNAME}::${LIBNAME}) + message(STATUS "Found ${LIBNAME} target: ${LIBNAME}::${LIBNAME}") + elseif(TARGET ${LIBNAME}) + target_link_libraries(${PROJECT_NAME} PUBLIC ${LIBNAME}) + message(STATUS "Found ${LIBNAME} target: ${LIBNAME}") + else() + message(FATAL_ERROR "${LIBNAME} target not found. Please check your ${LIBNAME} installation") + endif() + else() + message(FATAL_ERROR "${LIBNAME_UPPER} target not found. Please check your ${LIBNAME_UPPER} installation") + endif() + + # Display library info + get_library_version(${LIBNAME}) + if(DEFINED ${LIBNAME}_DIR) + message(STATUS " Config directory: ${${LIBNAME}_DIR}") + endif() + endif() + + message(STATUS "Enabled support for using ${LIBNAME_UPPER}") + + if(${LIBNAME_UPPER} STREQUAL "SUPERLU" OR ${LIBNAME_UPPER} STREQUAL "DSUPERLU" OR ${LIBNAME_UPPER} STREQUAL "UMPIRE") + target_link_libraries(${PROJECT_NAME} PUBLIC stdc++) + endif() + + set(${LIBNAME_UPPER}_FOUND TRUE PARENT_SCOPE) + endif() +endfunction() + +# Function to setup TPL or internal library implementation +function(setup_tpl_or_internal LIB_NAME) + string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) + + if(HYPRE_USING_HYPRE_${LIB_NAME_UPPER}) + # Use internal library + add_subdirectory(${LIB_NAME}) + message(STATUS "Using internal ${LIB_NAME_UPPER}") + target_include_directories(${PROJECT_NAME} PUBLIC + $ + $ + ) + else() + # Use external library + if(TPL_${LIB_NAME_UPPER}_LIBRARIES) + # Use specified TPL libraries + message(STATUS "Enabled support for using external ${LIB_NAME_UPPER}.") + foreach(lib ${TPL_${LIB_NAME_UPPER}_LIBRARIES}) + if(EXISTS ${lib}) + message(STATUS "${LIB_NAME_UPPER} library found: ${lib}") + else() + message(WARNING "${LIB_NAME_UPPER} library not found at specified path: ${lib}") + endif() + endforeach() + target_link_libraries(${PROJECT_NAME} PUBLIC ${TPL_${LIB_NAME_UPPER}_LIBRARIES}) + else() + # Find system library + find_package(${LIB_NAME_UPPER} REQUIRED) + if(${LIB_NAME_UPPER}_FOUND) + message(STATUS "Using system ${LIB_NAME_UPPER}") + if(TARGET ${LIB_NAME_UPPER}::${LIB_NAME_UPPER}) + target_link_libraries(${PROJECT_NAME} PUBLIC ${LIB_NAME_UPPER}::${LIB_NAME_UPPER}) + else() + target_link_libraries(${PROJECT_NAME} PUBLIC ${${LIB_NAME_UPPER}_LIBRARIES}) + endif() + else() + message(FATAL_ERROR "${LIB_NAME_UPPER} not found") + endif() + endif() + endif() +endfunction() + +# Function to setup FEI (to be phased out) +function(setup_fei) + if (HYPRE_USING_FEI) + set(HYPRE_NEEDS_CXX TRUE PARENT_SCOPE) + + if (NOT TPL_FEI_INCLUDE_DIRS) + message(FATAL_ERROR "TPL_FEI_INCLUDE_DIRS option should be set for FEI support.") + endif () + + foreach (dir ${TPL_FEI_INCLUDE_DIRS}) + if (NOT EXISTS ${dir}) + message(FATAL_ERROR "FEI include directory not found: ${dir}") + endif () + target_compile_options(${PROJECT_NAME} PUBLIC -I${dir}) + endforeach () + + message(STATUS "Enabled support for using FEI.") + set(FEI_FOUND TRUE PARENT_SCOPE) + target_include_directories(${PROJECT_NAME} PUBLIC ${TPL_FEI_INCLUDE_DIRS}) + endif() +endfunction() + # A handy function to add the current source directory to a local # filename. To be used for creating a list of sources. function(convert_filenames_to_full_paths NAMES) @@ -13,6 +393,14 @@ function(convert_filenames_to_full_paths NAMES) set(${NAMES} ${tmp_names} PARENT_SCOPE) endfunction() +# A function to add hypre subdirectories to the build +function(add_hypre_subdirectories DIRS) + foreach(DIR IN LISTS DIRS) + add_subdirectory(${DIR}) + target_include_directories(${PROJECT_NAME} PRIVATE $) + endforeach() +endfunction() + # A function to add each executable in the list to the build with the # correct flags, includes, and linkage. function(add_hypre_executables EXE_SRCS) @@ -20,30 +408,182 @@ function(add_hypre_executables EXE_SRCS) foreach(SRC_FILE IN LISTS ${EXE_SRCS}) get_filename_component(SRC_FILENAME ${SRC_FILE} NAME) + # If CUDA is enabled, tag source files to be compiled with nvcc. if (HYPRE_USING_CUDA) - # If CUDA is enabled, tag source files to be compiled with nvcc. set_source_files_properties(${SRC_FILENAME} PROPERTIES LANGUAGE CUDA) - endif (HYPRE_USING_CUDA) + endif () + + # If HIP is enabled, tag source files to be compiled with hipcc/clang + if (HYPRE_USING_HIP) + set_source_files_properties(${SRC_FILENAME} PROPERTIES LANGUAGE HIP) + endif () + # If SYCL is enabled, tag source files to be compiled with dpcpp. if (HYPRE_USING_SYCL) - # If SYCL is enabled, tag source files to be compiled with dpcpp. set_source_files_properties(${SRC_FILENAME} PROPERTIES LANGUAGE CXX) - endif (HYPRE_USING_SYCL) - + endif () + # Get executable name string(REPLACE ".c" "" EXE_NAME ${SRC_FILENAME}) - # Actually add the exe + + # Add the executable add_executable(${EXE_NAME} ${SRC_FILE}) - # Link libraries - set(HYPRE_LIBS "HYPRE") + # Link with HYPRE and inherit its compile properties + target_link_libraries(${EXE_NAME} PUBLIC HYPRE) - # Link libraries for Unix systems + # For Unix systems, also link with math library if (UNIX) - list(APPEND HYPRE_LIBS m) - endif (UNIX) + target_link_libraries(${EXE_NAME} PUBLIC m) + endif () + + # Explicitly specify the linker + if ((HYPRE_USING_CUDA AND NOT HYPRE_ENABLE_LTO) OR HYPRE_USING_HIP OR HYPRE_USING_SYCL) + set_target_properties(${EXE_NAME} PROPERTIES LINKER_LANGUAGE CXX) + endif () + + # Turn on LTO if requested + if (HYPRE_ENABLE_LTO) + set_target_properties(${EXE_NAME} PROPERTIES INTERPROCEDURAL_OPTIMIZATION TRUE) + endif () + + # Inherit compile definitions and options from HYPRE target + get_target_property(HYPRE_COMPILE_OPTS HYPRE COMPILE_OPTIONS) + #message(STATUS "${EXE_NAME}: ${HYPRE_COMPILE_OPTS}") + if (HYPRE_COMPILE_OPTS) + if (HYPRE_USING_CUDA OR HYPRE_USING_HIP OR HYPRE_USING_SYCL) + get_language_flags("${HYPRE_COMPILE_OPTS}" CXX_OPTS "CXX") + target_compile_options(${EXE_NAME} PRIVATE $<$:${CXX_OPTS}>) + #message(STATUS "Added CXX compile options: ${CXX_OPTS} to ${EXE_NAME}") + else () + get_language_flags("${HYPRE_COMPILE_OPTS}" C_OPTS "C") + target_compile_options(${EXE_NAME} PRIVATE $<$:${C_OPTS}>) + #message(STATUS "Added C compile options: ${C_OPTS} to ${EXE_NAME}") + endif () + endif () + + # Copy executable to original source directory + add_custom_command(TARGET ${EXE_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy $ ${CMAKE_CURRENT_SOURCE_DIR} + COMMENT "Copied ${EXE_NAME} to ${CMAKE_CURRENT_SOURCE_DIR}" + ) + endforeach (SRC_FILE) +endfunction () + +# Function to add a tags target if etags is found +function(add_hypre_target_tags) + find_program(ETAGS_EXECUTABLE etags) + if(ETAGS_EXECUTABLE) + add_custom_target(tags + COMMAND find ${CMAKE_CURRENT_SOURCE_DIR} + -type f + "(" -name "*.h" -o -name "*.c" -o -name "*.cpp" + -o -name "*.hpp" -o -name "*.cxx" + -o -name "*.f" -o -name "*.f90" ")" + -not -path "*/build/*" + -print | ${ETAGS_EXECUTABLE} + --declarations + --ignore-indentation + --no-members + - + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMENT "Generating TAGS file with etags" + VERBATIM + ) + endif() +endfunction() + +# Function to add a distclean target +function(add_hypre_target_distclean) + add_custom_target(distclean + COMMAND find ${CMAKE_BINARY_DIR} -mindepth 1 -delete + COMMAND find ${CMAKE_SOURCE_DIR} -name "*.o" -type f -delete + COMMAND find ${CMAKE_SOURCE_DIR} -name "*.mod" -type f -delete + COMMAND find ${CMAKE_SOURCE_DIR} -name "*~" -type f -delete + COMMAND find ${CMAKE_SOURCE_DIR}/test -name "*.out*" -type f -delete + COMMAND find ${CMAKE_SOURCE_DIR}/test -name "*.err*" -type f -delete + COMMAND find ${CMAKE_SOURCE_DIR}/examples -type f -name "ex[0-9]" -name "ex[10-19]" -delete + COMMAND find ${CMAKE_SOURCE_DIR}/test -type f -name "ij|struct|sstruct|ams_driver|maxwell_unscalled|struct_migrate|sstruct_fac|ij_assembly" -delete + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMENT "Removing all build artifacts and generated files" + VERBATIM + ) +endfunction() + +# Function to add an uninstall target +function(add_hypre_target_uninstall) + add_custom_target(uninstall + COMMAND ${CMAKE_COMMAND} -E remove_directory "${CMAKE_INSTALL_PREFIX}" + COMMAND ${CMAKE_COMMAND} -E echo "Removed installation directory: ${CMAKE_INSTALL_PREFIX}" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Uninstalling HYPRE" + VERBATIM + ) +endfunction() + +# Function to print the status of build options +function(print_option_status) + # Define column widths + set(COLUMN1_WIDTH 40) + set(COLUMN2_WIDTH 10) + math(EXPR HEADER1_PAD "${COLUMN1_WIDTH} - 3") + math(EXPR HEADER2_PAD "${COLUMN2_WIDTH} - 1") + + # Create separator line + string(REPEAT "-" ${HEADER1_PAD} SEPARATOR1) + string(REPEAT "-" ${HEADER2_PAD} SEPARATOR2) + set(separator "+${SEPARATOR1}+${SEPARATOR2}+") + + # Function to print a block of options + function(print_option_block title options) + message(STATUS "") + message(STATUS " ${title}:") + message(STATUS " ${separator}") + message(STATUS " | Option | Status |") + message(STATUS " ${separator}") + + foreach(opt ${options}) + if(${${opt}}) + set(status "ON") + else() + set(status "OFF") + endif() + + string(LENGTH "${opt}" opt_length) + math(EXPR padding "${COLUMN1_WIDTH} - ${opt_length} - 5") + if(${padding} GREATER 0) + string(REPEAT " " ${padding} pad_spaces) + else() + set(pad_spaces "") + endif() + + string(LENGTH "${status}" status_length) + math(EXPR status_padding "${COLUMN2_WIDTH} - ${status_length} - 3") + if(${status_padding} GREATER 0) + string(REPEAT " " ${status_padding} status_pad_spaces) + else() + set(status_pad_spaces "") + endif() + + message(STATUS " | ${opt}${pad_spaces} | ${status}${status_pad_spaces} |") + endforeach() + + message(STATUS " ${separator}") + endfunction() + + message(STATUS "") + message(STATUS "HYPRE Configuration Summary:") + + # Print BASE_OPTIONS + print_option_block("Base Options" "${BASE_OPTIONS}") + + # Print GPU_OPTIONS + if(HYPRE_ENABLE_CUDA OR HYPRE_ENABLE_HIP OR HYPRE_ENABLE_SYCL) + print_option_block("GPU Options" "${GPU_OPTIONS}") + endif() + + # Print TPL_OPTIONS + print_option_block("Third-Party Library Options" "${TPL_OPTIONS}") - # Append the additional libraries and options - target_link_libraries(${EXE_NAME} PRIVATE "${HYPRE_LIBS}") - endforeach(SRC_FILE) + message(STATUS "") endfunction() diff --git a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake index e66d040e70..3b08d8327b 100644 --- a/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake +++ b/src/config/cmake/HYPRE_SetupCUDAToolkit.cmake @@ -3,130 +3,255 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -# This handles the non-compiler aspect of the CUDA toolkit. -# Uses cmake find_package to locate the NVIDIA CUDA C tools -# for shared libraries. Otherwise for static libraries, assumes -# the libraries are located in ${CUDA_TOOLKIT_ROOT_DIR}/lib64. -# Please set cmake variable CUDA_TOOLKIT_ROOT_DIR. +message(STATUS "Enabling CUDA toolkit") -# Collection of CUDA optional libraries -set(EXPORT_INTERFACE_CUDA_LIBS "") - -if (NOT CUDA_FOUND) - find_package(CUDA REQUIRED) -endif () - -if (CMAKE_VERSION VERSION_LESS 3.17) - - if (HYPRE_ENABLE_CUSPARSE) - set(HYPRE_USING_CUSPARSE ON CACHE BOOL "" FORCE) - if (HYPRE_SHARED) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS ${CUDA_cusparse_LIBRARY}) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcusparse_static.a) - endif () - endif () - - if (HYPRE_ENABLE_CURAND) - set(HYPRE_USING_CURAND ON CACHE BOOL "" FORCE) - if (HYPRE_SHARED) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS ${CUDA_curand_LIBRARY}) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcurand_static.a) - endif () - endif () - - if (HYPRE_ENABLE_CUBLAS) - set(HYPRE_USING_CUBLAS ON CACHE BOOL "" FORCE) - if (HYPRE_SHARED) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS ${CUDA_CUBLAS_LIBRARIES}) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcublas_static.a) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcublasLt_static.a) - endif (HYPRE_SHARED) - endif (HYPRE_ENABLE_CUBLAS) - - if (HYPRE_ENABLE_CUSOLVER) - set(HYPRE_USING_CUSOLVER ON CACHE BOOL "" FORCE) - if (HYPRE_SHARED) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS ${CUDA_cusolver_LIBRARY}) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libcusolver_static.a) - endif () - endif () - - if (NOT HYPRE_SHARED) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS - ${CUDA_TOOLKIT_ROOT_DIR}/lib64/libculibos.a) - endif () - - if (HYPRE_ENABLE_GPU_PROFILING) - set(HYPRE_USING_NVTX ON CACHE BOOL "" FORCE) - find_library(NVTX_LIBRARY - NAME libnvToolsExt.so - PATHS ${CUDA_TOOLKIT_ROOT_DIR}/lib64 ${CUDA_TOOLKIT_ROOT_DIR}/lib) - message(STATUS "NVidia tools extension library found in " ${NVTX_LIBRARY}) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS ${NVTX_LIBRARY}) - endif (HYPRE_ENABLE_GPU_PROFILING) +# Check for CUDA_PATH, CUDA_HOME or CUDA_DIR +if(DEFINED CUDAToolkit_ROOT) + set(CUDA_DIR ${CUDAToolkit_ROOT}) +elseif(DEFINED ENV{CUDAToolkit_ROOT}) + set(CUDA_DIR $ENV{CUDAToolkit_ROOT}) +elseif(DEFINED CUDA_DIR) + set(CUDA_DIR ${CUDA_DIR}) +elseif(DEFINED ENV{CUDA_DIR}) + set(CUDA_DIR $ENV{CUDA_DIR}) +elseif(DEFINED CUDA_PATH) + set(CUDA_DIR ${CUDA_PATH}) +elseif(DEFINED ENV{CUDA_PATH}) + set(CUDA_DIR $ENV{CUDA_PATH}) +elseif(DEFINED CUDA_HOME) + set(CUDA_DIR ${CUDA_HOME}) +elseif(DEFINED ENV{CUDA_HOME}) + set(CUDA_DIR $ENV{CUDA_HOME}) +elseif(EXISTS "/opt/cuda") + set(CUDA_DIR "/opt/cuda") +elseif(EXISTS "/usr/bin/nvcc") + set(CUDA_DIR "/usr") +else() + message(FATAL_ERROR "CUDA_PATH or CUDA_HOME not set. Please set one of them to point to your CUDA installation.") +endif() +message(STATUS "Using CUDA installation: ${CUDA_DIR}") +# Specify the path to the custom nvcc compiler +if(WIN32) + set(CMAKE_CUDA_COMPILER "${CUDA_DIR}/bin/nvcc.exe" CACHE FILEPATH "CUDA compiler") else() + set(CMAKE_CUDA_COMPILER "${CUDA_DIR}/bin/nvcc" CACHE FILEPATH "CUDA compiler") +endif() + +# Specify the CUDA Toolkit root directory +set(CUDAToolkit_ROOT "${CUDA_DIR}" CACHE PATH "Path to the CUDA toolkit") + +# Optionally, prioritize the custom CUDA path in CMAKE_PREFIX_PATH +list(APPEND CMAKE_PREFIX_PATH "${CUDA_DIR}") + +# Set CUDA standard to match C++ standard if not already set +if(NOT DEFINED CMAKE_CUDA_STANDARD) + set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD}) +endif() +set(CMAKE_CUDA_STANDARD_REQUIRED ON) +set(CMAKE_CUDA_EXTENSIONS OFF) + +# Visual Studio does not support CMAKE_CUDA_HOST_COMPILER +if (NOT MSVC) + set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER} CACHE STRING "CXX compiler used by CUDA" FORCE) +endif() + +# Check if CUDA is available and enable it if found +include(CheckLanguage) +check_language(CUDA) +if(DEFINED CMAKE_CUDA_COMPILER) + enable_language(CUDA) +else() + message(FATAL_ERROR "CUDA language not found. Please check your CUDA installation.") +endif() - find_package(CUDAToolkit REQUIRED) +# Find the CUDA Toolkit +find_package(CUDAToolkit REQUIRED) - if (HYPRE_SHARED OR WIN32) - set(HYPRE_CUDA_TOOLKIT_STATIC FALSE) +# Add a dummy cuda target if it doesn't exist (avoid error when building with BLT dependencies) +if(NOT TARGET cuda) + add_library(cuda INTERFACE) +endif() + +# Detection CUDA architecture if not given by the user +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "52") + message(STATUS "Detecting CUDA GPU architectures using nvidia-smi...") + + # Platform-specific NVIDIA smi command + if (WIN32) + # Try multiple possible locations on Windows + find_program(NVIDIA_SMI_CMD + NAMES nvidia-smi.exe + PATHS + "${CUDA_DIR}/bin" + "$ENV{ProgramFiles}/NVIDIA Corporation/NVSMI" + "$ENV{ProgramFiles}/NVIDIA GPU Computing Toolkit/CUDA/v*/bin" + "$ENV{ProgramW6432}/NVIDIA Corporation/NVSMI" + NO_DEFAULT_PATH + ) + + if(NOT NVIDIA_SMI_CMD) + find_program(NVIDIA_SMI_CMD nvidia-smi.exe) + endif() + message(STATUS "Found nvidia-smi: ${NVIDIA_SMI_CMD}") else() - set(HYPRE_CUDA_TOOLKIT_STATIC TRUE) + set(NVIDIA_SMI_CMD "nvidia-smi") endif() - if (HYPRE_ENABLE_CUSPARSE) - set(HYPRE_USING_CUSPARSE ON CACHE BOOL "" FORCE) - if (HYPRE_CUDA_TOOLKIT_STATIC) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cusparse_static) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cusparse) - endif () - endif () - - if (HYPRE_ENABLE_CURAND) - set(HYPRE_USING_CURAND ON CACHE BOOL "" FORCE) - if (HYPRE_CUDA_TOOLKIT_STATIC) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::curand_static) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::curand) - endif () - endif () - - if (HYPRE_ENABLE_CUBLAS) - set(HYPRE_USING_CUBLAS ON CACHE BOOL "" FORCE) - if (HYPRE_CUDA_TOOLKIT_STATIC) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cublas_static CUDA::cublasLt_static) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cublas CUDA::cublasLt) - endif (HYPRE_CUDA_TOOLKIT_STATIC) - endif (HYPRE_ENABLE_CUBLAS) - - if (HYPRE_ENABLE_CUSOLVER) - set(HYPRE_USING_CUSOLVER ON CACHE BOOL "" FORCE) - if (HYPRE_CUDA_TOOLKIT_STATIC) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cusolver_static) - else () - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::cusolver) - endif () - endif () - - if (HYPRE_CUDA_TOOLKIT_STATIC) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::culibos) - endif (HYPRE_CUDA_TOOLKIT_STATIC) - - if (HYPRE_ENABLE_GPU_PROFILING) - set(HYPRE_USING_NVTX ON CACHE BOOL "" FORCE) - list(APPEND EXPORT_INTERFACE_CUDA_LIBS CUDA::nvToolsExt) - endif (HYPRE_ENABLE_GPU_PROFILING) + if(NOT NVIDIA_SMI_CMD) + message(WARNING "nvidia-smi not found. Using default CUDA architecture 70.") + set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "Default CUDA architectures" FORCE) + else() + # Execute nvidia-smi to get GPU compute capabilities + execute_process( + COMMAND ${NVIDIA_SMI_CMD} --query-gpu=compute_cap --format=csv,noheader + OUTPUT_VARIABLE NVIDIA_SMI_OUTPUT + RESULT_VARIABLE NVIDIA_SMI_RESULT + ERROR_VARIABLE NVIDIA_SMI_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + if(NOT NVIDIA_SMI_RESULT EQUAL 0) + message(WARNING "${NVIDIA_SMI_CMD} failed to execute: ${NVIDIA_SMI_ERROR}") + set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "Default CUDA architectures" FORCE) + message(STATUS "Setting CMAKE_CUDA_ARCHITECTURES to default '70'") + else() + # Clean the output (remove extra newlines and spaces) + string(STRIP "${NVIDIA_SMI_OUTPUT}" CUDA_ARCHS) # Remove trailing/leading whitespaces + string(REPLACE "." "" CUDA_ARCHS "${CUDA_ARCHS}") # Replace '.' with nothing to format '7.0' as '70' + string(REPLACE "\n" ";" CUDA_ARCHS "${CUDA_ARCHS}") # Replace newline with semicolon for list format + + # Remove any duplicates CUDA archictectures + list(REMOVE_DUPLICATES CUDA_ARCHS) + if(CUDA_ARCHS) + string(REPLACE ";" "," CUDA_ARCHS_STR "${CUDA_ARCHS}") + set(CMAKE_CUDA_ARCHITECTURES "${CUDA_ARCHS_STR}" CACHE STRING "Detected CUDA architectures" FORCE) + message(STATUS "Detected CUDA GPU architectures: ${CMAKE_CUDA_ARCHITECTURES}") + else() + message(WARNING "No GPUs detected. Setting CMAKE_CUDA_ARCHITECTURES to default '70'") + set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "Default CUDA architectures" FORCE) + endif() + endif() + endif() +else() + # Remove duplicates from the pre-set CMAKE_CUDA_ARCHITECTURES + string(REPLACE "," ";" CUDA_ARCH_LIST "${CMAKE_CUDA_ARCHITECTURES}") + list(REMOVE_DUPLICATES CUDA_ARCH_LIST) + string(REPLACE ";" "," CUDA_ARCH_STR "${CUDA_ARCH_LIST}") + set(CMAKE_CUDA_ARCHITECTURES "${CUDA_ARCH_STR}" CACHE STRING "Detected CUDA architectures" FORCE) + message(STATUS "CMAKE_CUDA_ARCHITECTURES is already set to: ${CMAKE_CUDA_ARCHITECTURES}") endif() + +# Set the CUDA architectures to the HYPRE target +set_property(TARGET HYPRE PROPERTY CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}") + +# Show CUDA Toolkit location +if(CUDAToolkit_FOUND) + if (CUDAToolkit_ROOT) + message(STATUS "CUDA Toolkit found at: ${CUDAToolkit_ROOT}") + endif() + message(STATUS "CUDA Toolkit version: ${CUDAToolkit_VERSION}") + message(STATUS "CUDA Toolkit include directory: ${CUDAToolkit_INCLUDE_DIRS}") + message(STATUS "CUDA Toolkit library directory: ${CUDAToolkit_LIBRARY_DIR}") +else() + message(FATAL_ERROR "CUDA Toolkit not found") +endif() + +# Check for Thrust headers +find_path(THRUST_INCLUDE_DIR thrust/version.h + HINTS ${CUDAToolkit_INCLUDE_DIRS} ${CUDAToolkit_INCLUDE_DIRS}/cuda-thrust + PATH_SUFFIXES thrust + NO_DEFAULT_PATH +) + +if(THRUST_INCLUDE_DIR) + message(STATUS "CUDA Thrust headers found in: ${THRUST_INCLUDE_DIR}") +else() + message(FATAL_ERROR "CUDA Thrust headers not found! Please specify -DTHRUST_INCLUDE_DIR.") +endif() + +# Collection of CUDA optional libraries +set(CUDA_LIBS "") + +# Function to handle CUDA libraries +function(find_and_add_cuda_library LIB_NAME HYPRE_ENABLE_VAR) + string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) + if(${HYPRE_ENABLE_VAR}) + set(HYPRE_USING_${LIB_NAME_UPPER} ON CACHE INTERNAL "") + + # Use CUDAToolkit to find the component + find_package(CUDAToolkit REQUIRED COMPONENTS ${LIB_NAME}) + + if(TARGET CUDA::${LIB_NAME}) + message(STATUS "Found ${LIB_NAME_UPPER} library") + list(APPEND CUDA_LIBS CUDA::${LIB_NAME}) + else() + message(STATUS "CUDA::${LIB_NAME} target not found. Attempting manual linking.") + find_library(${LIB_NAME}_LIBRARY ${LIB_NAME} HINTS ${CUDAToolkit_LIBRARY_DIR}) + if(${LIB_NAME}_LIBRARY) + message(STATUS "Found ${LIB_NAME_UPPER} library: ${${LIB_NAME}_LIBRARY}") + add_library(CUDA::${LIB_NAME} UNKNOWN IMPORTED) + set_target_properties(CUDA::${LIB_NAME} PROPERTIES + IMPORTED_LOCATION "${${LIB_NAME}_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${CUDAToolkit_INCLUDE_DIRS}") + list(APPEND CUDA_LIBS CUDA::${LIB_NAME}) + else() + message(FATAL_ERROR "Could not find ${LIB_NAME_UPPER} library. Please check your CUDA installation.") + endif() + endif() + + set(CUDA_LIBS ${CUDA_LIBS} PARENT_SCOPE) + endif() +endfunction() + +# Handle CUDA libraries +find_and_add_cuda_library(cusparse HYPRE_ENABLE_CUSPARSE) +find_and_add_cuda_library(curand HYPRE_ENABLE_CURAND) +find_and_add_cuda_library(cublas HYPRE_ENABLE_CUBLAS) +find_and_add_cuda_library(cublasLt HYPRE_ENABLE_CUBLAS) +find_and_add_cuda_library(cusolver HYPRE_ENABLE_CUSOLVER) + +# Handle GPU Profiling with nvToolsExt +if(HYPRE_ENABLE_GPU_PROFILING) + find_package(nvToolsExt REQUIRED) + set(HYPRE_USING_NVTX ON CACHE BOOL "" FORCE) + list(APPEND CUDA_LIBS CUDA::nvToolsExt) +endif() + +# Add CUDA Toolkit include directories to the target +target_include_directories(HYPRE PUBLIC ${CUDAToolkit_INCLUDE_DIRS}) + +# Add Thrust include directory +target_include_directories(HYPRE PUBLIC ${THRUST_INCLUDE_DIR}) + +# Link CUDA libraries to the target +target_link_libraries(HYPRE PUBLIC ${CUDA_LIBS}) +message(STATUS "Linking to CUDA libraries: ${CUDA_LIBS}") + +# Set additional CUDA compiler flags +set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --extended-lambda") + +# Ensure LTO-specific flags are included +if (HYPRE_ENABLE_LTO AND CUDAToolkit_VERSION VERSION_LESS 11.2) + # See https://developer.nvidia.com/blog/improving-gpu-app-performance-with-cuda-11-2-device-lto + message(WARNING "Device LTO not available on CUDAToolkit_VERSION (${CUDAToolkit_VERSION}) < 11.2. Turning it off...") + +elseif (HYPRE_ENABLE_LTO AND CMAKE_VERSION VERSION_LESS 3.25) + # See https://gitlab.kitware.com/cmake/cmake/-/commit/96bc59b1ca01be231347404d178445263687dd22 + message(WARNING "Device LTO not available with CUDA on CMAKE_VERSION (${CMAKE_VERSION}) < 3.25. Turning it off...") + +elseif (HYPRE_ENABLE_LTO) + message(STATUS "Enabling Device LTO") + + # Enable LTO for the target + set_target_properties(${PROJECT_NAME} PROPERTIES + CUDA_SEPARABLE_COMPILATION ON + CUDA_RESOLVE_DEVICE_SYMBOLS ON + ) +endif() + +# Print CUDA info +message(STATUS "CUDA C++ standard: ${CMAKE_CUDA_STANDARD}") +message(STATUS "CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}") +message(STATUS "CUDA flags: ${CMAKE_CUDA_FLAGS}") diff --git a/src/config/cmake/HYPRE_SetupGPUToolkit.cmake b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake new file mode 100644 index 0000000000..a2d37af979 --- /dev/null +++ b/src/config/cmake/HYPRE_SetupGPUToolkit.cmake @@ -0,0 +1,66 @@ +# Copyright (c) 1998 Lawrence Livermore National Security, LLC and other +# HYPRE Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# Enable CXX language +enable_language(CXX) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +if(HYPRE_ENABLE_SYCL) + # We enforce the use of Intel's oneAPI DPC++/C++ Compiler + if(NOT CMAKE_CXX_COMPILER MATCHES "dpcpp|icpx") + message(FATAL_ERROR "SYCL requires DPC++ or Intel C++ compiler") + endif() + + # Enforce C++17 at least for SYCL + if(NOT DEFINED CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 17) + set(CMAKE_CXX_STANDARD 17) + endif() +else() + # Enforce C++14 at least for CUDA and HIP + if(NOT DEFINED CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD LESS 14) + set(CMAKE_CXX_STANDARD 14) + endif() +endif() +set_property(TARGET HYPRE PROPERTY CXX_STANDARD ${CMAKE_CXX_STANDARD}) +message(STATUS "Enabling support for CXX.") +message(STATUS "Using CXX standard: C++${CMAKE_CXX_STANDARD}") + +# Set GPU-related variables +set(HYPRE_USING_GPU ON CACHE INTERNAL "") +set(HYPRE_USING_HOST_MEMORY OFF CACHE INTERNAL "") +set(HYPRE_ENABLE_HOST_MEMORY OFF CACHE BOOL "Use host memory" FORCE) + +if(HYPRE_ENABLE_UNIFIED_MEMORY) + set(HYPRE_USING_UNIFIED_MEMORY ON CACHE INTERNAL "") +else() + set(HYPRE_USING_DEVICE_MEMORY ON CACHE INTERNAL "") +endif() + +# Check if examples are enabled, but not unified memory +if(HYPRE_BUILD_EXAMPLES AND NOT HYPRE_ENABLE_UNIFIED_MEMORY) + message(WARNING "Running the examples on GPUs requires Unified Memory!\nExamples will not be built!\n") + set(HYPRE_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +endif() + +# Add any extra CXX compiler flags +if(NOT HYPRE_WITH_EXTRA_CXXFLAGS STREQUAL "") + string(REPLACE " " ";" HYPRE_WITH_EXTRA_CXXFLAGS_LIST ${HYPRE_WITH_EXTRA_CXXFLAGS}) + target_compile_options(HYPRE PRIVATE + $<$:${HYPRE_WITH_EXTRA_CXXFLAGS_LIST}>) +endif() + +# Include the toolkit setup file for the selected GPU architecture +if(HYPRE_ENABLE_CUDA) + include(HYPRE_SetupCUDAToolkit) + +elseif(HYPRE_ENABLE_HIP) + include(HYPRE_SetupHIPToolkit) + +elseif(HYPRE_ENABLE_SYCL) + include(HYPRE_SetupSYCLToolkit) + set(EXPORT_DEVICE_LIBS ${EXPORT_INTERFACE_SYCL_LIBS}) + +else() + message(FATAL_ERROR "Neither CUDA nor HIP nor SYCL is enabled. Please enable one of them.") +endif() diff --git a/src/config/cmake/HYPRE_SetupHIPToolkit.cmake b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake new file mode 100644 index 0000000000..8fc918b7ac --- /dev/null +++ b/src/config/cmake/HYPRE_SetupHIPToolkit.cmake @@ -0,0 +1,192 @@ +# Copyright (c) 1998 Lawrence Livermore National Security, LLC and other +# HYPRE Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# Check for ROCM_PATH or HIP_PATH +message(STATUS "Enabling HIP toolkit") +if(DEFINED ROCM_PATH) + set(HIP_PATH ${ROCM_PATH}) +elseif(DEFINED ENV{ROCM_PATH}) + set(HIP_PATH $ENV{ROCM_PATH}) +elseif(DEFINED ENV{HIP_PATH}) + set(HIP_PATH $ENV{HIP_PATH}) +elseif(EXISTS "/opt/rocm") + set(HIP_PATH "/opt/rocm") +else() + message(FATAL_ERROR "ROCM_PATH or HIP_PATH not set. Please set one of them to point to your ROCm installation.") +endif() +message(STATUS "Using ROCm installation: ${HIP_PATH}") + +# Add HIP_PATH to CMAKE_PREFIX_PATH +list(APPEND CMAKE_PREFIX_PATH ${HIP_PATH}) + +# Set HIP standard to match C++ standard if not already set +if(NOT DEFINED CMAKE_HIP_STANDARD) + set(CMAKE_HIP_STANDARD ${CMAKE_CXX_STANDARD} CACHE STRING "C++ standard for HIP" FORCE) +endif() +set(CMAKE_HIP_STANDARD_REQUIRED ON CACHE BOOL "Require C++ standard for HIP" FORCE) + +# Check if HIP is available and enable it if found +include(CheckLanguage) +check_language(HIP) +if(CMAKE_HIP_COMPILER) + enable_language(HIP) +else() + message(FATAL_ERROR "HIP language not found. Please check your HIP/ROCm installation.") +endif() + +# Find HIP package +find_package(hip REQUIRED CONFIG) + +# Function to detect GPU architectures using rocm-smi +if(NOT DEFINED CMAKE_HIP_ARCHITECTURES) + message(STATUS "Detecting GPU architectures using rocm-smi...") + + # Execute rocm-smi to get GPU architecture info + execute_process( + COMMAND rocm-smi --showUniqueId --format=json + OUTPUT_VARIABLE ROCM_SMI_OUTPUT + RESULT_VARIABLE ROCM_SMI_RESULT + ERROR_VARIABLE ROCM_SMI_ERROR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + if(NOT ROCM_SMI_RESULT EQUAL 0) + message(WARNING "rocm-smi failed to execute: ${ROCM_SMI_ERROR}") + set(CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "Default HIP architectures" FORCE) + message(STATUS "Setting CMAKE_HIP_ARCHITECTURES to default 'gfx90a'") + return() + endif() + + # Parse JSON output to extract GPU architectures + include(Jsoncpp) + jsoncpp_parse(json_output "${ROCM_SMI_OUTPUT}") + + # Extract GPU architecture codes + set(GPU_ARCHS "") + foreach(gpu ${json_output@/gpu/@}) + get_property(gpu_arch PROPERTY GPU ARCHITECTURE "${gpu}") + if(NOT gpu_arch STREQUAL "") + list(APPEND GPU_ARCHS ${gpu_arch}) + endif() + endforeach() + + list(REMOVE_DUPLICATES GPU_ARCHS) + + if(GPU_ARCHS) + string(REPLACE ";" "," GPU_ARCHS_STR "${GPU_ARCHS}") + set(CMAKE_HIP_ARCHITECTURES "${GPU_ARCHS_STR}" CACHE STRING "Detected HIP architectures" FORCE) + message(STATUS "Detected GPU architectures: ${CMAKE_HIP_ARCHITECTURES}") + else() + message(WARNING "No GPUs detected. Setting CMAKE_HIP_ARCHITECTURES to default 'gfx90a'") + set(CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "Default HIP architectures" FORCE) + endif() +else() + # Remove duplicates from the pre-set CMAKE_HIP_ARCHITECTURES + string(REPLACE "," ";" HIP_ARCH_LIST "${CMAKE_HIP_ARCHITECTURES}") + list(REMOVE_DUPLICATES HIP_ARCH_LIST) + string(REPLACE ";" "," HIP_ARCH_STR "${HIP_ARCH_LIST}") + set(CMAKE_HIP_ARCHITECTURES "${HIP_ARCH_STR}" CACHE STRING "Detected HIP architectures" FORCE) + message(STATUS "CMAKE_HIP_ARCHITECTURES is already set to: ${CMAKE_HIP_ARCHITECTURES}") +endif() +set_property(TARGET ${PROJECT_NAME} PROPERTY HIP_ARCHITECTURES "${CMAKE_HIP_ARCHITECTURES}") + +# Collection of ROCm optional libraries +set(ROCM_LIBS "") + +# Function to find and add libraries +function(find_and_add_rocm_library LIB_NAME) + string(TOUPPER ${LIB_NAME} LIB_NAME_UPPER) + set(HYPRE_ENABLE_VAR "HYPRE_ENABLE_${LIB_NAME_UPPER}") + if(${HYPRE_ENABLE_VAR}) + set(HYPRE_USING_${LIB_NAME_UPPER} ON CACHE INTERNAL "") + find_package(${LIB_NAME} REQUIRED) + if(TARGET roc::${LIB_NAME}) + message(STATUS "Found target: ${${LIB_NAME}_LIBRARY}") + list(APPEND ROCM_LIBS ${${LIB_NAME}_LIBRARY}) + else() + #message(WARNING "roc::${LIB_NAME} target not found. Attempting manual linking.") + find_library(${LIB_NAME}_LIBRARY ${LIB_NAME} HINTS ${HIP_PATH}/lib ${HIP_PATH}/lib64) + if(${LIB_NAME}_LIBRARY) + message(STATUS "Found ${LIB_NAME} library: ${${LIB_NAME}_LIBRARY}") + add_library(roc::${LIB_NAME} UNKNOWN IMPORTED) + set_target_properties(roc::${LIB_NAME} PROPERTIES + IMPORTED_LOCATION "${${LIB_NAME}_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${HIP_PATH}/include") + list(APPEND ROCM_LIBS roc::${LIB_NAME}) + else() + message(FATAL_ERROR "Could not find ${LIB_NAME} library. Please check your ROCm installation.") + endif() + endif() + set(ROCM_LIBS ${ROCM_LIBS} PARENT_SCOPE) + endif() +endfunction() + +# Find and add libraries +find_and_add_rocm_library(rocblas) +find_and_add_rocm_library(rocsparse) +find_and_add_rocm_library(rocrand) +find_and_add_rocm_library(rocsolver) + +if(HYPRE_ENABLE_GPU_PROFILING) + set(HYPRE_USING_ROCTRACER ON CACHE BOOL "" FORCE) + find_library(ROCTRACER_LIBRARY + NAMES libroctracer64.so + PATHS ${HIP_PATH}/lib ${HIP_PATH}/lib64 + NO_DEFAULT_PATH) + if(ROCTRACER_LIBRARY) + message(STATUS "ROCm tracer library found in ${ROCTRACER_LIBRARY}") + list(APPEND ROCM_LIBS ${ROCTRACER_LIBRARY}) + else() + message(WARNING "ROCm tracer library not found. GPU profiling may not work correctly.") + endif() +endif() + +# Add HIP include directory +target_include_directories(${PROJECT_NAME} PUBLIC ${HIP_PATH}/include) + +# Link HIP libraries to the target +target_link_libraries(${PROJECT_NAME} PUBLIC ${ROCM_LIBS}) +message(STATUS "Linking to ROCm libraries: ${ROCM_LIBS}") + +# Turn on Position Independent Code (PIC) by default +set_target_properties(${PROJECT_NAME} PROPERTIES POSITION_INDEPENDENT_CODE TRUE) + +# Signal to downstream targets that they need PIC +set_target_properties(${PROJECT_NAME} PROPERTIES INTERFACE_POSITION_INDEPENDENT_CODE TRUE) + +# Ensure LTO-specific flags are included +if (HYPRE_ENABLE_LTO AND NOT MSVC) + if (CMAKE_C_COMPILER_ID MATCHES "Clang") + message(STATUS "Enabling Device LTO") + else () + message(FATAL_ERROR "HIP Device LTO available only with Clang") + endif () + + # HIP compilation options + target_compile_options(${PROJECT_NAME} + PRIVATE + $<$:-fgpu-rdc -foffload-lto> + $<$:-foffload-lto> + INTERFACE + $<$:-foffload-lto> + $<$:-foffload-lto> + ) + + # Link options need to be more specific + target_link_options(${PROJECT_NAME} + PRIVATE + -fgpu-rdc + -foffload-lto + --hip-link + -Wl,--no-as-needed # Ensure all symbols are kept + INTERFACE + -foffload-lto + ) +endif () + +# Print HIP info +message(STATUS "HIP C++ standard: ${CMAKE_HIP_STANDARD}") +message(STATUS "HIP architectures: ${CMAKE_HIP_ARCHITECTURES}") +message(STATUS "HIP flags: ${CMAKE_HIP_FLAGS}") diff --git a/src/config/cmake/HYPRE_SetupSYCLToolkit.cmake b/src/config/cmake/HYPRE_SetupSYCLToolkit.cmake new file mode 100644 index 0000000000..dbaa06b639 --- /dev/null +++ b/src/config/cmake/HYPRE_SetupSYCLToolkit.cmake @@ -0,0 +1,104 @@ +# Copyright (c) 1998 Lawrence Livermore National Security, LLC and other +# HYPRE Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +# This handles the non-compiler aspect of the SYCL toolkit. +message(STATUS "Enabling SYCL toolkit") + +# limit C++ errors to one +if(CMAKE_CXX_COMPILER_ID MATCHES "Intel|Clang") + target_compile_options(${PROJECT_NAME} PRIVATE $<$:-ferror-limit=1>) +endif() + +# Find Intel SYCL +find_package(IntelSYCL REQUIRED) + +# Set up SYCL flags +if(IntelSYCL_FOUND) + # Standard SYCL flags + target_compile_options(${PROJECT_NAME} PUBLIC $<$:-fsycl>) + target_compile_options(${PROJECT_NAME} PUBLIC $<$:-fsycl-unnamed-lambda>) + target_link_options(${PROJECT_NAME} PUBLIC -fsycl) + target_link_options(${PROJECT_NAME} PUBLIC -fsycl-device-code-split=per_kernel) + target_link_options(${PROJECT_NAME} PUBLIC -Wl,--no-relax) + + # Use either user-specified target or IntelSYCL's default + if(HYPRE_SYCL_TARGET) + target_compile_options(${PROJECT_NAME} PUBLIC $<$:-fsycl-targets=${HYPRE_SYCL_TARGET}>) + target_link_options(${PROJECT_NAME} PUBLIC -fsycl-targets=${HYPRE_SYCL_TARGET}) + elseif(INTEL_SYCL_TARGETS) + target_compile_options(${PROJECT_NAME} PUBLIC $<$:-fsycl-targets=${INTEL_SYCL_TARGETS}>) + target_link_options(${PROJECT_NAME} PUBLIC -fsycl-targets=${INTEL_SYCL_TARGETS}) + endif() + + # Use either user-specified backend or IntelSYCL's default + if(HYPRE_SYCL_TARGET_BACKEND) + target_compile_options(${PROJECT_NAME} PUBLIC $<$:-Xsycl-target-backend=${HYPRE_SYCL_TARGET_BACKEND}>) + target_link_options(${PROJECT_NAME} PUBLIC -Xsycl-target-backend=${HYPRE_SYCL_TARGET_BACKEND}) + elseif(INTEL_SYCL_BACKEND) + target_compile_options(${PROJECT_NAME} PUBLIC $<$:-Xsycl-target-backend=${INTEL_SYCL_BACKEND}>) + target_link_options(${PROJECT_NAME} PUBLIC -Xsycl-target-backend=${INTEL_SYCL_BACKEND}) + endif() +endif() + +# Find Intel DPCT +if(NOT DEFINED DPCTROOT) + if(DEFINED ENV{DPCTROOT}) + set(DPCTROOT $ENV{DPCTROOT}) + elseif(DEFINED ENV{DPCT_BUNDLE_ROOT}) + set(DPCTROOT $ENV{DPCT_BUNDLE_ROOT}) + elseif(DEFINED ENV{ONEAPI_ROOT} AND EXISTS "$ENV{ONEAPI_ROOT}/dpcpp-ct/latest") + set(DPCTROOT "$ENV{ONEAPI_ROOT}/dpcpp-ct/latest") + endif() +endif() + +# Check if DPCT is found +if(NOT EXISTS "${DPCTROOT}/include/dpct/dpct.hpp") + message(FATAL_ERROR "Could not find DPCT installation. Please set DPCTROOT") +endif() + +# Add DPCT include directory +target_include_directories(${PROJECT_NAME} PUBLIC "${DPCTROOT}/include") +message(STATUS "DPCT include directory: ${DPCTROOT}/include") + +if (HYPRE_ENABLE_ONEMKLSPARSE) + set(HYPRE_USING_ONEMKLSPARSE ON CACHE BOOL "" FORCE) +endif() + +if (HYPRE_ENABLE_ONEMKLBLAS) + set(HYPRE_USING_ONEMKLBLAS ON CACHE BOOL "" FORCE) +endif() + +if (HYPRE_ENABLE_ONEMKLRAND) + set(HYPRE_USING_ONEMKLRAND ON CACHE BOOL "" FORCE) +endif() + +# Setup MKL +if (HYPRE_USING_ONEMKLSPARSE OR HYPRE_USING_ONEMKLBLAS OR HYPRE_USING_ONEMKLRAND) + # Find MKL + set(MKL_LINK dynamic) + set(MKL_THREADING sequential) + #set(ENABLE_TRY_SYCL_COMPILE ON) # This option slows down the build + find_package(MKL CONFIG REQUIRED HINTS "$ENV{MKLROOT}/lib/cmake/mkl") + + # Add all required MKL components explicitly + target_link_libraries(${PROJECT_NAME} + PUBLIC + $ + $ + $ + $ + $ + $ + $ + ) + + # Ensure compile options and include directories are properly propagated + target_compile_options(${PROJECT_NAME} PUBLIC + $ + ) + target_include_directories(${PROJECT_NAME} PUBLIC + $ + ) +endif() diff --git a/src/config/configure.in b/src/config/configure.in index 26e53b9fc6..ffc5f8cbfe 100644 --- a/src/config/configure.in +++ b/src/config/configure.in @@ -399,7 +399,7 @@ AS_HELP_STRING([--enable-hopscotch], ) if test "$hypre_using_hopscotch" = "yes" then - AC_DEFINE(HYPRE_HOPSCOTCH, 1, [Define to 1 if hopscotch hashing]) + AC_DEFINE(HYPRE_USING_HOPSCOTCH, 1, [Define to 1 if using hopscotch hashing]) fi AC_ARG_ENABLE(fortran, @@ -2865,7 +2865,7 @@ fi if test "$hypre_gpu_mpi" = "yes" then - AC_DEFINE([HYPRE_WITH_GPU_AWARE_MPI],1,[Define to 1 if using GPU aware MPI]) + AC_DEFINE([HYPRE_USING_GPU_AWARE_MPI],1,[Define to 1 if using GPU aware MPI]) fi CUFLAGS="${CUFLAGS} ${EXTRA_CUFLAGS}" diff --git a/src/configure b/src/configure index c3cff232b0..fba5d370d8 100755 --- a/src/configure +++ b/src/configure @@ -3671,7 +3671,7 @@ fi if test "$hypre_using_hopscotch" = "yes" then -printf "%s\n" "#define HYPRE_HOPSCOTCH 1" >>confdefs.h +printf "%s\n" "#define HYPRE_USING_HOPSCOTCH 1" >>confdefs.h fi @@ -11120,7 +11120,7 @@ fi if test "$hypre_gpu_mpi" = "yes" then -printf "%s\n" "#define HYPRE_WITH_GPU_AWARE_MPI 1" >>confdefs.h +printf "%s\n" "#define HYPRE_USING_GPU_AWARE_MPI 1" >>confdefs.h fi diff --git a/src/docs/usr-manual/ch-intro.rst b/src/docs/usr-manual/ch-intro.rst index cb55a1b4bb..d14568fc75 100644 --- a/src/docs/usr-manual/ch-intro.rst +++ b/src/docs/usr-manual/ch-intro.rst @@ -122,19 +122,24 @@ How to get started Installing hypre ------------------------------------------------------------------------------ -As previously noted, on most systems hypre can be built by simply typing -``configure`` followed by ``make`` in the top-level source directory. -Alternatively, the CMake system [CMakeWeb]_ can be used, and is the best -approach for building hypre on Windows systems in particular. For more detailed -instructions, read the ``INSTALL`` file provided with the hypre distribution or -the :ref:`ch-General` section of this manual. Note the following requirements: +As previously noted, on most systems hypre can be built by typing +``./configure`` followed by ``make`` in the top-level source directory. +Alternatively, `CMake `_ can be used and is the recommended +method for building hypre on Windows. For more detailed instructions, refer to +the `INSTALL `_ file +provided in the hypre distribution or see the :ref:`ch-General` section of this manual. -* To run in parallel, hypre requires an installation of MPI. +.. note:: -* Configuration of hypre with threads requires an implementation of OpenMP. - Currently, only a subset of hypre is threaded. + * To run in parallel, hypre requires an installation of MPI. -* The hypre library currently does not directly support complex-valued systems. + * Configuration of hypre with threads requires an implementation of OpenMP. + Currently, only a subset of hypre is threaded. + + * To run on GPUs, hypre must be configured with support for the appropriate GPU toolkit. + Available options are CUDA for NVIDIA GPUs, HIP for AMD GPUs, and SYCL for Intel GPUs. + + * The hypre library currently does not directly support complex-valued systems. .. _choosing-interface: @@ -148,12 +153,12 @@ represent the way that applications developers naturally think of their linear problem and to provide natural interfaces for them to pass the data that defines their linear system into hypre. Essentially, these conceptual interfaces can be considered convenient utilities for helping a user build a matrix data structure -for hypre solvers and preconditioners. The top row of :ref:`fig-ls-interface` +for hypre solvers and preconditioners. The top row of :numref:`fig-ls-interface` illustrates a number of conceptual interfaces. Generally, the conceptual interfaces are denoted by different types of computational grids, but other application features might also be used, such as geometrical information. For example, applications that use structured grids (such as in the left-most -interface in :ref:`fig-ls-interface`) typically view their linear problems in +interface in :numref:`fig-ls-interface`) typically view their linear problems in terms of stencils and grids. On the other hand, applications that use unstructured grids and finite elements typically view their linear problems in terms of elements and element stiffness matrices. Finally, the right-most @@ -195,36 +200,31 @@ application. for building matrices in particular formats, find the IJ interface relatively easy to use. See Chapter :ref:`ch-IJ` for details. -.. _fig-ls-interface: - -.. figure:: figConcepIface.* +.. figure:: figConcepIface.svg :align: center + :name: fig-ls-interface - Figure 1 - - Graphic illustrating the notion of conceptual linear system interfaces. + : Graphic illustrating the notion of conceptual linear system interfaces. Generally, a user should choose the most specific interface that matches their -application, because this will allow them to use specialized and more efficient -solvers and preconditioners without losing access to more general solvers. For -example, the second row of Figure :ref:`fig-ls-interface` is a set of linear -solver algorithms. Each linear solver group requires different information from -the user through the conceptual interfaces. So, the geometric multigrid -algorithm (GMG) listed in the left-most box, for example, can only be used with -the left-most conceptual interface. On the other hand, the ILU algorithm in the -right-most box may be used with any conceptual interface. Matrix requirements -for each solver and preconditioner are provided in Chapter :ref:`ch-Solvers` and -in Chapter :ref:`ch-API`. Your desired solver strategy may influence your -choice of conceptual interface. A typical user will select a single Krylov -method and a single preconditioner to solve their system. - -The third row of Figure :ref:`fig-ls-interface` is a list of data layouts or -matrix/vector storage schemes. The relationship between linear solver and -storage scheme is similar to that of the conceptual interface and linear solver. -Note that some of the interfaces in hypre currently only support one -matrix/vector storage scheme choice. The conceptual interface, the desired -solvers and preconditioners, and the matrix storage class must all be -compatible. +application, because this will allow them to use specialized and more efficient solvers +and preconditioners without losing access to more general solvers. For example, the +second row of :numref:`fig-ls-interface` is a set of linear solver algorithms. Each linear +solver group requires different information from the user through the conceptual +interfaces. So, the geometric multigrid algorithm (GMG) listed in the left-most box, for +example, can only be used with the left-most conceptual interface. On the other hand, the +ILU algorithm in the right-most box may be used with any conceptual interface. Matrix +requirements for each solver and preconditioner are provided in Chapter :ref:`ch-Solvers` +and in Chapter :ref:`ch-API`. Your desired solver strategy may influence your choice of +conceptual interface. A typical user will select a single Krylov method and a single +preconditioner to solve their system. + +The third row of :numref:`fig-ls-interface` is a list of data layouts or matrix/vector +storage schemes. The relationship between linear solver and storage scheme is similar to +that of the conceptual interface and linear solver. Note that some of the interfaces in +hypre currently only support one matrix/vector storage scheme choice. The conceptual +interface, the desired solvers and preconditioners, and the matrix storage class must all +be compatible. .. _writing-your-code: @@ -270,4 +270,3 @@ fully understand the function of each conceptual interface and each solver. Remember that a comprehensive list of all available functions is provided in Chapter :ref:`ch-API`, and the provided example codes may prove helpful as templates for your specific application. - diff --git a/src/docs/usr-manual/ch-misc.rst b/src/docs/usr-manual/ch-misc.rst index c79362c85c..2cfec7601a 100644 --- a/src/docs/usr-manual/ch-misc.rst +++ b/src/docs/usr-manual/ch-misc.rst @@ -10,42 +10,60 @@ General Information ****************************************************************************** +In this and the following sections, we discuss how to obtain and build hypre, +interpret error flags, report bugs, and call hypre from different programming languages. +We provide instructions for two build systems: autotools (configure & make) and CMake. +While autotools is traditionally used on Unix-like systems (Linux, macOS, etc.), +CMake provides cross-platform support and is required for Windows builds. +Both systems are actively maintained and supported. + +.. _getting-source: Getting the Source Code ============================================================================== -The most recent hypre distribution is available at -`https://github.com/hypre-space/hypre/tags`_ along with previous distribution -versions. +There are two ways to obtain hypre: + +1. **Clone from GitHub (recommended)**:: + + git clone https://github.com/hypre-space/hypre.git + cd hypre/src + +2. **Download a Release** + + Download the latest release from our `GitHub releases page `_. + After extracting the archive, you'll find the source code in the ``src`` directory:: + + tar -xvf hypre-x.y.z.tar.gz + cd hypre-x.y.z/src + + where ``x.y.z`` represents the version number (e.g., 2.29.0). Building the Library ============================================================================== -In this and the following several sections, we discuss the steps to install and -use hypre. First, we focus on the primary method targeting Unix-like operating -systems, such as Linux, AIX, and Mac OS X. Then in `CMake instructions`_, we -explain an alternative approach using the CMake build system [CMakeWeb]_, which -is the best approach for building hypre on Windows systems in particular. +After obtaining the source code (see :ref:`getting-source`), there are three main ways to build hypre: -After unpacking the hypre tar file, the source code will be in the ``src`` -sub-directory of a directory named hypre-VERSION, where VERSION is the current -version number (e.g., hypre-2.29.0). +1. Using autotools (Configure & Make) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Move to the ``src`` sub-directory to build hypre for the host platform. The -simplest method is to configure, compile and install the libraries in -``./hypre/lib`` and ``./hypre/include`` directories, which is accomplished by: +The simplest method is to use the traditional configure and make: .. code-block:: bash - ./configure - make + cd ${HYPRE_HOME}/src # Move to the source directory + ./configure # Configure the build system + make -j 4 # Use threads for a faster parallel build + make install # (Optional) Install hypre on a user-specified path via --prefix= -NOTE: when executing on an IBM platform ``configure`` must be executed under the -nopoe script (``./nopoe ./configure