SYCL in homme #6569
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| #module restore | ||
|
There was a problem hiding this comment. I'm curious: what does "-aot" stand for? There was a problem hiding this comment. |
||
| #module load oneapi/eng-compiler/2022.12.30.005 | ||
| #module load intel_compute_runtime/release/agama-devel-627 | ||
| #module load spack cmake | ||
| #module list | ||
|
|
||
|
|
||
| SET (SUNSPOT_MACHINE TRUE CACHE BOOL "") | ||
|
|
||
| SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") | ||
| SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") | ||
|
|
||
| SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") | ||
|
|
||
| SET(WITH_PNETCDF FALSE CACHE FILEPATH "") | ||
|
|
||
| SET(USE_QUEUING FALSE CACHE BOOL "") | ||
|
|
||
| #temp hack | ||
| SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") | ||
|
|
||
| SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") | ||
| SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") | ||
|
|
||
| #set(KOKKOS_HOME "/home/onguba/kokkos-build/mar05-aot/install" CACHE STRING "") | ||
| #set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") | ||
|
|
||
| SET(USE_TRILINOS OFF CACHE BOOL "") | ||
|
|
||
| SET(SYCL_BUILD TRUE CACHE BOOL "") | ||
| SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") | ||
|
|
||
| SET(CMAKE_CXX_STANDARD 17) | ||
|
|
||
| SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") | ||
| SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") | ||
| SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") | ||
|
|
||
| # -fsycl-link-huge-device-code for theta to get build | ||
| #JIT flags | ||
| #SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") | ||
| #SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") | ||
|
|
||
| #AOT flags | ||
| SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") | ||
| SET(SYCL_LINK_FLAGS "-fsycl -fsycl-device-code-split=per_kernel -fsycl-link-huge-device-code -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") | ||
|
|
||
| SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") | ||
| SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") | ||
|
|
||
| SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") | ||
| SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") | ||
|
|
||
| set (ENABLE_OPENMP OFF CACHE BOOL "") | ||
| set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") | ||
| set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") | ||
|
|
||
| set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") | ||
|
|
||
| set (USE_NUM_PROCS 4 CACHE STRING "") | ||
|
|
||
| SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| #module restore | ||
|
There was a problem hiding this comment. And "-jit"? I doubt it is "Just In Time"? There was a problem hiding this comment. |
||
| #module load oneapi/eng-compiler/2022.12.30.005 | ||
| #module load intel_compute_runtime/release/agama-devel-627 | ||
| #module load spack cmake | ||
| #module list | ||
|
|
||
|
|
||
|
|
||
| SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") | ||
| SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") | ||
|
|
||
| SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") | ||
|
|
||
| SET(WITH_PNETCDF FALSE CACHE FILEPATH "") | ||
|
|
||
| SET(USE_QUEUING FALSE CACHE BOOL "") | ||
|
|
||
| #temp hack | ||
| SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") | ||
|
|
||
| SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") | ||
| SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") | ||
|
|
||
| #set(KOKKOS_HOME "/home/onguba/kokkos-build/jan03-2024/install" CACHE STRING "") | ||
| #set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") | ||
|
|
||
| SET(USE_TRILINOS OFF CACHE BOOL "") | ||
|
|
||
| SET(SYCL_BUILD TRUE CACHE BOOL "") | ||
| SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") | ||
|
|
||
| SET(CMAKE_CXX_STANDARD 17) | ||
|
|
||
| SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") | ||
| SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") | ||
| SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") | ||
|
|
||
| # -fsycl-link-huge-device-code for theta to get build | ||
| SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") | ||
| SET(SYCL_LINK_FLAGS "-fsycl -fsycl-link-huge-device-code -fsycl-device-code-split=per_kernel -fsycl-targets=spir64") | ||
|
|
||
| SET(ADD_Fortran_FLAGS "-fc=ifx -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") | ||
| SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") | ||
|
|
||
| SET(ADD_CXX_FLAGS "-std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") | ||
| SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") | ||
|
|
||
| set (ENABLE_OPENMP OFF CACHE BOOL "") | ||
| set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") | ||
| set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") | ||
|
|
||
| set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") | ||
|
|
||
| set (USE_NUM_PROCS 4 CACHE STRING "") | ||
|
|
||
| SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") | ||
|
|
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| #Currently Loaded Modules: | ||
| # 1) craype-x86-rome 6) craype/2.7.15 11) cray-libpals/1.1.7 16) nvhpc-mixed/21.9 | ||
| # 2) libfabric/1.11.0.4.125 7) cray-dsmml/0.2.2 12) PrgEnv-gnu/8.3.3 17) cudatoolkit-standalone/11.6.2 | ||
| # 3) craype-network-ofi 8) cray-pmi/6.1.2 13) gnu-parallel/2021-09-22 18) cmake/3.23.2 | ||
| # 4) perftools-base/22.05.0 9) cray-pmi-lib/6.0.17 14) gcc/11.2.0 | ||
| # 5) craype-accel-nvidia80 10) cray-pals/1.1.7 15) cray-mpich/8.1.16 | ||
|
|
||
|
|
||
|
|
||
| #SET(HOMMEXX_EXEC_SPACE CUDA CACHE STRING "") | ||
| #SET(HOMMEXX_MPI_ON_DEVICE FALSE CACHE BOOL "") | ||
| #SET(HOMMEXX_CUDA_MAX_WARP_PER_TEAM "16" CACHE STRING "") | ||
|
|
||
| # cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.6 | ||
| #SET(NETCDF_DIR $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") | ||
| #SET(PNETCDF_DIR $ENV{CRAY_PARALLEL_NETCDF_DIR} CACHE FILEPATH "") | ||
| #SET(HDF5_DIR $ENV{CRAY_HDF5_PARALLEL_PREFIX} CACHE FILEPATH "") | ||
|
|
||
| #for scorpio | ||
| #SET (NetCDF_C_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") | ||
| #SET (NetCDF_Fortran_PATH $ENV{CRAY_NETCDF_HDF5PARALLEL_PREFIX} CACHE FILEPATH "") | ||
|
|
||
| SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") | ||
|
|
||
| SET(HOMME_FIND_BLASLAPACK FALSE CACHE BOOL "") | ||
|
|
||
| SET(WITH_PNETCDF FALSE CACHE FILEPATH "") | ||
|
|
||
| SET(USE_QUEUING FALSE CACHE BOOL "") | ||
|
|
||
| SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") | ||
|
|
||
| SET(CUDA_BUILD TRUE CACHE BOOL "") | ||
|
|
||
| #SET(HOMMEXX_BFB_TESTING TRUE CACHE BOOL "") | ||
|
|
||
| SET(USE_TRILINOS OFF CACHE BOOL "") | ||
|
|
||
| SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "") | ||
| SET(Kokkos_ENABLE_CUDA ON CACHE BOOL "") | ||
| SET(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "") | ||
| SET(Kokkos_ARCH_AMPERE80 ON CACHE BOOL "") | ||
| #SET(Kokkos_ARCH_ZEN2 ON CACHE BOOL "") # works, and perf same if both AMPERE80 and ZEN2 are on | ||
| #SET(Kokkos_ENABLE_CUDA_UVM ON CACHE BOOL "") | ||
| SET(Kokkos_ENABLE_EXPLICIT_INSTANTIATION OFF CACHE BOOL "") | ||
| #SET(Kokkos_ENABLE_CUDA_ARCH_LINKING OFF CACHE BOOL "") | ||
|
|
||
| #SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") | ||
| #SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") | ||
| #SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") | ||
| SET(CMAKE_C_COMPILER "cc" CACHE STRING "") | ||
| SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") | ||
| SET(CMAKE_CXX_COMPILER "CC" CACHE STRING "") | ||
|
|
||
| #SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") | ||
| #SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") | ||
| #SET(CMAKE_CXX_COMPILER "${CMAKE_CURRENT_SOURCE_DIR}/../../externals/kokkos/bin/nvcc_wrapper" CACHE STRING "") | ||
|
|
||
| # Note: need to set MPICH_CXX env variable and perhaps NVCC_WRAPPER_DEFAULT_COMPILER | ||
|
|
||
| SET(CXXLIB_SUPPORTED_CACHE FALSE CACHE BOOL "") | ||
|
|
||
| SET(ENABLE_OPENMP OFF CACHE BOOL "") | ||
| SET(ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") | ||
| SET(ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") | ||
|
|
||
| SET(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "") | ||
|
|
||
| #SET(HOMME_TESTING_PROFILE "dev" CACHE STRING "") | ||
|
|
||
| SET(USE_NUM_PROCS 4 CACHE STRING "") | ||
|
|
||
| SET(USE_MPIEXEC "srun" CACHE STRING "") | ||
| #SET(CPRNC_DIR /global/cfs/cdirs/e3sm/tools/cprnc CACHE FILEPATH "") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| #module restore | ||
| #module load oneapi/eng-compiler/2022.12.30.005 | ||
| #module load intel_compute_runtime/release/agama-devel-627 | ||
| #module load spack cmake | ||
| #module list | ||
|
|
||
| SET (SUNSPOT_MACHINE TRUE CACHE BOOL "") | ||
|
|
||
| SET (HOMMEXX_MPI_ON_DEVICE TRUE CACHE BOOL "") | ||
|
|
||
| #SET(BUILD_HOMME_WITHOUT_PIOLIBRARY TRUE CACHE BOOL "") | ||
|
|
||
| SET(HOMME_FIND_BLASLAPACK TRUE CACHE BOOL "") | ||
|
|
||
| SET(WITH_PNETCDF FALSE CACHE FILEPATH "") | ||
|
|
||
| SET(USE_QUEUING FALSE CACHE BOOL "") | ||
|
|
||
| #temp hack | ||
| SET(HOMME_USE_KOKKOS TRUE CACHE BOOL "") | ||
|
|
||
| SET(BUILD_HOMME_PREQX_KOKKOS TRUE CACHE BOOL "") | ||
| SET(BUILD_HOMME_THETA_KOKKOS TRUE CACHE BOOL "") | ||
|
|
||
| #set(KOKKOS_HOME "/home/onguba/kokkos-build/june22-2024-aot/install" CACHE STRING "") | ||
| #set(E3SM_KOKKOS_PATH ${KOKKOS_HOME} CACHE STRING "") | ||
|
|
||
| SET (NetCDF_Fortran_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") | ||
| SET (NetCDF_C_PATH "/lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf" CACHE STRING "") | ||
|
|
||
| SET(USE_TRILINOS OFF CACHE BOOL "") | ||
|
|
||
| SET(SYCL_BUILD TRUE CACHE BOOL "") | ||
| SET(HOMME_ENABLE_COMPOSE FALSE CACHE BOOL "") | ||
|
|
||
| #SET(CMAKE_CXX_STANDARD 17) | ||
| SET(CMAKE_CXX_STANDARD 17 CACHE STRING "CXX Standard") | ||
|
|
||
| SET(CMAKE_C_COMPILER "mpicc" CACHE STRING "") | ||
| SET(CMAKE_Fortran_COMPILER "mpifort" CACHE STRING "") | ||
| SET(CMAKE_CXX_COMPILER "mpicxx" CACHE STRING "") | ||
|
|
||
| SET(SYCL_COMPILE_FLAGS "-std=c++17 -fsycl -fsycl-device-code-split=per_kernel -fno-sycl-id-queries-fit-in-int -fsycl-unnamed-lambda") | ||
| SET(SYCL_LINK_FLAGS "-fsycl-max-parallel-link-jobs=32 -fsycl-link-huge-device-code -fsycl -fsycl-device-code-split=per_kernel -fsycl-targets=spir64_gen -Xsycl-target-backend \"-device 12.60.7\"") | ||
|
|
||
| #-fpscomp does not actually solve the issue with bools in here,another suggestion was -fp-model=precise, not working either | ||
| SET(ADD_Fortran_FLAGS " -fc=ifx -fpscomp logicals -O3 -DNDEBUG -DCPRINTEL -g" CACHE STRING "") | ||
| SET(ADD_C_FLAGS "-O3 -DNDEBUG " CACHE STRING "") | ||
|
|
||
| SET(ADD_CXX_FLAGS " -std=c++17 -O3 -DNDEBUG ${SYCL_COMPILE_FLAGS}" CACHE STRING "") | ||
| SET(ADD_LINKER_FLAGS "-O3 -DNDEBUG ${SYCL_LINK_FLAGS} -fortlib" CACHE STRING "") | ||
|
|
||
| set (ENABLE_OPENMP OFF CACHE BOOL "") | ||
| set (ENABLE_COLUMN_OPENMP OFF CACHE BOOL "") | ||
| set (ENABLE_HORIZ_OPENMP OFF CACHE BOOL "") | ||
|
|
||
| set (HOMME_TESTING_PROFILE "dev" CACHE STRING "") | ||
|
|
||
| set (USE_NUM_PROCS 4 CACHE STRING "") | ||
|
|
||
| SET (USE_MPI_OPTIONS "--bind-to core" CACHE FILEPATH "") | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This puzzles me. The linker language should always be the one of the main entry point (in this case, F90). Why do we need this?