Skip to content

Commit

Permalink
change compile related
Browse files Browse the repository at this point in the history
  • Loading branch information
yhmtsai committed Aug 22, 2023
1 parent 17302d4 commit d2d2de8
Show file tree
Hide file tree
Showing 49 changed files with 153 additions and 152 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/intel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
spack find --loaded
mkdir build
cd build
cmake .. -DCMAKE_INSTALL_PREFIX=install_ginkgo -DGINKGO_COMPILER_FLAGS="-ffp-model=precise" -DCMAKE_CXX_COMPILER=${{ matrix.config.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} -DGINKGO_DPCPP_SINGLE_MODE=ON
cmake .. -DCMAKE_INSTALL_PREFIX=install_ginkgo -DGINKGO_COMPILER_FLAGS="-ffp-model=precise" -DCMAKE_CXX_COMPILER=${{ matrix.config.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.config.build_type }} -DGINKGO_MIXED_PRECISION=${{ matrix.config.mixed }} -DGINKGO_SYCL_SINGLE_MODE=ON
make -j8
ONEAPI_DEVICE_SELECTOR=level_zero:gpu ctest -j10 --output-on-failure
Expand Down
10 changes: 5 additions & 5 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,7 @@ build/dpcpp/igpu/release/shared:
GKO_COMPILER_FLAGS: "-ffp-model=precise"
BUILD_TYPE: "Release"
BUILD_SHARED_LIBS: "ON"
DPCPP_SINGLE_MODE: "ON"
SYCL_SINGLE_MODE: "ON"
ONEAPI_DEVICE_SELECTOR: "*:gpu"

# TODO: Enable when debug shared library size issues are fixed
Expand All @@ -630,7 +630,7 @@ build/dpcpp/igpu/release/shared:
# GKO_COMPILER_FLAGS: "-ffp-model=precise"
# BUILD_TYPE: "Debug"
# BUILD_SHARED_LIBS: "ON"
# DPCPP_SINGLE_MODE: "ON"
# SYCL_SINGLE_MODE: "ON"
# ONEAPI_DEVICE_SELECTOR: "level_zero:gpu"

# It gives two available backends of GPU on tests
Expand All @@ -647,7 +647,7 @@ build/dpcpp/dgpu/release/static:
GKO_COMPILER_FLAGS: "-ffp-model=precise"
BUILD_TYPE: "Release"
BUILD_SHARED_LIBS: "OF"
DPCPP_SINGLE_MODE: "ON"
SYCL_SINGLE_MODE: "ON"
ONEAPI_DEVICE_SELECTOR: "*:gpu"

build/dpcpp/level_zero_dgpu/release/shared:
Expand All @@ -662,7 +662,7 @@ build/dpcpp/level_zero_dgpu/release/shared:
BUILD_SYCL: "ON"
GKO_COMPILER_FLAGS: "-ffp-model=precise"
BUILD_TYPE: "Release"
DPCPP_SINGLE_MODE: "ON"
SYCL_SINGLE_MODE: "ON"
ONEAPI_DEVICE_SELECTOR: "level_zero:gpu"

build/icpx/level_zero_dgpu/release/shared:
Expand All @@ -677,7 +677,7 @@ build/icpx/level_zero_dgpu/release/shared:
BUILD_SYCL: "ON"
GKO_COMPILER_FLAGS: "-ffp-model=precise"
BUILD_TYPE: "Release"
DPCPP_SINGLE_MODE: "ON"
SYCL_SINGLE_MODE: "ON"
ONEAPI_DEVICE_SELECTOR: "level_zero:gpu"

# Job with important warnings as error
Expand Down
4 changes: 2 additions & 2 deletions .gitlab/scripts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@
-DGINKGO_MIXED_PRECISION=${MIXED_PRECISION}
-DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
-DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
-DGINKGO_DPCPP_SINGLE_MODE=${DPCPP_SINGLE_MODE}
-DGINKGO_SYCL_SINGLE_MODE=${SYCL_SINGLE_MODE}
-DGINKGO_EXPORT_BUILD_DIR=${EXPORT_BUILD_DIR}
- ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
- awk '!/^#/ { print ($2 - $1)/1000 " " $4 }' .ninja_log | sort -nr
Expand Down Expand Up @@ -93,7 +93,7 @@
-DGINKGO_FAST_TESTS=${FAST_TESTS}
-DGINKGO_MIXED_PRECISION=${MIXED_PRECISION}
-DGINKGO_CONFIG_LOG_DETAILED=${CONFIG_LOG}
-DGINKGO_DPCPP_SINGLE_MODE=${DPCPP_SINGLE_MODE}
-DGINKGO_SYCL_SINGLE_MODE=${SYCL_SINGLE_MODE}
-DGINKGO_RUN_EXAMPLES=${RUN_EXAMPLES}
-DGINKGO_EXPORT_BUILD_DIR=${EXPORT_BUILD_DIR}
- ninja -j${NUM_CORES} -l${CI_LOAD_LIMIT} install
Expand Down
2 changes: 1 addition & 1 deletion .gitlab/variables.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
MPI_AS_ROOT: "OFF"
FAST_TESTS: "OFF"
NONDEFAULT_STREAM: "OFF"
DPCPP_SINGLE_MODE: "OFF"
SYCL_SINGLE_MODE: "OFF"
MIXED_PRECISION: "ON"
RUN_EXAMPLES: "OFF"
CONFIG_LOG: "ON"
Expand Down
10 changes: 6 additions & 4 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,8 @@ else()
option(GINKGO_BUILD_HWLOC "Build Ginkgo with HWLOC. Enabled if a system installation is found." ${HWLOC_FOUND})
endif()
option(GINKGO_BUILD_PAPI_SDE "Build Ginkgo with PAPI SDE. Enabled if a system installation is found." ${PAPI_SDE_FOUND})
option(GINKGO_DPCPP_SINGLE_MODE "Do not compile double kernels for the DPC++ backend." OFF)
gko_rename_cache(GINKGO_DPCPP_SINGLE_MODE GINKGO_SYCL_SINGLE_MODE BOOL)
option(GINKGO_SYCL_SINGLE_MODE "Do not compile double kernels for the DPC++ backend." OFF)
option(GINKGO_INSTALL_RPATH "Set the RPATH when installing its libraries." ON)
option(GINKGO_INSTALL_RPATH_ORIGIN "Add $ORIGIN (Linux) or @loader_path (MacOS) to the installation RPATH." ON)
option(GINKGO_INSTALL_RPATH_DEPENDENCIES "Add dependencies to the installation RPATH." OFF)
Expand Down Expand Up @@ -301,10 +302,11 @@ if(MSVC)
endif()

if(GINKGO_BUILD_SYCL)
ginkgo_extract_dpcpp_version(${CMAKE_CXX_COMPILER} GINKGO_DPCPP_MAJOR_VERSION __LIBSYCL_MAJOR_VERSION)
ginkgo_extract_dpcpp_version(${CMAKE_CXX_COMPILER} GINKGO_DPCPP_VERSION __SYCL_COMPILER_VERSION)
# It may only make sense for intel sycl
ginkgo_extract_dpcpp_version(${CMAKE_CXX_COMPILER} GINKGO_SYCL_MAJOR_VERSION __LIBSYCL_MAJOR_VERSION)
ginkgo_extract_dpcpp_version(${CMAKE_CXX_COMPILER} GINKGO_SYCL_VERSION __SYCL_COMPILER_VERSION)
else()
set(GINKGO_DPCPP_MAJOR_VERSION "0")
set(GINKGO_SYCL_MAJOR_VERSION "0")
endif()
configure_file(${Ginkgo_SOURCE_DIR}/include/ginkgo/config.hpp.in
${Ginkgo_BINARY_DIR}/include/ginkgo/config.hpp @ONLY)
Expand Down
6 changes: 3 additions & 3 deletions benchmark/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ function(ginkgo_add_single_benchmark_executable name use_lib_linops macro_def ty
target_link_libraries("${name}" hip_timer)
endif()
if (GINKGO_BUILD_SYCL)
target_compile_definitions("${name}" PRIVATE HAS_DPCPP_TIMER=1)
target_compile_definitions("${name}" PRIVATE HAS_SYCL_TIMER=1)
target_link_libraries("${name}" dpcpp_timer)
endif()
if (GINKGO_BUILD_MPI)
Expand All @@ -88,7 +88,7 @@ function(ginkgo_add_single_benchmark_executable name use_lib_linops macro_def ty
target_link_libraries("${name}" hipsparse_linops_${type})
endif()
if (GINKGO_BUILD_SYCL)
target_compile_definitions("${name}" PRIVATE HAS_DPCPP=1)
target_compile_definitions("${name}" PRIVATE HAS_SYCL=1)
target_link_libraries("${name}" onemkl_linops_${type})
endif()
endif()
Expand Down Expand Up @@ -140,7 +140,7 @@ if (GINKGO_BUILD_SYCL)
ginkgo_benchmark_onemkl_linops(z GKO_BENCHMARK_USE_DOUBLE_COMPLEX_PRECISION)
ginkgo_benchmark_onemkl_linops(c GKO_BENCHMARK_USE_SINGLE_COMPLEX_PRECISION)
add_library(dpcpp_timer utils/dpcpp_timer.dp.cpp)
target_compile_options(dpcpp_timer PRIVATE ${GINKGO_DPCPP_FLAGS})
target_compile_options(dpcpp_timer PRIVATE ${GINKGO_SYCL_FLAGS})
gko_add_sycl_to_target(TARGET dpcpp_timer SOURCES utils/dpcpp_timer.dp.cpp)
target_link_libraries(dpcpp_timer ginkgo)
endif()
Expand Down
12 changes: 6 additions & 6 deletions benchmark/utils/formats.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@ std::string available_format =
", hipsparse_csr, hipsparse_csrmm, hipsparse_coo, hipsparse_ell, "
"hipsparse_hybrid"
#endif // HAS_HIP
#ifdef HAS_DPCPP
#ifdef HAS_SYCL
", onemkl_csr, onemkl_optimized_csr"
#endif // HAS_DPCPP
#endif // HAS_SYCL
".\n";

std::string format_description =
Expand Down Expand Up @@ -129,11 +129,11 @@ std::string format_description =
"hipsparse_ell: hipSPARSE CSR SpMV using hipsparseXhybmv\n"
" with HIPSPARSE_HYB_PARTITION_MAX\n"
#endif // HAS_HIP
#ifdef HAS_DPCPP
#ifdef HAS_SYCL
"onemkl_csr: oneMKL Csr SpMV\n"
"onemkl_optimized_csr: oneMKL optimized Csr SpMV using optimize_gemv after "
"reading the matrix"
#endif // HAS_DPCPP
#endif // HAS_SYCL
;

std::string format_command =
Expand Down Expand Up @@ -262,10 +262,10 @@ const std::map<std::string, std::function<std::unique_ptr<gko::LinOp>(
{"hipsparse_coo", create_sparselib_linop<hipsparse_coo>},
{"hipsparse_ell", create_sparselib_linop<hipsparse_ell>},
#endif // HAS_HIP
#ifdef HAS_DPCPP
#ifdef HAS_SYCL
{"onemkl_csr", create_sparselib_linop<onemkl_csr>},
{"onemkl_optimized_csr", create_sparselib_linop<onemkl_optimized_csr>},
#endif // HAS_DPCPP
#endif // HAS_SYCL
{"hybrid", create_matrix_type<hybrid>()},
{"hybrid0",create_matrix_type<hybrid>( std::make_shared<hybrid::imbalance_limit>(0))},
{"hybrid25",create_matrix_type<hybrid>( std::make_shared<hybrid::imbalance_limit>(0.25))},
Expand Down
8 changes: 4 additions & 4 deletions benchmark/utils/timer_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -236,10 +236,10 @@ std::shared_ptr<Timer> get_hip_timer(
#endif // HAS_HIP_TIMER


#ifdef HAS_DPCPP_TIMER
#ifdef HAS_SYCL_TIMER
std::shared_ptr<Timer> get_sycl_timer(
std::shared_ptr<const gko::SyclExecutor> exec);
#endif // HAS_DPCPP_TIMER
#endif // HAS_SYCL_TIMER


#if HAS_MPI_TIMER
Expand Down Expand Up @@ -277,12 +277,12 @@ inline std::shared_ptr<Timer> get_timer(
}
#endif // HAS_HIP_TIMER

#ifdef HAS_DPCPP_TIMER
#ifdef HAS_SYCL_TIMER
if (auto sycl =
std::dynamic_pointer_cast<const gko::SyclExecutor>(exec)) {
return get_sycl_timer(sycl);
}
#endif // HAS_DPCPP_TIMER
#endif // HAS_SYCL_TIMER
}
// No cuda/hip/sycl executor available or no gpu_timer used
return std::make_shared<CpuTimer>(exec);
Expand Down
7 changes: 4 additions & 3 deletions cmake/GinkgoConfig.cmake.in
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,10 @@ set(GINKGO_HIP_AMDGPU "@GINKGO_HIP_AMDGPU@")
set(GINKGO_HIP_VERSION @GINKGO_HIP_VERSION@)
set(GINKGO_AMD_ARCH_FLAGS "@GINKGO_AMD_ARCH_FLAGS@")

set(GINKGO_DPCPP_VERSION @GINKGO_DPCPP_VERSION@)
set(GINKGO_DPCPP_MAJOR_VERSION @GINKGO_DPCPP_MAJOR_VERSION@)
set(GINKGO_DPCPP_FLAGS "@GINKGO_DPCPP_FLAGS@")
// TODO: are these considered as public interface?
set(GINKGO_SYCL_VERSION @GINKGO_SYCL_VERSION@)
set(GINKGO_SYCL_MAJOR_VERSION @GINKGO_SYCL_MAJOR_VERSION@)
set(GINKGO_SYCL_FLAGS "@GINKGO_SYCL_FLAGS@")
set(GINKGO_MKL_ROOT "@GINKGO_MKL_ROOT@")
set(GINKGO_DPL_ROOT "@GINKGO_DPL_ROOT@")

Expand Down
36 changes: 16 additions & 20 deletions cmake/create_test.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ function(ginkgo_set_test_target_properties test_target_name)
if (GINKGO_TEST_NONDEFAULT_STREAM)
target_compile_definitions(${test_target_name} PRIVATE GKO_TEST_NONDEFAULT_STREAM)
endif()
if (GINKGO_COMPILING_DPCPP_TEST AND GINKGO_DPCPP_SINGLE_MODE)
target_compile_definitions(${test_target_name} PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
if (GINKGO_COMPILING_SYCL_TEST AND GINKGO_SYCL_SINGLE_MODE)
target_compile_definitions(${test_target_name} PRIVATE GINKGO_SYCL_SINGLE_MODE=1)
endif()
if (GINKGO_CHECK_CIRCULAR_DEPS)
target_link_libraries(${test_target_name} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
Expand Down Expand Up @@ -92,12 +92,12 @@ function(ginkgo_create_test test_name)
ginkgo_add_test(${test_name} ${test_target_name} ${ARGN})
endfunction(ginkgo_create_test)

## Test compiled with dpcpp
function(ginkgo_create_dpcpp_test test_name)
## Test compiled with sycl
function(ginkgo_create_sycl_test test_name)
ginkgo_build_test_name(${test_name} test_target_name)
add_executable(${test_target_name} ${test_name}.dp.cpp)
target_compile_features(${test_target_name} PUBLIC cxx_std_17)
target_compile_options(${test_target_name} PRIVATE ${GINKGO_DPCPP_FLAGS})
target_compile_options(${test_target_name} PRIVATE ${GINKGO_SYCL_FLAGS})
gko_add_sycl_to_target(TARGET ${test_target_name} SOURCES ${test_name}.dp.cpp)
target_link_options(${test_target_name} PRIVATE -fsycl-device-code-split=per_kernel)
ginkgo_set_test_target_properties(${test_target_name} ${ARGN})
Expand All @@ -106,7 +106,7 @@ function(ginkgo_create_dpcpp_test test_name)
if (MKL_ENV)
set_tests_properties(${test_target_name} PROPERTIES ENVIRONMENT "${MKL_ENV}")
endif()
endfunction(ginkgo_create_dpcpp_test)
endfunction(ginkgo_create_sycl_test)

## Test compiled with CUDA
function(ginkgo_create_cuda_test test_name)
Expand Down Expand Up @@ -206,7 +206,7 @@ function(ginkgo_create_common_test test_name)
ginkgo_create_common_test_internal(${test_name} CudaExecutor cuda ${ARGN})
endif()
if(GINKGO_BUILD_SYCL)
ginkgo_create_common_test_internal(${test_name} SyclExecutor dpcpp ${ARGN})
ginkgo_create_common_test_internal(${test_name} SyclExecutor sycl ${ARGN})
endif()
endfunction(ginkgo_create_common_test)

Expand All @@ -220,16 +220,12 @@ function(ginkgo_create_common_test_internal test_name exec_type exec)
# set up actual test
set(test_target_name ${test_target_name}_${exec})
add_executable(${test_target_name} ${test_name}.cpp)
if(exec STREQUAL "dpcpp")
target_compile_definitions(${test_target_name} PRIVATE EXEC_TYPE=${exec_type} EXEC_NAMESPACE=sycl GKO_COMPILING_${exec_upper})
else()
target_compile_definitions(${test_target_name} PRIVATE EXEC_TYPE=${exec_type} EXEC_NAMESPACE=${exec} GKO_COMPILING_${exec_upper})
endif()
target_compile_definitions(${test_target_name} PRIVATE EXEC_TYPE=${exec_type} EXEC_NAMESPACE=${exec} GKO_COMPILING_${exec_upper})
target_link_libraries(${test_target_name} PRIVATE ${common_test_ADDITIONAL_LIBRARIES})
# use float for DPC++ if necessary
if((exec STREQUAL "dpcpp") AND GINKGO_DPCPP_SINGLE_MODE)
# use float for SYCL if necessary
if((exec STREQUAL "sycl") AND GINKGO_SYCL_SINGLE_MODE)
target_compile_definitions(${test_target_name} PRIVATE GINKGO_COMMON_SINGLE_MODE=1)
target_compile_definitions(${test_target_name} PRIVATE GINKGO_DPCPP_SINGLE_MODE=1)
target_compile_definitions(${test_target_name} PRIVATE GINKGO_SYCL_SINGLE_MODE=1)
endif()
ginkgo_set_test_target_properties(${test_target_name} ${ARGN})
ginkgo_add_test(${test_name}_${exec} ${test_target_name} ${ARGN})
Expand All @@ -240,13 +236,13 @@ function(ginkgo_create_common_device_test test_name)
cmake_parse_arguments(PARSE_ARGV 1 common_device_test "" "${gko_test_single_args}" "${gko_test_multi_args}")
ginkgo_build_test_name(${test_name} test_target_name)
if(GINKGO_BUILD_SYCL)
ginkgo_create_common_test_internal(${test_name} SyclExecutor dpcpp ${ARGN})
target_compile_features(${test_target_name}_dpcpp PRIVATE cxx_std_17)
target_compile_options(${test_target_name}_dpcpp PRIVATE ${GINKGO_DPCPP_FLAGS})
ginkgo_create_common_test_internal(${test_name} SyclExecutor sycl ${ARGN})
target_compile_features(${test_target_name}_sycl PRIVATE cxx_std_17)
target_compile_options(${test_target_name}_sycl PRIVATE ${GINKGO_SYCL_FLAGS})
# We need to use a new file to avoid sycl setting in other backends because add_sycl_to_target will change the source property.
configure_file(${test_name}.cpp ${test_name}.dp.cpp COPYONLY)
gko_add_sycl_to_target(TARGET ${test_target_name}_dpcpp SOURCES ${test_name}.dp.cpp)
target_link_options(${test_target_name}_dpcpp PRIVATE -fsycl-device-lib=all -fsycl-device-code-split=per_kernel)
gko_add_sycl_to_target(TARGET ${test_target_name}_sycl SOURCES ${test_name}.dp.cpp)
target_link_options(${test_target_name}_sycl PRIVATE -fsycl-device-lib=all -fsycl-device-code-split=per_kernel)
endif()
if(GINKGO_BUILD_OMP)
ginkgo_create_common_test_internal(${test_name} OmpExecutor omp ${ARGN})
Expand Down
4 changes: 2 additions & 2 deletions common/unified/base/kernel_launch.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ GKO_INLINE GKO_ATTRIBUTES constexpr unpack_member_type<T> unpack_member(T value)
} // namespace gko


#elif defined(GKO_COMPILING_DPCPP)
#elif defined(GKO_COMPILING_SYCL)

#define GKO_DEVICE_NAMESPACE sycl
#define GKO_KERNEL
Expand Down Expand Up @@ -302,7 +302,7 @@ typename to_device_type_impl<T>::type map_to_device(T&& param)
#include "cuda/base/kernel_launch.cuh"
#elif defined(GKO_COMPILING_HIP)
#include "hip/base/kernel_launch.hip.hpp"
#elif defined(GKO_COMPILING_DPCPP)
#elif defined(GKO_COMPILING_SYCL)
#include "dpcpp/base/kernel_launch.dp.hpp"
#elif defined(GKO_COMPILING_OMP)
#include "omp/base/kernel_launch.hpp"
Expand Down
2 changes: 1 addition & 1 deletion common/unified/base/kernel_launch_reduction.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "cuda/base/kernel_launch_reduction.cuh"
#elif defined(GKO_COMPILING_HIP)
#include "hip/base/kernel_launch_reduction.hip.hpp"
#elif defined(GKO_COMPILING_DPCPP)
#elif defined(GKO_COMPILING_SYCL)
#include "dpcpp/base/kernel_launch_reduction.dp.hpp"
#elif defined(GKO_COMPILING_OMP)
#include "omp/base/kernel_launch_reduction.hpp"
Expand Down
2 changes: 1 addition & 1 deletion common/unified/base/kernel_launch_solver.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ const device_type<ValueType>* row_vector(const matrix::Dense<ValueType>* mtx)
#include "cuda/base/kernel_launch_solver.cuh"
#elif defined(GKO_COMPILING_HIP)
#include "hip/base/kernel_launch_solver.hip.hpp"
#elif defined(GKO_COMPILING_DPCPP)
#elif defined(GKO_COMPILING_SYCL)
#include "dpcpp/base/kernel_launch_solver.dp.hpp"
#elif defined(GKO_COMPILING_OMP)
#include "omp/base/kernel_launch_solver.hpp"
Expand Down
Loading

0 comments on commit d2d2de8

Please sign in to comment.