Skip to content

Commit

Permalink
Merge pull request #10 from eth-cscs/features/pdgemm_wrapper
Browse files Browse the repository at this point in the history
Added ScaLAPACK wrappers, i.e. COSMA backend for matrices given in ScaLAPACK data layout.

Observe that this is less efficient than using cosma::multiply which would directly use the optimal COSMA layout, because here we pay the price of transforming matrices between scalapack and COSMA data layout, but is convenient as it provides exactly the same signatures as ScaLAPACK, which is widely used.
  • Loading branch information
Marko Kabic authored Jul 19, 2019
2 parents c721bc8 + 370fc91 commit 1126c73
Show file tree
Hide file tree
Showing 35 changed files with 1,553 additions and 203 deletions.
9 changes: 9 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,12 @@
[submodule "libs/gtest_mpi"]
path = libs/gtest_mpi
url = https://github.com/teonnik/gtest_mpi.git
[submodule "libs/grid2grid"]
path = libs/grid2grid
url = https://github.com/kabicm/grid2grid
[submodule "libs/semiprof"]
path = libs/semiprof
url = https://github.com/kabicm/semiprof
[submodule "libs/options"]
path = libs/options
url = https://github.com/kabicm/options
182 changes: 114 additions & 68 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,59 +19,105 @@ if(NOT CMAKE_BUILD_TYPE)
"Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release." FORCE)
endif()

include(git_utils)

# COSMA library
#
add_library(cosma STATIC
"${cosma_SOURCE_DIR}/src/cosma/blas.cpp"
"${cosma_SOURCE_DIR}/src/cosma/buffer.cpp"
"${cosma_SOURCE_DIR}/src/cosma/communicator.cpp"
"${cosma_SOURCE_DIR}/src/cosma/context.cpp"
"${cosma_SOURCE_DIR}/src/cosma/interval.cpp"
"${cosma_SOURCE_DIR}/src/cosma/layout.cpp"
"${cosma_SOURCE_DIR}/src/cosma/local_multiply.cpp"
"${cosma_SOURCE_DIR}/src/cosma/mapper.cpp"
"${cosma_SOURCE_DIR}/src/cosma/math_utils.cpp"
"${cosma_SOURCE_DIR}/src/cosma/matrix.cpp"
"${cosma_SOURCE_DIR}/src/cosma/multiply.cpp"
"${cosma_SOURCE_DIR}/src/cosma/one_sided_communicator.cpp"
"${cosma_SOURCE_DIR}/src/cosma/strategy.cpp"
"${cosma_SOURCE_DIR}/src/cosma/two_sided_communicator.cpp"
set(cosma_src_prefix "${cosma_SOURCE_DIR}/src/cosma")

set(cosma_src_files
"${cosma_src_prefix}/blas.cpp"
"${cosma_src_prefix}/buffer.cpp"
"${cosma_src_prefix}/communicator.cpp"
"${cosma_src_prefix}/context.cpp"
"${cosma_src_prefix}/interval.cpp"
"${cosma_src_prefix}/layout.cpp"
"${cosma_src_prefix}/local_multiply.cpp"
"${cosma_src_prefix}/mapper.cpp"
"${cosma_src_prefix}/math_utils.cpp"
"${cosma_src_prefix}/matrix.cpp"
"${cosma_src_prefix}/multiply.cpp"
"${cosma_src_prefix}/one_sided_communicator.cpp"
"${cosma_src_prefix}/strategy.cpp"
"${cosma_src_prefix}/two_sided_communicator.cpp"
)
add_library(cosma STATIC ${cosma_src_files})

target_include_directories(cosma PUBLIC
$<BUILD_INTERFACE:${cosma_SOURCE_DIR}/src>
)
$<BUILD_INTERFACE:${cosma_SOURCE_DIR}/src>
)
target_compile_features(cosma PUBLIC cxx_std_14)

# Non-cmake dependencies
#
find_package(grid2grid REQUIRED)
find_package(semiprof REQUIRED)
find_package(options REQUIRED)
find_package(MPI REQUIRED)
find_package(MKL REQUIRED)
target_link_libraries(cosma
PUBLIC
PUBLIC
MPI::MPI_CXX
PRIVATE
PRIVATE
MKL::MKL
grid2grid::grid2grid
semiprof::semiprof
options::options
)
)
target_compile_definitions(cosma PUBLIC COSMA_WITH_MKL)

set(cosma_submodules_prefix "${PROJECT_SOURCE_DIR}/libs")

# grid2grid dependency
find_package(ScaLAPACK)
if(ScaLAPACK_FOUND)
# TODO
add_library(cosma_with_scalapack STATIC
${cosma_src_prefix}/scalapack.cpp
${cosma_src_prefix}/pgemm.cpp
)

target_link_libraries(cosma_with_scalapack PUBLIC cosma MKL::ScaLAPACK)

check_git_submodule(grid2grid "${cosma_submodules_prefix}/grid2grid")
if(NOT grid2grid_avail)
find_package(grid2grid REQUIRED)
target_link_libraries(cosma PUBLIC grid2grid::grid2grid)
else()
add_subdirectory("${cosma_submodules_prefix}/grid2grid")
target_link_libraries(cosma PUBLIC grid2grid)
endif()
endif()

# options dependency
check_git_submodule(options "${cosma_submodules_prefix}/options")
if(NOT options_avail)
find_package(options REQUIRED)
target_link_libraries(cosma PRIVATE options::options)
else()
add_subdirectory("${cosma_submodules_prefix}/options")
target_link_libraries(cosma PRIVATE options)
# target_include_directories(cosma PUBLIC
# $<BUILD_INTERFACE:${cosma_submodules_prefix}/options/src>
# )
endif()

# semiprof dependency
check_git_submodule(semiprof "${cosma_submodules_prefix}/semiprof")
if(NOT semiprof_avail)
find_package(semiprof REQUIRED)
target_link_libraries(cosma PRIVATE semiprof::semiprof)
else()
add_subdirectory("${cosma_submodules_prefix}/semiprof")
target_link_libraries(cosma PRIVATE semiprof)
# target_include_directories(cosma PUBLIC
# $<BUILD_INTERFACE:${cosma_submodules_prefix}/semiprof/include/semiprof>
# )
endif()

# CUDA suppor
# CUDA support
#
if(COSMA_WITH_CUDA)
enable_language(CUDA)
target_compile_definitions(cosma PRIVATE -DCOSMA_HAVE_GPU)
target_link_libraries(cosma PRIVATE cublas Tiled-MM)
target_link_libraries(cosma PUBLIC cublas Tiled-MM)
target_compile_options(cosma PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:"-arch=sm_60">)
# target_include_directories(cosma PUBLIC
# $<BUILD_INTERFACE:${cosma_submodules_prefix}/Tiled-MM>
# )
endif()

# PROFILER
Expand All @@ -82,45 +128,45 @@ endif()

# Installation
#
if (COSMA_NOT_SUBPROJECT)
include(CMakePackageConfigHelpers)
include(GNUInstallDirs)

install(TARGETS cosma
EXPORT cosma_targets
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")

install(EXPORT cosma_targets
FILE cosmaTargets.cmake
NAMESPACE cosma::
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/cosma")

install(DIRECTORY "${cosma_SOURCE_DIR}/src/cosma"
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
FILES_MATCHING PATTERN "*.hpp")

write_basic_package_version_file(
"${cosma_BINARY_DIR}/cosmaConfigVersion.cmake"
VERSION ${cosma_VERSION}
COMPATIBILITY SameMajorVersion)

configure_file("${cosma_SOURCE_DIR}/cmake/cosma.pc.in"
"${cosma_BINARY_DIR}/cosma.pc"
@ONLY)

configure_file("${cosma_SOURCE_DIR}/cmake/cosmaConfig.cmake.in"
"${cosma_BINARY_DIR}/cosmaConfig.cmake"
@ONLY)

install(FILES "${cosma_BINARY_DIR}/cosmaConfig.cmake"
"${cosma_BINARY_DIR}/cosmaConfigVersion.cmake"
"${cosma_SOURCE_DIR}/cmake/FindMKL.cmake"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/cosma")
install(FILES "${cosma_BINARY_DIR}/cosma.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
if (COSMA_NOT_SUBPROJECT AND NOT grid2grid_avail)
include(CMakePackageConfigHelpers)
include(GNUInstallDirs)

install(TARGETS cosma
EXPORT cosma_targets
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")

install(EXPORT cosma_targets
FILE cosmaTargets.cmake
NAMESPACE cosma::
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/cosma")

install(DIRECTORY "${cosma_SOURCE_DIR}/src/cosma"
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
FILES_MATCHING PATTERN "*.hpp")

write_basic_package_version_file(
"${cosma_BINARY_DIR}/cosmaConfigVersion.cmake"
VERSION ${cosma_VERSION}
COMPATIBILITY SameMajorVersion)

configure_file("${cosma_SOURCE_DIR}/cmake/cosma.pc.in"
"${cosma_BINARY_DIR}/cosma.pc"
@ONLY)

configure_file("${cosma_SOURCE_DIR}/cmake/cosmaConfig.cmake.in"
"${cosma_BINARY_DIR}/cosmaConfig.cmake"
@ONLY)

install(FILES "${cosma_BINARY_DIR}/cosmaConfig.cmake"
"${cosma_BINARY_DIR}/cosmaConfigVersion.cmake"
"${cosma_SOURCE_DIR}/cmake/FindMKL.cmake"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/cosma")

install(FILES "${cosma_BINARY_DIR}/cosma.pc"
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
endif()

# Testing
Expand Down
68 changes: 58 additions & 10 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,28 +45,53 @@ uses `dgemm` for the local computations, but also has a support for the `GPU`
acceleration through our `Tiled-MM` library using `cublas`


## Building
## Building and Compiling the Library

The project uses submodules, to clone do :
The project uses submodules, to clone do (don't forget the recursive flag!):

```bash
git clone --recursive https://github.com/eth-cscs/COSMA.git
```

CMake is used to build the project. Required dependencies are `MPI`, `MKL`,
`grid2grid`, `options` and `semiprof`. The last three dependencies can be
installed using `scripts/install_dependencies.py`.
The easiest way to build the project is to do the following within the project folder:
```bash
# make a folder for building
mkdir build
cd build

# building the library
# specify MKL threading and MPI library to be used
# MKL threading to be used:
# - "GOMP": for GNU OpenMP or
# - "IOMP": for Intel OpenMP
# MPI to be used:
# - "MPICH": for MPICH
# - "OMPI": for OpenMPI
# specify which MPI library is used:
cmake -DMKL_THREADING="GOMP" -DMKL_MPI_TYPE="MPICH" ..

# compiling the library
make -j 4
```
This will automatically download the required dependencies as submodules.

Once the library is installed, you can try running some examples with the provided script:
```bash
sbatch schedule_miniapp_on_daint.sh
```
The script will use SLURM to submit a job on 64 nodes. The job will run 2 matrix multiplications and output the time COSMA algorithm took.

If the GPU backend is used, then `TiledMM` is required.
## Advanced Users

For unit tests, a required dependency is `gtest_mpi` (a git submodule).
Required dependencies are `MPI`, `OpenMP`, `MKL`, `grid2grid`, `options` and `semiprof`. If GPU backend is used, then one more dependency is `Tiled-MM` for the local multiplication of matrices. For unit tests, a required dependency is `gtest_mpi` (a git submodule).

For advanced users, who do not want to use submodules, but would rather like to install the library and its dependencies independently, the following python script can be used: `scripts/install_dependencies.py`.

The following script provide information on important build variables and
options: `scripts/build.sh`. The script can be used as a template to build COSMA
for your system.


## Testing
### Testing

To build all test targets:

Expand All @@ -83,7 +108,7 @@ ctest
Note: `COSMA_IS_OPENMPI=ON` has to be set if OpenMPI is used.


## Installing
### Installing

To install do `make install`.

Expand Down Expand Up @@ -141,6 +166,29 @@ The flags have the following meaning:
other. This flag therefore determines whether the topology is
communication-aware.

### COSMA PDGEMM Wrapper

COSMA also contains a wrapper for ScaLAPACK `pxgemm` calls which offers scalapack interface (pxgemm functions with exactly the same signatures as ScaLAPACK). Running these functions will take care of transforming the matrices between ScaLAPACK and COSMA data layout, perform the multiplication using COSMA algorithm and transform the result back to the specified ScaLAPACK data layout.

The miniapp consists of an executable `./build/miniapp/pdgemm-miniapp` which can
be run with the following command line on Piz Daint (assuming we are in the root folder of
the project):

```
OMP_NUM_THREADS=18 MKL_NUM_THREADS=18 srun -C mc -N 8 -n 16 ./miniapp/pdgemm-miniapp -m 1000 -n 1000 -k 1000 -p 4 -q 4 -bm 128 -bn 128 -bk 128 --trans_a
```

The flags have the following meaning:

- `-m (--m_dim)`: number of rows of matrices `A` and `C`
- `-n (--n_dim)`: number of columns of matrices `B` and `C`
- `-k (--k_dim)`: number of columns of matrix `A` and rows of matrix `B`
- `-bm (--m_block)` (optional, default 128): block size for dimension m
- `-bn (--n_block)` (optional, default 128): block size for dimension n
- `-bk (--k_block)` (optional, default 128): block size for dimension k
- `-ta (--trans_a)` (optional, default: no transpose): transpose A before mutliplication
- `-tb (--trans_b)` (optional, default: no transpose): transpose B before mutliplication

### Dry-run for statistics

If interested in the communication or computation volume, maximum buffer size or
Expand Down
Loading

0 comments on commit 1126c73

Please sign in to comment.