From 7ddb74c9701907d398571a47e30851894ec0f336 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 8 Apr 2024 12:52:59 -0700 Subject: [PATCH 01/13] changed the Signal.h header file name. --- src/MGmol.cc | 2 +- src/md.cc | 2 +- src/tools/{Signal.h => Signal_mgmol.h} | 4 ++-- src/tools/Timeout.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename src/tools/{Signal.h => Signal_mgmol.h} (94%) diff --git a/src/MGmol.cc b/src/MGmol.cc index 962bdf51..fa1d9e2e 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -106,7 +106,7 @@ extern Timer ions_setupInteractingIons_tm; extern Timer ions_setup_tm; extern Timer updateCenters_tm; -#include "Signal.h" +#include "Signal_mgmol.h" std::set Signal::recv_; template diff --git a/src/md.cc b/src/md.cc index 1adab46b..6b43048e 100644 --- a/src/md.cc +++ b/src/md.cc @@ -31,7 +31,7 @@ #include "ProjectedMatricesMehrstellen.h" #include "ProjectedMatricesSparse.h" #include "Rho.h" -#include "Signal.h" +#include "Signal_mgmol.h" #include "SpreadsAndCenters.h" #include "tools.h" diff --git a/src/tools/Signal.h b/src/tools/Signal_mgmol.h similarity index 94% rename from src/tools/Signal.h rename to src/tools/Signal_mgmol.h index 8e8d0a21..245a1ca2 100644 --- a/src/tools/Signal.h +++ b/src/tools/Signal_mgmol.h @@ -7,7 +7,7 @@ // This file is part of MGmol. For details, see https://github.com/llnl/mgmol. // Please also read this link https://github.com/llnl/mgmol/LICENSE -// Adapted from Jeep: Signal.h,v 1.5 2002/06/28 20:50:33 +// Adapted from Jeep: Signal_mgmol.h,v 1.5 2002/06/28 20:50:33 // The Signal class is a utility to catch UNIX signals. // A set of flags is maintained to remeber which signals were caught. @@ -15,7 +15,7 @@ // access the flag set, reset flags, or interrogate flags. // The Signal class can be used in an application by declaring // -// #include "Signal.h" +// #include "Signal_mgmol.h" // set Signal::recv_; // // A signal can be registered using, e.g. diff --git a/src/tools/Timeout.h b/src/tools/Timeout.h index 18940a78..db60d719 100644 --- a/src/tools/Timeout.h +++ b/src/tools/Timeout.h @@ -17,7 +17,7 @@ #include #include "MPIdata.h" -#include "Signal.h" +#include "Signal_mgmol.h" #if PCS #include From 9c90675c994288c249bc86eb85003f7e84290169 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 8 Apr 2024 12:53:27 -0700 Subject: [PATCH 02/13] additional routine to find scalapack library, if nothing is found. --- cmake_modules/FindSCALAPACK.cmake | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cmake_modules/FindSCALAPACK.cmake b/cmake_modules/FindSCALAPACK.cmake index dde12c26..e2558603 100644 --- a/cmake_modules/FindSCALAPACK.cmake +++ b/cmake_modules/FindSCALAPACK.cmake @@ -1,3 +1,5 @@ +message(STATUS "Entered FindSCALAPACK.cmake") + set(_SCALAPACK_SEARCHES) # Search SCALAPACK_ROOT first if it is set. @@ -29,6 +31,19 @@ if(NOT SCALAPACK_LIBRARY) endforeach() endif() +# Search for some default library paths +if (NOT SCALAPACK_FOUND) + find_library(SCALAPACK_LIBRARY + NAMES ${SCALAPACK_NAMES} + PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib + /opt/local/lib /opt/sw/lib /sw/lib + ENV LD_LIBRARY_PATH + ENV DYLD_FALLBACK_LIBRARY_PATH + ENV DYLD_LIBRARY_PATH + ENV SCALAPACKDIR + ENV BLACSDIR) +endif() + unset(SCALAPACK_NAMES) mark_as_advanced(SCALAPACK_LIBRARY SCALAPACK_INCLUDE_DIR) From 42117444b8f79e6dfe031d5bb32eaef7a92f07cf Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 8 Apr 2024 12:53:44 -0700 Subject: [PATCH 03/13] ci workflow with test and format --- .github/workflows/ci.yml | 43 +++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a88d7b48..87dbe096 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: pull_request: types: [opened, labeled, synchronize] branches: - - release + - ROMFPMD # push: # branches: # - release @@ -25,17 +25,14 @@ jobs: uses: styfle/cancel-workflow-action@0.11.0 with: access_token: ${{ github.token }} - - name: Set Swap Space - uses: pierotofy/set-swap-space@master - with: - swap-size-gb: 10 - - name: Check out pylibROM + # - name: Set Swap Space + # uses: pierotofy/set-swap-space@master + # with: + # swap-size-gb: 10 + - name: Check out mgmol uses: actions/checkout@v1 with: submodules: 'true' - - name: Git Submodules status - run: | - git submodule status - name: cmake run: | mkdir build @@ -44,4 +41,32 @@ jobs: - name: make run: | cd build && make -j 4 + - name: test + run: | + cd build && mctest --no-compress-output -T Test + code-style: + runs-on: ubuntu-latest + needs: [docker-image] + container: + image: ghcr.io/llnl/mgmol/mgmol_env:latest + options: --user 1001 --privileged + volumes: + - /mnt:/mnt + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.11.0 + with: + access_token: ${{ github.token }} + - name: Check out mgmol + uses: actions/checkout@v1 + with: + submodules: 'true' + - name: cmake + run: | + mkdir build + cd build + cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON + - name: make + run: | + cd build && make format From c82d387763eed792fe45320c224a3f56ee940e3c Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 8 Apr 2024 14:17:37 -0700 Subject: [PATCH 04/13] changed branch name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87dbe096..c65b7f81 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: pull_request: types: [opened, labeled, synchronize] branches: - - ROMFPMD + - 'ROMFPMD' # push: # branches: # - release @@ -43,7 +43,7 @@ jobs: cd build && make -j 4 - name: test run: | - cd build && mctest --no-compress-output -T Test + cd build && ctest --no-compress-output -T Test code-style: runs-on: ubuntu-latest needs: [docker-image] From b0c86354bd2ffa235c18ffb93287edd071e454e7 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 8 Apr 2024 14:41:57 -0700 Subject: [PATCH 05/13] ctest verbose --- .github/workflows/ci.yml | 52 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c65b7f81..14ee270c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,30 +43,30 @@ jobs: cd build && make -j 4 - name: test run: | - cd build && ctest --no-compress-output -T Test - code-style: - runs-on: ubuntu-latest - needs: [docker-image] - container: - image: ghcr.io/llnl/mgmol/mgmol_env:latest - options: --user 1001 --privileged - volumes: - - /mnt:/mnt - steps: - - name: Cancel previous runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - access_token: ${{ github.token }} - - name: Check out mgmol - uses: actions/checkout@v1 - with: - submodules: 'true' - - name: cmake - run: | - mkdir build - cd build - cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON - - name: make - run: | - cd build && make format + cd build && ctest --no-compress-output -V -T Test + # code-style: + # runs-on: ubuntu-latest + # needs: [docker-image] + # container: + # image: ghcr.io/llnl/mgmol/mgmol_env:latest + # options: --user 1001 --privileged + # volumes: + # - /mnt:/mnt + # steps: + # - name: Cancel previous runs + # uses: styfle/cancel-workflow-action@0.11.0 + # with: + # access_token: ${{ github.token }} + # - name: Check out mgmol + # uses: actions/checkout@v1 + # with: + # submodules: 'true' + # - name: cmake + # run: | + # mkdir build + # cd build + # cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON + # - name: make + # run: | + # cd build && make format From 35f9a099a08a10d7087ad0ea09aef1e20f301506 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 8 Apr 2024 14:52:06 -0700 Subject: [PATCH 06/13] added --oversubscribe to cmake MPIEXEC_PREFLAGS variable. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14ee270c..bb228f79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: run: | mkdir build cd build - cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 + cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" - name: make run: | cd build && make -j 4 From 468d20906a5afab274069a1be9dff869d27b2edc Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 8 Apr 2024 15:29:39 -0700 Subject: [PATCH 07/13] parallel hdf5 package fix. also librom is included. --- docker/Dockerfile | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4ccd50ac..a097df9b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,23 +10,35 @@ WORKDIR /$ENVDIR # install packages RUN sudo apt-get install -yq git RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake -RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpich-dev +RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev RUN sudo apt-get install -yq libboost-all-dev RUN sudo apt-get install -yq vim RUN sudo apt-get install -yq git-lfs RUN sudo apt-get install -yq valgrind RUN sudo apt-get install -yq wget -RUN sudo apt-get install -yq astyle +### clang-format seems to be updated to 14.0. Not using it for now. +# RUN sudo apt-get install -yq clang-format # install lldb and gdb for debugging RUN sudo apt-get install -yq lldb gdb RUN sudo apt-get clean -q -# NOTE: currently docker does not have a way to set environment variable with a command output. -# The following environment variable should use $(uname -m) for different architecture. -# ENV SCALAPACK_ROOT=/usr/lib/aarch64-linux-gnu/ -ENV SCALAPACK_ROOT=/usr/lib/x86_64-linux-gnu/ +ENV LIB_DIR=/$ENVDIR/dependencies +WORKDIR $LIB_DIR + +# cmake toolchain file for librom +RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\ +set(CMAKE_CXX_COMPILER mpicxx)\n\ +set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake +ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake + +# install libROM for scaleupROM +RUN sudo git clone https://github.com/LLNL/libROM.git +WORKDIR ./libROM/build +# libROM without MFEM. +RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF +RUN sudo make -j 16 # create and switch to a user ENV USERNAME=test From eff6bd1a3d5f648cf7977dc78f61d03252cf542c Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 8 Apr 2024 15:29:58 -0700 Subject: [PATCH 08/13] ci workflow only checks the first 20 tests. --- .github/workflows/ci.yml | 2 +- scripts/build_ubuntu22_openmpi.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb228f79..05942636 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: cd build && make -j 4 - name: test run: | - cd build && ctest --no-compress-output -V -T Test + cd build && ctest --no-compress-output -V -T Test -I 1,20,1 # code-style: # runs-on: ubuntu-latest # needs: [docker-image] diff --git a/scripts/build_ubuntu22_openmpi.sh b/scripts/build_ubuntu22_openmpi.sh index 5825759c..421d16b4 100755 --- a/scripts/build_ubuntu22_openmpi.sh +++ b/scripts/build_ubuntu22_openmpi.sh @@ -23,7 +23,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DMPIEXEC_PREFLAGS="-report-bindings;--map-by;core;-bind-to;core" \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ - -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ + -DCMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ .. # call make install From abedba905bad3a742dcd0a9cf1798732c9a8d926 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 8 Apr 2024 15:42:06 -0700 Subject: [PATCH 09/13] minor fix --- cmake_modules/FindSCALAPACK.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmake_modules/FindSCALAPACK.cmake b/cmake_modules/FindSCALAPACK.cmake index e2558603..2aa422af 100644 --- a/cmake_modules/FindSCALAPACK.cmake +++ b/cmake_modules/FindSCALAPACK.cmake @@ -1,5 +1,3 @@ -message(STATUS "Entered FindSCALAPACK.cmake") - set(_SCALAPACK_SEARCHES) # Search SCALAPACK_ROOT first if it is set. From 61490208f3e982f90b5251e615bdd72e23181ef3 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 8 Apr 2024 16:53:26 -0700 Subject: [PATCH 10/13] rom-fpmd driver and librom dependency in cmake. librom must be compiled before mgmol cmake. --- CMakeLists.txt | 20 ++----- cmake_modules/FindlibROM.cmake | 11 ++++ scripts/build_quartz_libROM.sh | 3 + src/CMakeLists.txt | 21 +++++++ src/rom_main.cc | 102 +++++++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 16 deletions(-) create mode 100644 cmake_modules/FindlibROM.cmake create mode 100644 src/rom_main.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index fe1fdd01..5b55890b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,24 +121,12 @@ endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) set(USE_LIBROM False CACHE BOOL "Build with libROM") set(LIBROM_PATH "" CACHE STRING "Path of libROM") if(USE_LIBROM) + message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") if(NOT LIBROM_PATH) - message(FATAL_ERROR "libROM PATH not specified.") - else(NOT LIBROM_PATH) - set(LIBROM_SCRIPTS_PATH ${LIBROM_PATH}/scripts) - message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") - message(STATUS "LIBROM_SCRIPTS_PATH: ${LIBROM_SCRIPTS_PATH}") - include(ExternalProject) - ExternalProject_Add( - libROM - SOURCE_DIR ${LIBROM_SCRIPTS_PATH} - CONFIGURE_COMMAND "" - BINARY_DIR ${LIBROM_PATH} - BUILD_COMMAND ${LIBROM_SCRIPTS_PATH}/compile.sh -t ${LIBROM_PATH}/cmake/toolchains/simple.cmake - INSTALL_COMMAND "" - ) - include_directories(${LIBROM_PATH}/lib) - link_directories(${LIBROM_PATH}/build/lib) + message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.") endif(NOT LIBROM_PATH) + + find_package(libROM REQUIRED) endif(USE_LIBROM) # ARPACK (optional) diff --git a/cmake_modules/FindlibROM.cmake b/cmake_modules/FindlibROM.cmake new file mode 100644 index 00000000..9fed8fb6 --- /dev/null +++ b/cmake_modules/FindlibROM.cmake @@ -0,0 +1,11 @@ +if(NOT LIBROM_PATH) + message(FATAL_ERROR "LIBROM_PATH not specified.") +endif(NOT LIBROM_PATH) + +find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib") +find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib") + +mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES) \ No newline at end of file diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh index dc2b689b..20353bda 100644 --- a/scripts/build_quartz_libROM.sh +++ b/scripts/build_quartz_libROM.sh @@ -31,6 +31,9 @@ cd ${BUILD_DIR} set USE_LIBROM="On" set LIBROM_PATH = ${BUILD_DIR}/libROM git clone https://github.com/LLNL/libROM +cd libROM +./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake +cd ${BUILD_DIR} # call cmake cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3b0207ee..f300aed4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -191,3 +191,24 @@ endif (${MGMOL_WITH_LIBXC}) install(TARGETS mgmol-opt DESTINATION bin) +# build ROM executable +if(USE_LIBROM) + add_executable(mgmol-rom rom_main.cc) + target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS} ${LIBROM_INCLUDES}) + + target_link_libraries(mgmol-rom mgmol_src ${link_libs}) + target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_HL_LIBRARIES}) + target_link_libraries(mgmol-rom ${BLAS_LIBRARIES}) + target_link_libraries(mgmol-rom ${LAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${Boost_LIBRARIES}) + target_link_libraries(mgmol-rom ${LIBROM_LIB}) + if (${OPENMP_CXX_FOUND}) + target_link_libraries(mgmol-rom OpenMP::OpenMP_CXX) + endif() + if(${MGMOL_WITH_LIBXC}) + target_link_libraries(mgmol-rom ${LIBXC_DIR}/lib/libxc.a) + endif (${MGMOL_WITH_LIBXC}) + install(TARGETS mgmol-rom DESTINATION bin) +endif(USE_LIBROM) \ No newline at end of file diff --git a/src/rom_main.cc b/src/rom_main.cc new file mode 100644 index 00000000..c7ec278e --- /dev/null +++ b/src/rom_main.cc @@ -0,0 +1,102 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// +// main.cc +// +// Description: +// Real grid, finite difference, molecular dynamics program +// for with nonorthogonal localized orbitals. +// +// Uses Mehrstellen operators, multigrid accelerations, and +// non-local pseudopotentials. +// +// Includes LDA and PBE exchange and correlation functionals. +// +// Units: +// Potentials, eigenvalues and operators in Rydberg +// Energies in Hartree +// +#include +#include +#include +#include +using namespace std; + +#ifdef _OPENMP +#include +#endif + +#ifdef USE_CNR +#include +#endif + +#include + +#include "Control.h" +#include "DistMatrix.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "MatricesBlacsContext.h" +#include "Mesh.h" +#include "PackedCommunicationBuffer.h" +#include "ReplicatedWorkSpace.h" +#include "SparseDistMatrix.h" +#include "magma_singleton.h" +#include "tools.h" + +#include +#include +#include + +#include +namespace po = boost::program_options; + +#include "librom.h" + +//#include "MemTrack.h" + +int main(int argc, char** argv) +{ + // change handling of memory allocation errors + set_new_handler(noMoreMemory); + + cout.sync_with_stdio(); + + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + cerr << "MPI Initialization failed!!!" << endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + MPI_Comm_rank(MPI_COMM_WORLD, &mype); + assert(mype > -1); + onpe0 = (mype == 0); + + CAROM::Vector librom_vector(10, false); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + cerr << "MPI Finalize failed!!!" << endl; + } + + time_t tt; + time(&tt); + if (onpe0) cout << " Run ended at " << ctime(&tt) << endl; + + // MemTrack::TrackDumpBlocks(); + + // MemTrack::TrackListMemoryUsage(); + + return 0; +} From 6965fa86beb029b2e7be9fde361d32143b91d5bb Mon Sep 17 00:00:00 2001 From: Seung Whan Chung Date: Mon, 8 Apr 2024 18:14:08 -0700 Subject: [PATCH 11/13] cmake fix and toolchain file for LC quartz. --- cmake_modules/FindSCALAPACK.cmake | 16 ++++++++-------- cmake_toolchains/quartz.default.cmake | 6 ++++++ 2 files changed, 14 insertions(+), 8 deletions(-) create mode 100644 cmake_toolchains/quartz.default.cmake diff --git a/cmake_modules/FindSCALAPACK.cmake b/cmake_modules/FindSCALAPACK.cmake index 2aa422af..29282a1d 100644 --- a/cmake_modules/FindSCALAPACK.cmake +++ b/cmake_modules/FindSCALAPACK.cmake @@ -7,7 +7,7 @@ if(DEFINED ENV{SCALAPACK_ROOT}) endif(DEFINED ENV{SCALAPACK_ROOT}) if(SCALAPACK_ROOT) - set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT}) + set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT} ${SCALAPACK_ROOT}/lib/intel64) list(APPEND _SCALAPACK_SEARCHES ${_SCALAPACK_SEARCH_DIR}) endif(SCALAPACK_ROOT) @@ -29,6 +29,13 @@ if(NOT SCALAPACK_LIBRARY) endforeach() endif() +unset(SCALAPACK_NAMES) + +mark_as_advanced(SCALAPACK_LIBRARY SCALAPACK_INCLUDE_DIR) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY) + # Search for some default library paths if (NOT SCALAPACK_FOUND) find_library(SCALAPACK_LIBRARY @@ -42,13 +49,6 @@ if (NOT SCALAPACK_FOUND) ENV BLACSDIR) endif() -unset(SCALAPACK_NAMES) - -mark_as_advanced(SCALAPACK_LIBRARY SCALAPACK_INCLUDE_DIR) - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY) - if(SCALAPACK_FOUND) # Only Intel's scalapack requires an include directory if(SCALAPACK_INCLUDE_DIR) diff --git a/cmake_toolchains/quartz.default.cmake b/cmake_toolchains/quartz.default.cmake new file mode 100644 index 00000000..9901dcd6 --- /dev/null +++ b/cmake_toolchains/quartz.default.cmake @@ -0,0 +1,6 @@ +set(CMAKE_C_COMPILER mpicc) +set(CMAKE_CXX_COMPILER mpicxx) +set(CMAKE_Fortran_COMPILER mpif90) + +set(SCALAPACK_ROOT $ENV{MKLROOT}) +set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so) From 35822cd7a328caa28927e01f73c84600244a58c8 Mon Sep 17 00:00:00 2001 From: Kevin Chung Date: Mon, 8 Apr 2024 18:38:23 -0700 Subject: [PATCH 12/13] bug fix on FindSCALAPACK.cmake --- cmake_modules/FindSCALAPACK.cmake | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmake_modules/FindSCALAPACK.cmake b/cmake_modules/FindSCALAPACK.cmake index 29282a1d..74e58e7d 100644 --- a/cmake_modules/FindSCALAPACK.cmake +++ b/cmake_modules/FindSCALAPACK.cmake @@ -29,8 +29,6 @@ if(NOT SCALAPACK_LIBRARY) endforeach() endif() -unset(SCALAPACK_NAMES) - mark_as_advanced(SCALAPACK_LIBRARY SCALAPACK_INCLUDE_DIR) include(FindPackageHandleStandardArgs) @@ -47,8 +45,12 @@ if (NOT SCALAPACK_FOUND) ENV DYLD_LIBRARY_PATH ENV SCALAPACKDIR ENV BLACSDIR) + + FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY) endif() +unset(SCALAPACK_NAMES) + if(SCALAPACK_FOUND) # Only Intel's scalapack requires an include directory if(SCALAPACK_INCLUDE_DIR) From dcb21af1e9c099e985e790ca115237a07ddf51f6 Mon Sep 17 00:00:00 2001 From: Seung Whan Chung Date: Mon, 8 Apr 2024 20:13:29 -0700 Subject: [PATCH 13/13] librom-mgmol installation script for LC quartz. --- scripts/build_quartz_libROM.sh | 68 ++++++++++++++++++---------------- scripts/modules.quartz | 21 +++++++---- 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh index 20353bda..458dd662 100644 --- a/scripts/build_quartz_libROM.sh +++ b/scripts/build_quartz_libROM.sh @@ -1,55 +1,61 @@ -##! /bin/csh -f +#!/bin/bash ## An example script to build on LLNL Peloton systems. ## For now, this script assumes intel/ mkl libraries are being used. -# load some modules +## load some modules source scripts/modules.quartz -# set some environment variables. Set them explicitly or use loaded module path (preferred) -# Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid -# benign cmake warnings -#setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib -setenv SCALAPACK_ROOT ${MKLROOT} -setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 - -# We need to define the cmake blas vendor option here to find the right one. -set BLAS_VENDOR = Intel10_64lp - -# manually set the location of BLACS libraries for scalapack -set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 - -set MGMOL_ROOT = `pwd` - -set INSTALL_DIR = ${MGMOL_ROOT}/install_quartz +## set some environment variables. Set them explicitly or use loaded module path (preferred) +## Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid +## benign cmake warnings +##setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib +#setenv SCALAPACK_ROOT ${MKLROOT} +#setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 +# +## We need to define the cmake blas vendor option here to find the right one. +#set BLAS_VENDOR = Intel10_64lp +# +## manually set the location of BLACS libraries for scalapack +#set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 + +MGMOL_ROOT="$(pwd)" + +INSTALL_DIR=${MGMOL_ROOT}/install_quartz mkdir -p ${INSTALL_DIR} -set BUILD_DIR = ${MGMOL_ROOT}/build_quartz +BUILD_DIR=${MGMOL_ROOT}/build_quartz mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} # clone the libROM GitHub repo in BUILD_DIR -set USE_LIBROM="On" -set LIBROM_PATH = ${BUILD_DIR}/libROM +USE_LIBROM="On" +LIBROM_PATH=${BUILD_DIR}/libROM git clone https://github.com/LLNL/libROM cd libROM -./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake +#./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake +./scripts/compile.sh cd ${BUILD_DIR} # call cmake -cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ - -DCMAKE_CXX_COMPILER=mpic++ \ - -DCMAKE_Fortran_COMPILER=mpif77 \ - -DMPIEXEC_NUMPROC_FLAG="-n" \ - -DBLA_VENDOR=${BLAS_VENDOR} \ - -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ +cmake -DCMAKE_TOOLCHAIN_FILE=${MGMOL_ROOT}/cmake_toolchains/quartz.default.cmake \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DUSE_LIBROM=${USE_LIBROM} \ -DLIBROM_PATH=${LIBROM_PATH} \ - -DCMAKE_BUILD_TYPE=DEBUG \ .. +# -DCMAKE_CXX_COMPILER=mpic++ \ +# -DCMAKE_Fortran_COMPILER=mpif77 \ +# -DMPIEXEC_NUMPROC_FLAG="-n" \ +# -DBLA_VENDOR=${BLAS_VENDOR} \ +# -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ +# -DCMAKE_BUILD_TYPE=DEBUG \ + # call make install -make -j -make install +make -j 16 +### Currently libROM does not have the installation procedure, +### so copying binary file to installation directory will disrupt the relative path to libROM.so, +### causing a run-time error. +#make install # -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ # -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ diff --git a/scripts/modules.quartz b/scripts/modules.quartz index 220e0a31..82433acb 100644 --- a/scripts/modules.quartz +++ b/scripts/modules.quartz @@ -1,12 +1,19 @@ -module load intel/2022.1.0 +### choose either gcc or intel +#module load intel/2022.1.0 +module load gcc/11.2.1 + +module load cmake module load hdf5-parallel/1.14.0 -#module load hdf5-parallel -#module load boost -module load mkl -module load cmake/3.14.5 +module load boost + +### choose either one +module load mkl-interfaces +#module load mkl + module load python -## manually add boost path -setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH + +### manually add boost path +#setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH #setenv MKLROOT $LIBRARY_PATH #setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0