Restart file collection + POD training

.github/workflows/ci.yml

@@ -0,0 +1,72 @@
+name: CI
+on:
+  workflow_dispatch: {}
+  pull_request:
+    types: [opened, labeled, synchronize]
+    branches:
+      - 'ROMFPMD'
+  # push:
+  #   branches:
+  #     - release
+
+jobs:
+  docker-image:
+    uses: ./.github/workflows/docker_image.yml
+  build:
+    runs-on: ubuntu-latest
+    needs: [docker-image]
+    container:
+      image: ghcr.io/llnl/mgmol/mgmol_env:latest
+      options: --user 1001 --privileged
+      volumes:
+        - /mnt:/mnt
+    steps:
+      - name: Cancel previous runs
+        uses: styfle/cancel-workflow-action@0.11.0
+        with:
+          access_token: ${{ github.token }}
+      # - name: Set Swap Space
+      #   uses: pierotofy/set-swap-space@master
+      #   with:
+      #     swap-size-gb: 10
+      - name: Check out mgmol
+        uses: actions/checkout@v1
+        with:
+            submodules: 'true'
+      - name: cmake
+        run: |
+            mkdir build
+            cd build
+            cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe"
+      - name: make
+        run: |
+            cd build && make -j 4
+      - name: test
+        run: |
+            cd build && ctest --no-compress-output -V -T Test -I 1,20,1
+  # code-style:
+  #   runs-on: ubuntu-latest
+  #   needs: [docker-image]
+  #   container:
+  #     image: ghcr.io/llnl/mgmol/mgmol_env:latest
+  #     options: --user 1001 --privileged
+  #     volumes:
+  #       - /mnt:/mnt
+  #   steps:
+  #     - name: Cancel previous runs
+  #       uses: styfle/cancel-workflow-action@0.11.0
+  #       with:
+  #         access_token: ${{ github.token }}
+  #     - name: Check out mgmol
+  #       uses: actions/checkout@v1
+  #       with:
+  #           submodules: 'true'
+  #     - name: cmake
+  #       run: |
+  #           mkdir build
+  #           cd build
+  #           cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON
+  #     - name: make
+  #       run: |
+  #           cd build && make format
+

.github/workflows/docker_image.yml

@@ -0,0 +1,59 @@
+name: docker-image
+on:
+  workflow_call:
+
+env:
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: llnl/mgmol/mgmol_env
+  DOCKERPATH: docker
+
+jobs:
+  docker-ci:
+    runs-on: ubuntu-latest
+    name: "docker env"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: Ana06/get-changed-files@v2.2.0       
+        id: files
+      - name: DockerPATH configuration
+        run: echo "DOCKERPATH=$DOCKERPATH"
+      - name: DockerPATH - check if files in docker path changed
+        if: contains(steps.files.outputs.all,env.DOCKERPATH) || contains(steps.files.outputs.all,'docker_image.yml')
+        run: |
+          echo "CI container needs rebuilding..."
+          echo "CI_NEEDS_REBUILD=true" >> $GITHUB_ENV
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: env.CI_NEEDS_REBUILD
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        if: env.CI_NEEDS_REBUILD
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: type=sha
+          flavor: latest=true
+      - name: Build Container motd
+        if: env.CI_NEEDS_REBUILD
+        run: |
+          echo "#!/bin/bash" > ${{env.DOCKERPATH}}/motd.sh
+          echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh
+          echo "echo mgmol_env/CI Development Container"  >> ${{env.DOCKERPATH}}/motd.sh
+          echo "echo \"Revision: `echo ${GITHUB_SHA} | cut -c1-8`\"" >> ${{env.DOCKERPATH}}/motd.sh
+          echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh
+          chmod 755 ${{env.DOCKERPATH}}/motd.sh
+          cat ${{env.DOCKERPATH}}/motd.sh
+      - name: Docker Image - Build and push
+        if: env.CI_NEEDS_REBUILD
+        uses: docker/build-push-action@v3
+        with:
+          push: true
+          context: ${{ env.DOCKERPATH }}
+          tags: ${{ steps.meta.outputs.tags }}

.gitignore

@@ -0,0 +1 @@
+src/mgmol_config.h

CMakeLists.txt

@@ -117,6 +117,22 @@ if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
   endif(${SCALAPACK_FOUND})  
 endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT)
 
+# libROM (required)
+set(MGMOL_WITH_LIBROM True CACHE BOOL "Build with libROM")
+set(LIBROM_PATH "" CACHE STRING "Path of libROM")
+if(USE_LIBROM)
+  message(STATUS "LIBROM_PATH: ${LIBROM_PATH}")
+  if(NOT LIBROM_PATH)
+    message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.")
+  endif(NOT LIBROM_PATH)
+
+  find_package(libROM REQUIRED)
+
+  if(libROM_FOUND)
+    set(MGMOL_HAS_LIBROM 1)
+  endif(libROM_FOUND)
+endif(USE_LIBROM)
+
 # ARPACK (optional)
 set(MGMOL_WITH_ARPACK FALSE CACHE BOOL "Compile with ARPACK package")
 if(${MGMOL_WITH_ARPACK} OR DEFINED ARPACK_ROOT)
@@ -240,8 +256,10 @@ include_directories("${PROJECT_SOURCE_DIR}/src/sparse_linear_algebra")
 include_directories("${PROJECT_SOURCE_DIR}/src/tools")
 include_directories("${PROJECT_SOURCE_DIR}/src")
 
+include_directories("${LIBROM_PATH}/lib")
+link_libraries(${LIBROM_LIB})
+
 # add subdirectories for source files, tests
 add_subdirectory(src)
 
 add_subdirectory(tests)
-

cmake_modules/FindSCALAPACK.cmake

@@ -7,7 +7,7 @@ if(DEFINED ENV{SCALAPACK_ROOT})
 endif(DEFINED ENV{SCALAPACK_ROOT})
 
 if(SCALAPACK_ROOT)
-  set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT})
+  set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT} ${SCALAPACK_ROOT}/lib/intel64)
   list(APPEND _SCALAPACK_SEARCHES ${_SCALAPACK_SEARCH_DIR})
 endif(SCALAPACK_ROOT)
 
@@ -29,13 +29,28 @@ if(NOT SCALAPACK_LIBRARY)
   endforeach()
 endif()
 
-unset(SCALAPACK_NAMES)
-
 mark_as_advanced(SCALAPACK_LIBRARY SCALAPACK_INCLUDE_DIR)
 
 include(FindPackageHandleStandardArgs)
 FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY)
 
+# Search for some default library paths
+if (NOT SCALAPACK_FOUND)
+  find_library(SCALAPACK_LIBRARY
+    NAMES ${SCALAPACK_NAMES}
+    PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib
+    /opt/local/lib /opt/sw/lib /sw/lib
+    ENV LD_LIBRARY_PATH
+    ENV DYLD_FALLBACK_LIBRARY_PATH
+    ENV DYLD_LIBRARY_PATH
+    ENV SCALAPACKDIR
+    ENV BLACSDIR)
+
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY)
+endif()
+
+unset(SCALAPACK_NAMES)
+
 if(SCALAPACK_FOUND)
   # Only Intel's scalapack requires an include directory
   if(SCALAPACK_INCLUDE_DIR)

cmake_modules/FindlibROM.cmake

@@ -0,0 +1,11 @@
+if(NOT LIBROM_PATH)
+    message(FATAL_ERROR "LIBROM_PATH not specified.")
+endif(NOT LIBROM_PATH)
+
+find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib")
+find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib")
+
+mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES)

cmake_toolchains/quartz.default.cmake

@@ -0,0 +1,6 @@
+set(CMAKE_C_COMPILER mpicc)
+set(CMAKE_CXX_COMPILER mpicxx)
+set(CMAKE_Fortran_COMPILER mpif90)
+
+set(SCALAPACK_ROOT $ENV{MKLROOT})
+set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so)

docker/Dockerfile

@@ -0,0 +1,50 @@
+FROM ubuntu:22.04
+
+ENV ENVDIR=env
+
+# install sudo
+RUN apt-get -yq update && apt-get -yq install sudo
+
+WORKDIR /$ENVDIR
+
+# install packages
+RUN sudo apt-get install -yq git
+RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake
+RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev
+RUN sudo apt-get install -yq libboost-all-dev
+RUN sudo apt-get install -yq vim
+RUN sudo apt-get install -yq git-lfs
+RUN sudo apt-get install -yq valgrind hdf5-tools
+RUN sudo apt-get install -yq wget
+### clang-format seems to be updated to 14.0. Not using it for now.
+# RUN sudo apt-get install -yq clang-format
+
+# install lldb and gdb for debugging
+RUN sudo apt-get install -yq lldb gdb
+
+RUN sudo apt-get clean -q
+
+ENV LIB_DIR=/$ENVDIR/dependencies
+WORKDIR $LIB_DIR
+
+# cmake toolchain file for librom
+RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\
+set(CMAKE_CXX_COMPILER mpicxx)\n\
+set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake
+ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake
+
+# install libROM for scaleupROM
+RUN sudo git clone https://github.com/LLNL/libROM.git
+WORKDIR ./libROM/build
+# libROM without MFEM.
+RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF
+RUN sudo make -j 16
+
+# create and switch to a user
+ENV USERNAME=test
+RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+RUN useradd --no-log-init -u 1001 --create-home --shell /bin/bash $USERNAME
+RUN adduser $USERNAME sudo
+USER $USERNAME
+WORKDIR /home/$USERNAME
+

examples/Carbyne/carbyne.cfg

@@ -0,0 +1,26 @@
+verbosity=2
+xcFunctional=PBE
+FDtype=4th
+[Mesh]
+nx= 96
+ny= 96
+nz= 192
+[Domain]
+ox= -10.
+oy= -10.
+oz= -20.
+lx= 20.
+ly= 20.
+lz= 40.
+[Potentials]
+pseudopotential=pseudo.H_ONCV_PBE_SG15
+pseudopotential=pseudo.C_ONCV_PBE_SG15
+[Run]
+type=QUENCH
+[Quench]
+max_steps=200
+atol=1.e-8
+[Orbitals]
+initial_type=Fourier
+[Restart]
+output_level=4

examples/Carbyne/carbyne.in

@@ -0,0 +1,14 @@
+H00 1  -0.0000   -0.0000   15.2674
+C01 2  -0.0000    0.0000   13.2519
+C02 2  -0.0000    0.0000   10.9495
+C03 2  -0.0000   -0.0000    8.4221
+C04 2   0.0000    0.0000    6.0897
+C05 2  -0.0000    0.0000    3.5892
+C06 2  -0.0000   -0.0000    1.2470
+C07 2   0.0000   -0.0000   -1.2469
+C08 2   0.0000   -0.0000   -3.5891
+C09 2  -0.0000   -0.0000   -6.0897
+C10 2  -0.0000    0.0000   -8.4221
+C11 2   0.0000   -0.0000  -10.9495
+C12 2   0.0000    0.0000  -13.2520
+H13 1   0.0000    0.0000  -15.2675

examples/Carbyne/carbyne.rom.cfg

@@ -0,0 +1,34 @@
+verbosity=2
+xcFunctional=PBE
+FDtype=4th
+[Mesh]
+nx= 96
+ny= 96
+nz= 192
+[Domain]
+ox= -10.
+oy= -10.
+oz= -20.
+lx= 20.
+ly= 20.
+lz= 40.
+[Potentials]
+pseudopotential=pseudo.H_ONCV_PBE_SG15
+pseudopotential=pseudo.C_ONCV_PBE_SG15
+[Run]
+type=QUENCH
+[Quench]
+max_steps=5
+atol=1.e-8
+[Orbitals]
+initial_type=Fourier
+[Restart]
+output_level=4
+input_level=4
+input_filename=snapshot0_000
+
+[ROM.offline]
+restart_filefmt=snapshot0_%03d
+restart_min_idx=0
+restart_max_idx=1
+basis_file=carom