diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..05942636 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,72 @@ +name: CI +on: + workflow_dispatch: {} + pull_request: + types: [opened, labeled, synchronize] + branches: + - 'ROMFPMD' + # push: + # branches: + # - release + +jobs: + docker-image: + uses: ./.github/workflows/docker_image.yml + build: + runs-on: ubuntu-latest + needs: [docker-image] + container: + image: ghcr.io/llnl/mgmol/mgmol_env:latest + options: --user 1001 --privileged + volumes: + - /mnt:/mnt + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.11.0 + with: + access_token: ${{ github.token }} + # - name: Set Swap Space + # uses: pierotofy/set-swap-space@master + # with: + # swap-size-gb: 10 + - name: Check out mgmol + uses: actions/checkout@v1 + with: + submodules: 'true' + - name: cmake + run: | + mkdir build + cd build + cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMPIEXEC_PREFLAGS="--oversubscribe" + - name: make + run: | + cd build && make -j 4 + - name: test + run: | + cd build && ctest --no-compress-output -V -T Test -I 1,20,1 + # code-style: + # runs-on: ubuntu-latest + # needs: [docker-image] + # container: + # image: ghcr.io/llnl/mgmol/mgmol_env:latest + # options: --user 1001 --privileged + # volumes: + # - /mnt:/mnt + # steps: + # - name: Cancel previous runs + # uses: styfle/cancel-workflow-action@0.11.0 + # with: + # access_token: ${{ github.token }} + # - name: Check out mgmol + # uses: actions/checkout@v1 + # with: + # submodules: 'true' + # - name: cmake + # run: | + # mkdir build + # cd build + # cmake .. -DCMAKE_CXX_COMPILER=mpic++ -DCMAKE_Fortran_COMPILER=mpif90 -DMGMOL_WITH_CLANG_FORMAT=ON + # - name: make + # run: | + # cd build && make format + diff --git a/.github/workflows/docker_image.yml b/.github/workflows/docker_image.yml new file mode 100644 index 00000000..90a2447b --- /dev/null +++ b/.github/workflows/docker_image.yml @@ -0,0 +1,59 @@ +name: docker-image +on: + workflow_call: + +env: + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: llnl/mgmol/mgmol_env + DOCKERPATH: docker + +jobs: + docker-ci: + runs-on: ubuntu-latest + name: "docker env" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: Ana06/get-changed-files@v2.2.0 + id: files + - name: DockerPATH configuration + run: echo "DOCKERPATH=$DOCKERPATH" + - name: DockerPATH - check if files in docker path changed + if: contains(steps.files.outputs.all,env.DOCKERPATH) || contains(steps.files.outputs.all,'docker_image.yml') + run: | + echo "CI container needs rebuilding..." + echo "CI_NEEDS_REBUILD=true" >> $GITHUB_ENV + - name: Log into registry ${{ env.REGISTRY }} + if: env.CI_NEEDS_REBUILD + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract metadata (tags, labels) for Docker + id: meta + if: env.CI_NEEDS_REBUILD + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: type=sha + flavor: latest=true + - name: Build Container motd + if: env.CI_NEEDS_REBUILD + run: | + echo "#!/bin/bash" > ${{env.DOCKERPATH}}/motd.sh + echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo mgmol_env/CI Development Container" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo \"Revision: `echo ${GITHUB_SHA} | cut -c1-8`\"" >> ${{env.DOCKERPATH}}/motd.sh + echo "echo --------------------------" >> ${{env.DOCKERPATH}}/motd.sh + chmod 755 ${{env.DOCKERPATH}}/motd.sh + cat ${{env.DOCKERPATH}}/motd.sh + - name: Docker Image - Build and push + if: env.CI_NEEDS_REBUILD + uses: docker/build-push-action@v3 + with: + push: true + context: ${{ env.DOCKERPATH }} + tags: ${{ steps.meta.outputs.tags }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..81c6b154 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +src/mgmol_config.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 25f2e61d..d9d4e96f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,6 +117,22 @@ if (${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) endif(${SCALAPACK_FOUND}) endif(${MGMOL_WITH_SCALAPACK} OR DEFINED SCALAPACK_ROOT) +# libROM (required) +set(MGMOL_WITH_LIBROM True CACHE BOOL "Build with libROM") +set(LIBROM_PATH "" CACHE STRING "Path of libROM") +if(USE_LIBROM) + message(STATUS "LIBROM_PATH: ${LIBROM_PATH}") + if(NOT LIBROM_PATH) + message(FATAL_ERROR "Cmake is asked to use libROM, but LIBROM_PATH not specified.") + endif(NOT LIBROM_PATH) + + find_package(libROM REQUIRED) + + if(libROM_FOUND) + set(MGMOL_HAS_LIBROM 1) + endif(libROM_FOUND) +endif(USE_LIBROM) + # ARPACK (optional) set(MGMOL_WITH_ARPACK FALSE CACHE BOOL "Compile with ARPACK package") if(${MGMOL_WITH_ARPACK} OR DEFINED ARPACK_ROOT) @@ -240,8 +256,10 @@ include_directories("${PROJECT_SOURCE_DIR}/src/sparse_linear_algebra") include_directories("${PROJECT_SOURCE_DIR}/src/tools") include_directories("${PROJECT_SOURCE_DIR}/src") +include_directories("${LIBROM_PATH}/lib") +link_libraries(${LIBROM_LIB}) + # add subdirectories for source files, tests add_subdirectory(src) add_subdirectory(tests) - diff --git a/cmake_modules/FindSCALAPACK.cmake b/cmake_modules/FindSCALAPACK.cmake index dde12c26..74e58e7d 100644 --- a/cmake_modules/FindSCALAPACK.cmake +++ b/cmake_modules/FindSCALAPACK.cmake @@ -7,7 +7,7 @@ if(DEFINED ENV{SCALAPACK_ROOT}) endif(DEFINED ENV{SCALAPACK_ROOT}) if(SCALAPACK_ROOT) - set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT}) + set(_SCALAPACK_SEARCH_DIR ${SCALAPACK_ROOT} ${SCALAPACK_ROOT}/lib/intel64) list(APPEND _SCALAPACK_SEARCHES ${_SCALAPACK_SEARCH_DIR}) endif(SCALAPACK_ROOT) @@ -29,13 +29,28 @@ if(NOT SCALAPACK_LIBRARY) endforeach() endif() -unset(SCALAPACK_NAMES) - mark_as_advanced(SCALAPACK_LIBRARY SCALAPACK_INCLUDE_DIR) include(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY) +# Search for some default library paths +if (NOT SCALAPACK_FOUND) + find_library(SCALAPACK_LIBRARY + NAMES ${SCALAPACK_NAMES} + PATHS /usr/lib64 /usr/lib /usr/local/lib64 /usr/local/lib + /opt/local/lib /opt/sw/lib /sw/lib + ENV LD_LIBRARY_PATH + ENV DYLD_FALLBACK_LIBRARY_PATH + ENV DYLD_LIBRARY_PATH + ENV SCALAPACKDIR + ENV BLACSDIR) + + FIND_PACKAGE_HANDLE_STANDARD_ARGS(SCALAPACK REQUIRED_VARS SCALAPACK_LIBRARY) +endif() + +unset(SCALAPACK_NAMES) + if(SCALAPACK_FOUND) # Only Intel's scalapack requires an include directory if(SCALAPACK_INCLUDE_DIR) diff --git a/cmake_modules/FindlibROM.cmake b/cmake_modules/FindlibROM.cmake new file mode 100644 index 00000000..9fed8fb6 --- /dev/null +++ b/cmake_modules/FindlibROM.cmake @@ -0,0 +1,11 @@ +if(NOT LIBROM_PATH) + message(FATAL_ERROR "LIBROM_PATH not specified.") +endif(NOT LIBROM_PATH) + +find_library(LIBROM_LIB libROM.so HINTS "${LIBROM_PATH}/build/lib") +find_path(LIBROM_INCLUDES librom.h HINTS "${LIBROM_PATH}/lib") + +mark_as_advanced(LIBROM_LIB LIBROM_INCLUDES) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(libROM REQUIRED_VARS LIBROM_LIB LIBROM_INCLUDES) \ No newline at end of file diff --git a/cmake_toolchains/quartz.default.cmake b/cmake_toolchains/quartz.default.cmake new file mode 100644 index 00000000..9901dcd6 --- /dev/null +++ b/cmake_toolchains/quartz.default.cmake @@ -0,0 +1,6 @@ +set(CMAKE_C_COMPILER mpicc) +set(CMAKE_CXX_COMPILER mpicxx) +set(CMAKE_Fortran_COMPILER mpif90) + +set(SCALAPACK_ROOT $ENV{MKLROOT}) +set(SCALAPACK_BLACS_LIBRARY $ENV{MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.so) diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..8b1ffa94 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:22.04 + +ENV ENVDIR=env + +# install sudo +RUN apt-get -yq update && apt-get -yq install sudo + +WORKDIR /$ENVDIR + +# install packages +RUN sudo apt-get install -yq git +RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake +RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev +RUN sudo apt-get install -yq libboost-all-dev +RUN sudo apt-get install -yq vim +RUN sudo apt-get install -yq git-lfs +RUN sudo apt-get install -yq valgrind hdf5-tools +RUN sudo apt-get install -yq wget +### clang-format seems to be updated to 14.0. Not using it for now. +# RUN sudo apt-get install -yq clang-format + +# install lldb and gdb for debugging +RUN sudo apt-get install -yq lldb gdb + +RUN sudo apt-get clean -q + +ENV LIB_DIR=/$ENVDIR/dependencies +WORKDIR $LIB_DIR + +# cmake toolchain file for librom +RUN echo 'set(CMAKE_C_COMPILER mpicc)\n\ +set(CMAKE_CXX_COMPILER mpicxx)\n\ +set(CMAKE_Fortran_COMPILER mpif90)' > ./librom_env.cmake +ENV TOOLCHAIN_FILE=$LIB_DIR/librom_env.cmake + +# install libROM for scaleupROM +RUN sudo git clone https://github.com/LLNL/libROM.git +WORKDIR ./libROM/build +# libROM without MFEM. +RUN sudo cmake .. -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DCMAKE_BUILD_TYPE=Optimized -DUSE_MFEM=OFF +RUN sudo make -j 16 + +# create and switch to a user +ENV USERNAME=test +RUN echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers +RUN useradd --no-log-init -u 1001 --create-home --shell /bin/bash $USERNAME +RUN adduser $USERNAME sudo +USER $USERNAME +WORKDIR /home/$USERNAME + diff --git a/examples/Carbyne/carbyne.cfg b/examples/Carbyne/carbyne.cfg new file mode 100644 index 00000000..f65a01d4 --- /dev/null +++ b/examples/Carbyne/carbyne.cfg @@ -0,0 +1,26 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=200 +atol=1.e-8 +[Orbitals] +initial_type=Fourier +[Restart] +output_level=4 diff --git a/examples/Carbyne/carbyne.in b/examples/Carbyne/carbyne.in new file mode 100644 index 00000000..9f4e975a --- /dev/null +++ b/examples/Carbyne/carbyne.in @@ -0,0 +1,14 @@ +H00 1 -0.0000 -0.0000 15.2674 +C01 2 -0.0000 0.0000 13.2519 +C02 2 -0.0000 0.0000 10.9495 +C03 2 -0.0000 -0.0000 8.4221 +C04 2 0.0000 0.0000 6.0897 +C05 2 -0.0000 0.0000 3.5892 +C06 2 -0.0000 -0.0000 1.2470 +C07 2 0.0000 -0.0000 -1.2469 +C08 2 0.0000 -0.0000 -3.5891 +C09 2 -0.0000 -0.0000 -6.0897 +C10 2 -0.0000 0.0000 -8.4221 +C11 2 0.0000 -0.0000 -10.9495 +C12 2 0.0000 0.0000 -13.2520 +H13 1 0.0000 0.0000 -15.2675 diff --git a/examples/Carbyne/carbyne.rom.cfg b/examples/Carbyne/carbyne.rom.cfg new file mode 100644 index 00000000..cb0cd295 --- /dev/null +++ b/examples/Carbyne/carbyne.rom.cfg @@ -0,0 +1,34 @@ +verbosity=2 +xcFunctional=PBE +FDtype=4th +[Mesh] +nx= 96 +ny= 96 +nz= 192 +[Domain] +ox= -10. +oy= -10. +oz= -20. +lx= 20. +ly= 20. +lz= 40. +[Potentials] +pseudopotential=pseudo.H_ONCV_PBE_SG15 +pseudopotential=pseudo.C_ONCV_PBE_SG15 +[Run] +type=QUENCH +[Quench] +max_steps=5 +atol=1.e-8 +[Orbitals] +initial_type=Fourier +[Restart] +output_level=4 +input_level=4 +input_filename=snapshot0_000 + +[ROM.offline] +restart_filefmt=snapshot0_%03d +restart_min_idx=0 +restart_max_idx=1 +basis_file=carom diff --git a/potentials/pseudo.C_ONCV_PBE_SG15 b/potentials/pseudo.C_ONCV_PBE_SG15 new file mode 100644 index 00000000..0c9222be --- /dev/null +++ b/potentials/pseudo.C_ONCV_PBE_SG15 @@ -0,0 +1,1918 @@ +# This pseudopotential file has been produced using the code +# ONCVPSP (Optimized Norm-Conservinng Vanderbilt PSeudopotential) +# scalar-relativistic version 2.1.1, 03/26/2014 by D. R. Hamann +# The code is available through a link at URL www.mat-simresearch.com. +# Documentation with the package provides a full discription of the +# input data below. +# +# +# While it is not required under the terms of the GNU GPL, it is +# suggested that you cite D. R. Hamann, Phys. Rev. B 88, 085117 (2013) +# in any publication using these pseudopotentials. +# +# +# Copyright 2015 The Regents of the University of California +# +# This work is licensed under the Creative Commons Attribution-ShareAlike +# 4.0 International License. To view a copy of this license, visit +# http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to +# Creative Commons, PO Box 1866, Mountain View, CA 94042, USA. +# +# This pseudopotential is part of the Schlipf-Gygi norm-conserving +# pseudopotential library. Its construction parameters were tuned to +# reproduce materials of a training set with very high accuracy and +# should be suitable as a general purpose pseudopotential to treat a +# variety of different compounds. For details of the construction and +# testing of the pseudopotential please refer to: +# +# M. Schlipf, F. Gygi, Comp. Phys. Comm. 196, 36 (2015) +# http://dx.doi.org/10.1016/j.cpc.2015.05.011 +# +# We kindly ask that you include this reference in all publications +# associated to this pseudopotential. +# +# +# Input file for PP generation: +# +# # ATOM AND REFERENCE CONFIGURATION +# # atsym z nc nv iexc psfile +# C 6.00 1 2 4 fpmd +# # +# # n l f energy (Ha) +# 1 0 2.00 +# 2 0 2.00 +# 2 1 2.00 +# # +# # PSEUDOPOTENTIAL AND OPTIMIZATION +# # lmax +# 1 +# # +# # l, rc, ep, ncon, nbas, qcut +# 0 1.42330 -0.50533 5 8 6.97628 +# 1 1.38975 -0.19424 5 8 8.59046 +# # +# # LOCAL POTENTIAL +# # lloc, lpopt, rc(5), dvloc0 +# 4 5 0.82223 0.00000 +# # +# # VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# # l, nproj, debl +# 0 2 1.09624 +# 1 2 1.27170 +# # +# # MODEL CORE CHARGE +# # icmod, fcfact +# 0 0.00000 +# # +# # LOG DERIVATIVE ANALYSIS +# # epsh1, epsh2, depsh +# -5.00 3.00 0.02 +# # +# # OUTPUT GRID +# # rlmax, drl +# 6.00 0.01 +# # +# # TEST CONFIGURATIONS +# # ncnf +# 0 +# # nvcnf +# # n l f +# +C_ONCV_PBE-1 +# +color +#radii of balls and covalent bonds +-1. -1. +# Nlcc flag +0 +# Atomic number +6 +# Atomic mass +12.01099968 +# Number of valence electrons +4 +#Gaussian core charge parameter rc +1.0 +# Number of potentials +3 +# l-value for state which is local, then type of potential format +2 3 +# Local potential radius +3.2 +# Non-local potential radius +3.2 +# number of points in radial grid +602 +# VANDERBILT-KLEINMAN-BYLANDER PROJECTORs +# l, nproj +0 2 0.7218622823E+01 0.6690100785E+00 +1 2 -0.8249965422E+01 -0.1429901179E+01 +# l= 0 +0.0 -5.5619338072 2.3136597523 +0.01 -5.5613271269 2.306770973 +0.02 -5.5595256533 2.2858329492 +0.03 -5.5565106605 2.2511205952 +0.04 -5.552263423 2.2029088255 +0.05 -5.5467578832 2.1415793063 +0.06 -5.5399608107 2.0676172759 +0.07 -5.5318320057 1.9816075048 +0.08 -5.5223245478 1.8842294305 +0.09 -5.5113850902 1.7762515113 +0.1 -5.4989541981 1.6585248473 +0.11 -5.4849667317 1.5319761259 +0.12 -5.4693522732 1.3975999522 +0.13 -5.4520355973 1.256450634 +0.14 -5.4329371837 1.1096334907 +0.15 -5.4119737716 0.95829576459 +0.16 -5.3890589549 0.80361721172 +0.17 -5.3641038151 0.64680045627 +0.18 -5.337017592 0.48906119194 +0.19 -5.3077083896 0.3316183141 +0.2 -5.2760839134 0.17568407134 +0.21 -5.2420522381 0.022454314297 +0.22 -5.2055226016 -0.12690106182 +0.23 -5.1664062216 -0.27124741654 +0.24 -5.124617131 -0.40949429974 +0.25 -5.0800730283 -0.54060399148 +0.26 -5.0326961363 -0.66359951197 +0.27 -4.9824140688 -0.77757212406 +0.28 -4.9291606909 -0.88168813665 +0.29 -4.8728769765 -0.97519509555 +0.3 -4.8135118531 -1.0574272291 +0.31 -4.7510230264 -1.1278101258 +0.32 -4.6853777741 -1.185864557 +0.33 -4.6165537088 -1.2312094986 +0.34 -4.5445395309 -1.2635645454 +0.35 -4.4693356442 -1.2827506949 +0.36 -4.3909548687 -1.288691527 +0.37 -4.3094229277 -1.2814121561 +0.38 -4.2247789613 -1.2610381276 +0.39 -4.1370759967 -1.2277936429 +0.4 -4.0463812332 -1.1819982415 +0.41 -3.952776317 -1.1240632626 +0.42 -3.8563575066 -1.0544874328 +0.43 -3.7572357395 -0.97385176327 +0.44 -3.6555365994 -0.88281381205 +0.45 -3.5514001779 -0.78210137005 +0.46 -3.4449808298 -0.6725056329 +0.47 -3.3364468172 -0.55487392206 +0.48 -3.2259798151 -0.430101958 +0.49 -3.1137743869 -0.29912610155 +0.5 -3.0000373177 -0.16291514428 +0.51 -2.8849867403 -0.022461779631 +0.52 -2.7688512208 0.12122564068 +0.53 -2.651868964 0.26713145245 +0.54 -2.5342864393 0.41424112272 +0.55 -2.4163574033 0.56154934377 +0.56 -2.2983415401 0.70806809144 +0.57 -2.1805031333 0.85283401455 +0.58 -2.0631096741 0.99491618126 +0.59 -1.9464303841 1.1334225589 +0.6 -1.8307347092 1.2675064211 +0.61 -1.7162907413 1.3963732692 +0.62 -1.6033637906 1.5192835576 +0.63 -1.4922145525 1.635561325 +0.64 -1.3830977317 1.7445953042 +0.65 -1.2762606481 1.8458416828 +0.66 -1.1719411611 1.9388311169 +0.67 -1.0703666684 2.0231668468 +0.68 -0.97175265325 2.0985266927 +0.69 -0.87630124258 2.1646643489 +0.7 -0.78419961981 2.2214107898 +0.71 -0.69561930648 2.2686711678 +0.72 -0.61071494227 2.3064243138 +0.73 -0.52962337815 2.3347205995 +0.74 -0.45246293337 2.3536791392 +0.75 -0.37933214112 2.3634864197 +0.76 -0.31030987348 2.3643902622 +0.77 -0.24545473583 2.3566964038 +0.78 -0.18480483959 2.3407638141 +0.79 -0.12837773901 2.3169995992 +0.8 -0.076170535039 2.2858535395 +0.81 -0.028160147851 2.247812314 +0.82 0.015696241055 2.2033934812 +0.83 0.055460586594 2.1531391134 +0.84 0.091213316612 2.0976054334 +0.85 0.12305234801 2.0373520722 +0.86 0.1510917027 1.9729367543 +0.87 0.1754600313 1.9049120923 +0.88 0.1962991532 1.833822668 +0.89 0.21376261475 1.7602023128 +0.9 0.22801426385 1.6845715296 +0.91 0.23922683769 1.6074350329 +0.92 0.2475805604 1.5292794072 +0.93 0.25326174895 1.4505709014 +0.94 0.25646142793 1.3717533814 +0.95 0.25737395588 1.293246467 +0.96 0.25619567088 1.2154439876 +0.97 0.25312363541 1.1387132676 +0.98 0.24835419506 1.0633912411 +0.99 0.24208170916 0.98978610791 +1.0 0.23449734129 0.91817586315 +1.01 0.22578788478 0.84880790197 +1.02 0.21613465721 0.78189889487 +1.03 0.20571283628 0.71763618679 +1.04 0.19468974713 0.65617500298 +1.05 0.18322405869 0.59764025691 +1.06 0.17146545139 0.54212876666 +1.07 0.15955378048 0.48970956639 +1.08 0.14761925444 0.44042662809 +1.09 0.13578021615 0.39429608673 +1.1 0.12414335303 0.35130987984 +1.11 0.11280427911 0.31143887213 +1.12 0.10184763589 0.27463448531 +1.13 0.091346262517 0.24082885952 +1.14 0.081359714211 0.20993520907 +1.15 0.071937226969 0.18185367028 +1.16 0.06311806568 0.15647241716 +1.17 0.05492950145 0.13366685986 +1.18 0.047387808736 0.11330297722 +1.19 0.040500743816 0.095240536661 +1.2 0.034268558559 0.0793339749 +1.21 0.028679731635 0.065432327757 +1.22 0.02371719481 0.053383805875 +1.23 0.019358295956 0.043035458196 +1.24 0.015572467964 0.034235102126 +1.25 0.012324990262 0.026833489736 +1.26 0.009578060204 0.020683468773 +1.27 0.0072895436424 0.015643391488 +1.28 0.0054156693176 0.011578176587 +1.29 0.0039116184969 0.0083565147667 +1.3 0.0027316798311 0.0058570361906 +1.31 0.0018312761151 0.0039669563938 +1.32 0.0011660035349 0.0025776254427 +1.33 0.00069459948121 0.0015955354062 +1.34 0.00037829246653 0.0009342822342 +1.35 0.0001799237711 0.00051380833982 +1.36 6.9136389729e-05 0.00027171239954 +1.37 1.7229630482e-05 0.00014914664697 +1.38 1.0827177075e-06 0.00010030010168 +1.39 4.1385366266e-06 9.2442210335e-05 +1.4 1.0655785986e-05 9.4807098664e-05 +1.41 1.6544198764e-05 0.0001006010957 +1.42 1.7097183705e-05 9.7202461605e-05 +1.43 1.0038672529e-05 5.6070727265e-05 +1.44 3.1156129699e-07 1.7181193817e-06 +1.45 -9.8426031638e-07 -5.5758020489e-06 +1.46 0.0 0.0 +1.47 0.0 0.0 +1.48 0.0 0.0 +1.49 0.0 0.0 +1.5 0.0 0.0 +1.51 0.0 0.0 +1.52 0.0 0.0 +1.53 0.0 0.0 +1.54 0.0 0.0 +1.55 0.0 0.0 +1.56 0.0 0.0 +1.57 0.0 0.0 +1.58 0.0 0.0 +1.59 0.0 0.0 +1.6 0.0 0.0 +1.61 0.0 0.0 +1.62 0.0 0.0 +1.63 0.0 0.0 +1.64 0.0 0.0 +1.65 0.0 0.0 +1.66 0.0 0.0 +1.67 0.0 0.0 +1.68 0.0 0.0 +1.69 0.0 0.0 +1.7 0.0 0.0 +1.71 0.0 0.0 +1.72 0.0 0.0 +1.73 0.0 0.0 +1.74 0.0 0.0 +1.75 0.0 0.0 +1.76 0.0 0.0 +1.77 0.0 0.0 +1.78 0.0 0.0 +1.79 0.0 0.0 +1.8 0.0 0.0 +1.81 0.0 0.0 +1.82 0.0 0.0 +1.83 0.0 0.0 +1.84 0.0 0.0 +1.85 0.0 0.0 +1.86 0.0 0.0 +1.87 0.0 0.0 +1.88 0.0 0.0 +1.89 0.0 0.0 +1.9 0.0 0.0 +1.91 0.0 0.0 +1.92 0.0 0.0 +1.93 0.0 0.0 +1.94 0.0 0.0 +1.95 0.0 0.0 +1.96 0.0 0.0 +1.97 0.0 0.0 +1.98 0.0 0.0 +1.99 0.0 0.0 +2.0 0.0 0.0 +2.01 0.0 0.0 +2.02 0.0 0.0 +2.03 0.0 0.0 +2.04 0.0 0.0 +2.05 0.0 0.0 +2.06 0.0 0.0 +2.07 0.0 0.0 +2.08 0.0 0.0 +2.09 0.0 0.0 +2.1 0.0 0.0 +2.11 0.0 0.0 +2.12 0.0 0.0 +2.13 0.0 0.0 +2.14 0.0 0.0 +2.15 0.0 0.0 +2.16 0.0 0.0 +2.17 0.0 0.0 +2.18 0.0 0.0 +2.19 0.0 0.0 +2.2 0.0 0.0 +2.21 0.0 0.0 +2.22 0.0 0.0 +2.23 0.0 0.0 +2.24 0.0 0.0 +2.25 0.0 0.0 +2.26 0.0 0.0 +2.27 0.0 0.0 +2.28 0.0 0.0 +2.29 0.0 0.0 +2.3 0.0 0.0 +2.31 0.0 0.0 +2.32 0.0 0.0 +2.33 0.0 0.0 +2.34 0.0 0.0 +2.35 0.0 0.0 +2.36 0.0 0.0 +2.37 0.0 0.0 +2.38 0.0 0.0 +2.39 0.0 0.0 +2.4 0.0 0.0 +2.41 0.0 0.0 +2.42 0.0 0.0 +2.43 0.0 0.0 +2.44 0.0 0.0 +2.45 0.0 0.0 +2.46 0.0 0.0 +2.47 0.0 0.0 +2.48 0.0 0.0 +2.49 0.0 0.0 +2.5 0.0 0.0 +2.51 0.0 0.0 +2.52 0.0 0.0 +2.53 0.0 0.0 +2.54 0.0 0.0 +2.55 0.0 0.0 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# l= 1 +0.0 0.0 0.0 +0.01 -0.64780534934 0.033926819602 +0.02 -1.2915687784 0.067115131906 +0.03 -1.9272828815 0.098832572454 +0.04 -2.5510089296 0.1283590174 +0.05 -3.1589103313 0.15499259139 +0.06 -3.7472850521 0.17805554131 +0.07 -4.3125966635 0.19689993224 +0.08 -4.8515037091 0.21091312341 +0.09 -5.3608870923 0.21952298258 +0.1 -5.8378752164 0.22220279966 +0.11 -6.2798666291 0.21847586132 +0.12 -6.6845499547 0.20791965116 +0.13 -7.0499209258 0.19016964181 +0.14 -7.3742963609 0.16492264816 +0.15 -7.6563249674 0.13193971365 +0.16 -7.8949948837 0.091048504616 +0.17 -8.0896379132 0.04214519074 +0.18 -8.2399304373 -0.014804206349 +0.19 -8.3458910342 -0.079763010277 +0.2 -8.4078748568 -0.15262364645 +0.21 -8.4265648743 -0.23320795956 +0.22 -8.4029601006 -0.32126810011 +0.23 -8.3383609682 -0.41648797387 +0.24 -8.2343520548 -0.51848524956 +0.25 -8.0927823441 -0.62681391511 +0.26 -7.9157433195 -0.74096736404 +0.27 -7.7055450653 -0.86038199981 +0.28 -7.4646907971 -0.98444131928 +0.29 -7.1958499632 -1.1124804639 +0.3 -6.9018303107 -1.2437911968 +0.31 -6.5855492109 -1.3776272713 +0.32 -6.2500046411 -1.5132101364 +0.33 -5.8982460321 -1.6497349494 +0.34 -5.5333449188 -1.7863769263 +0.35 -5.158367357 -1.9222976618 +0.36 -4.7763450651 -2.0566520032 +0.37 -4.3902499322 -2.188594581 +0.38 -4.0029685747 -2.3172866264 +0.39 -3.6172780832 -2.4419028795 +0.4 -3.2358249895 -2.5616381055 +0.41 -2.8611052027 -2.6757137019 +0.42 -2.4954463623 -2.7833840563 +0.43 -2.1409925025 -2.8839426519 +0.44 -1.799691123 -2.9767278641 +0.45 -1.4732827402 -3.0611283953 +0.46 -1.1632929665 -3.1365882961 +0.47 -0.87102713825 -3.2026115192 +0.48 -0.59756774211 -3.2587658564 +0.49 -0.34377345327 -3.3046866329 +0.5 -0.11028102737 -3.3400797234 +0.51 0.10248962108 -3.3647235439 +0.52 0.29432899807 -3.3784706152 +0.53 0.46523115439 -3.3812495258 +0.54 0.61538117426 -3.3730640284 +0.55 0.74514485936 -3.3539936857 +0.56 0.85505453712 -3.3241924975 +0.57 0.94579475876 -3.283887267 +0.58 1.0181864156 -3.2333755366 +0.59 1.0731705285 -3.173022518 +0.6 1.1117912955 -3.1032576171 +0.61 1.1351777658 -3.0245705747 +0.62 1.1445292769 -2.9375066811 +0.63 1.1410935809 -2.8426618746 +0.64 1.1261533549 -2.7406772038 +0.65 1.1010111226 -2.6322334784 +0.66 1.0669671337 -2.5180441918 +0.67 1.0253102121 -2.3988499563 +0.68 0.97730314572 -2.2754120477 +0.69 0.92416897244 -2.1485056149 +0.7 0.86707688474 -2.0189121998 +0.71 0.80713617698 -1.8874142198 +0.72 0.74538605105 -1.7547880998 +0.73 0.68278857322 -1.6217980466 +0.74 0.62022324162 -1.4891901718 +0.75 0.55847896344 -1.3576850585 +0.76 0.49825562113 -1.2279741547 +0.77 0.44016101576 -1.1007141303 +0.78 0.38471071244 -0.97652230381 +0.79 0.33232918224 -0.8559725807 +0.8 0.28335215422 -0.73959194569 +0.81 0.23803007837 -0.62785754003 +0.82 0.19653258058 -0.5211943311 +0.83 0.15895386327 -0.41997355246 +0.84 0.12532063101 -0.32451537422 +0.85 0.095600791441 -0.23509186031 +0.86 0.069710132362 -0.15192507951 +0.87 0.047518368191 -0.075184277576 +0.88 0.028855300655 -0.0049840255013 +0.89 0.013517039951 0.05861662308 +0.9 0.0012722220923 0.11561397758 +0.91 -0.0081318523555 0.16605861394 +0.92 -0.01496324012 0.21005342134 +0.93 -0.01949961994 0.24775089312 +0.94 -0.022022877834 0.27934976934 +0.95 -0.0228140824 0.3050911397 +0.96 -0.022149006634 0.32525408527 +0.97 -0.020294730264 0.34015110422 +0.98 -0.017503578454 0.3501236019 +0.99 -0.014012391119 0.35553587811 +1.0 -0.010039421716 0.3567703707 +1.01 -0.005782469847 0.35422252624 +1.02 -0.0014175661889 0.34829577469 +1.03 0.0029006748609 0.33939773412 +1.04 0.0070423781443 0.32793343124 +1.05 0.010900950701 0.31430081113 +1.06 0.014391237266 0.29888798074 +1.07 0.017449258127 0.28206900894 +1.08 0.020029927756 0.26420289539 +1.09 0.022108738697 0.24562502229 +1.1 0.023677942226 0.22664632821 +1.11 0.024743739452 0.20755381676 +1.12 0.025324724614 0.1886095756 +1.13 0.025450946237 0.1700468885 +1.14 0.02516175063 0.15206467452 +1.15 0.024502167077 0.13483580138 +1.16 0.023522048509 0.11850708666 +1.17 0.022274618176 0.10319233632 +1.18 0.020813617369 0.088975109451 +1.19 0.019192395986 0.075915488104 +1.2 0.01746388826 0.064052115651 +1.21 0.015676085956 0.053390322657 +1.22 0.013873829173 0.043919625043 +1.23 0.0120988944 0.035612808782 +1.24 0.010385862022 0.028420365824 +1.25 0.0087640190687 0.022280799398 +1.26 0.0072588523059 0.017122604968 +1.27 0.0058881873633 0.012862435072 +1.28 0.004664238389 0.0094121802565 +1.29 0.003595862314 0.006679174829 +1.3 0.0026846783817 0.0045691116006 +1.31 0.0019281814044 0.0029901990184 +1.32 0.001321232365 0.0018494584294 +1.33 0.00085263675895 0.001063624522 +1.34 0.00050929459516 0.00055440064214 +1.35 0.000275550143 0.00024702703787 +1.36 0.00013220980019 8.4424103539e-05 +1.37 5.9143848213e-05 1.1903784385e-05 +1.38 3.4902866327e-05 -1.2570722298e-05 +1.39 3.2315666403e-05 -1.2426396709e-05 +1.4 1.1667938285e-05 -5.5809567759e-06 +1.41 -1.6962328005e-06 6.4699044005e-07 +1.42 -1.0808333026e-06 4.1225992911e-07 +1.43 0.0 0.0 +1.44 0.0 0.0 +1.45 0.0 0.0 +1.46 0.0 0.0 +1.47 0.0 0.0 +1.48 0.0 0.0 +1.49 0.0 0.0 +1.5 0.0 0.0 +1.51 0.0 0.0 +1.52 0.0 0.0 +1.53 0.0 0.0 +1.54 0.0 0.0 +1.55 0.0 0.0 +1.56 0.0 0.0 +1.57 0.0 0.0 +1.58 0.0 0.0 +1.59 0.0 0.0 +1.6 0.0 0.0 +1.61 0.0 0.0 +1.62 0.0 0.0 +1.63 0.0 0.0 +1.64 0.0 0.0 +1.65 0.0 0.0 +1.66 0.0 0.0 +1.67 0.0 0.0 +1.68 0.0 0.0 +1.69 0.0 0.0 +1.7 0.0 0.0 +1.71 0.0 0.0 +1.72 0.0 0.0 +1.73 0.0 0.0 +1.74 0.0 0.0 +1.75 0.0 0.0 +1.76 0.0 0.0 +1.77 0.0 0.0 +1.78 0.0 0.0 +1.79 0.0 0.0 +1.8 0.0 0.0 +1.81 0.0 0.0 +1.82 0.0 0.0 +1.83 0.0 0.0 +1.84 0.0 0.0 +1.85 0.0 0.0 +1.86 0.0 0.0 +1.87 0.0 0.0 +1.88 0.0 0.0 +1.89 0.0 0.0 +1.9 0.0 0.0 +1.91 0.0 0.0 +1.92 0.0 0.0 +1.93 0.0 0.0 +1.94 0.0 0.0 +1.95 0.0 0.0 +1.96 0.0 0.0 +1.97 0.0 0.0 +1.98 0.0 0.0 +1.99 0.0 0.0 +2.0 0.0 0.0 +2.01 0.0 0.0 +2.02 0.0 0.0 +2.03 0.0 0.0 +2.04 0.0 0.0 +2.05 0.0 0.0 +2.06 0.0 0.0 +2.07 0.0 0.0 +2.08 0.0 0.0 +2.09 0.0 0.0 +2.1 0.0 0.0 +2.11 0.0 0.0 +2.12 0.0 0.0 +2.13 0.0 0.0 +2.14 0.0 0.0 +2.15 0.0 0.0 +2.16 0.0 0.0 +2.17 0.0 0.0 +2.18 0.0 0.0 +2.19 0.0 0.0 +2.2 0.0 0.0 +2.21 0.0 0.0 +2.22 0.0 0.0 +2.23 0.0 0.0 +2.24 0.0 0.0 +2.25 0.0 0.0 +2.26 0.0 0.0 +2.27 0.0 0.0 +2.28 0.0 0.0 +2.29 0.0 0.0 +2.3 0.0 0.0 +2.31 0.0 0.0 +2.32 0.0 0.0 +2.33 0.0 0.0 +2.34 0.0 0.0 +2.35 0.0 0.0 +2.36 0.0 0.0 +2.37 0.0 0.0 +2.38 0.0 0.0 +2.39 0.0 0.0 +2.4 0.0 0.0 +2.41 0.0 0.0 +2.42 0.0 0.0 +2.43 0.0 0.0 +2.44 0.0 0.0 +2.45 0.0 0.0 +2.46 0.0 0.0 +2.47 0.0 0.0 +2.48 0.0 0.0 +2.49 0.0 0.0 +2.5 0.0 0.0 +2.51 0.0 0.0 +2.52 0.0 0.0 +2.53 0.0 0.0 +2.54 0.0 0.0 +2.55 0.0 0.0 +2.56 0.0 0.0 +2.57 0.0 0.0 +2.58 0.0 0.0 +2.59 0.0 0.0 +2.6 0.0 0.0 +2.61 0.0 0.0 +2.62 0.0 0.0 +2.63 0.0 0.0 +2.64 0.0 0.0 +2.65 0.0 0.0 +2.66 0.0 0.0 +2.67 0.0 0.0 +2.68 0.0 0.0 +2.69 0.0 0.0 +2.7 0.0 0.0 +2.71 0.0 0.0 +2.72 0.0 0.0 +2.73 0.0 0.0 +2.74 0.0 0.0 +2.75 0.0 0.0 +2.76 0.0 0.0 +2.77 0.0 0.0 +2.78 0.0 0.0 +2.79 0.0 0.0 +2.8 0.0 0.0 +2.81 0.0 0.0 +2.82 0.0 0.0 +2.83 0.0 0.0 +2.84 0.0 0.0 +2.85 0.0 0.0 +2.86 0.0 0.0 +2.87 0.0 0.0 +2.88 0.0 0.0 +2.89 0.0 0.0 +2.9 0.0 0.0 +2.91 0.0 0.0 +2.92 0.0 0.0 +2.93 0.0 0.0 +2.94 0.0 0.0 +2.95 0.0 0.0 +2.96 0.0 0.0 +2.97 0.0 0.0 +2.98 0.0 0.0 +2.99 0.0 0.0 +3.0 0.0 0.0 +3.01 0.0 0.0 +3.02 0.0 0.0 +3.03 0.0 0.0 +3.04 0.0 0.0 +3.05 0.0 0.0 +3.06 0.0 0.0 +3.07 0.0 0.0 +3.08 0.0 0.0 +3.09 0.0 0.0 +3.1 0.0 0.0 +3.11 0.0 0.0 +3.12 0.0 0.0 +3.13 0.0 0.0 +3.14 0.0 0.0 +3.15 0.0 0.0 +3.16 0.0 0.0 +3.17 0.0 0.0 +3.18 0.0 0.0 +3.19 0.0 0.0 +3.2 0.0 0.0 +3.21 0.0 0.0 +3.22 0.0 0.0 +3.23 0.0 0.0 +3.24 0.0 0.0 +3.25 0.0 0.0 +3.26 0.0 0.0 +3.27 0.0 0.0 +3.28 0.0 0.0 +3.29 0.0 0.0 +3.3 0.0 0.0 +3.31 0.0 0.0 +3.32 0.0 0.0 +3.33 0.0 0.0 +3.34 0.0 0.0 +3.35 0.0 0.0 +3.36 0.0 0.0 +3.37 0.0 0.0 +3.38 0.0 0.0 +3.39 0.0 0.0 +3.4 0.0 0.0 +3.41 0.0 0.0 +3.42 0.0 0.0 +3.43 0.0 0.0 +3.44 0.0 0.0 +3.45 0.0 0.0 +3.46 0.0 0.0 +3.47 0.0 0.0 +3.48 0.0 0.0 +3.49 0.0 0.0 +3.5 0.0 0.0 +3.51 0.0 0.0 +3.52 0.0 0.0 +3.53 0.0 0.0 +3.54 0.0 0.0 +3.55 0.0 0.0 +3.56 0.0 0.0 +3.57 0.0 0.0 +3.58 0.0 0.0 +3.59 0.0 0.0 +3.6 0.0 0.0 +3.61 0.0 0.0 +3.62 0.0 0.0 +3.63 0.0 0.0 +3.64 0.0 0.0 +3.65 0.0 0.0 +3.66 0.0 0.0 +3.67 0.0 0.0 +3.68 0.0 0.0 +3.69 0.0 0.0 +3.7 0.0 0.0 +3.71 0.0 0.0 +3.72 0.0 0.0 +3.73 0.0 0.0 +3.74 0.0 0.0 +3.75 0.0 0.0 +3.76 0.0 0.0 +3.77 0.0 0.0 +3.78 0.0 0.0 +3.79 0.0 0.0 +3.8 0.0 0.0 +3.81 0.0 0.0 +3.82 0.0 0.0 +3.83 0.0 0.0 +3.84 0.0 0.0 +3.85 0.0 0.0 +3.86 0.0 0.0 +3.87 0.0 0.0 +3.88 0.0 0.0 +3.89 0.0 0.0 +3.9 0.0 0.0 +3.91 0.0 0.0 +3.92 0.0 0.0 +3.93 0.0 0.0 +3.94 0.0 0.0 +3.95 0.0 0.0 +3.96 0.0 0.0 +3.97 0.0 0.0 +3.98 0.0 0.0 +3.99 0.0 0.0 +4.0 0.0 0.0 +4.01 0.0 0.0 +4.02 0.0 0.0 +4.03 0.0 0.0 +4.04 0.0 0.0 +4.05 0.0 0.0 +4.06 0.0 0.0 +4.07 0.0 0.0 +4.08 0.0 0.0 +4.09 0.0 0.0 +4.1 0.0 0.0 +4.11 0.0 0.0 +4.12 0.0 0.0 +4.13 0.0 0.0 +4.14 0.0 0.0 +4.15 0.0 0.0 +4.16 0.0 0.0 +4.17 0.0 0.0 +4.18 0.0 0.0 +4.19 0.0 0.0 +4.2 0.0 0.0 +4.21 0.0 0.0 +4.22 0.0 0.0 +4.23 0.0 0.0 +4.24 0.0 0.0 +4.25 0.0 0.0 +4.26 0.0 0.0 +4.27 0.0 0.0 +4.28 0.0 0.0 +4.29 0.0 0.0 +4.3 0.0 0.0 +4.31 0.0 0.0 +4.32 0.0 0.0 +4.33 0.0 0.0 +4.34 0.0 0.0 +4.35 0.0 0.0 +4.36 0.0 0.0 +4.37 0.0 0.0 +4.38 0.0 0.0 +4.39 0.0 0.0 +4.4 0.0 0.0 +4.41 0.0 0.0 +4.42 0.0 0.0 +4.43 0.0 0.0 +4.44 0.0 0.0 +4.45 0.0 0.0 +4.46 0.0 0.0 +4.47 0.0 0.0 +4.48 0.0 0.0 +4.49 0.0 0.0 +4.5 0.0 0.0 +4.51 0.0 0.0 +4.52 0.0 0.0 +4.53 0.0 0.0 +4.54 0.0 0.0 +4.55 0.0 0.0 +4.56 0.0 0.0 +4.57 0.0 0.0 +4.58 0.0 0.0 +4.59 0.0 0.0 +4.6 0.0 0.0 +4.61 0.0 0.0 +4.62 0.0 0.0 +4.63 0.0 0.0 +4.64 0.0 0.0 +4.65 0.0 0.0 +4.66 0.0 0.0 +4.67 0.0 0.0 +4.68 0.0 0.0 +4.69 0.0 0.0 +4.7 0.0 0.0 +4.71 0.0 0.0 +4.72 0.0 0.0 +4.73 0.0 0.0 +4.74 0.0 0.0 +4.75 0.0 0.0 +4.76 0.0 0.0 +4.77 0.0 0.0 +4.78 0.0 0.0 +4.79 0.0 0.0 +4.8 0.0 0.0 +4.81 0.0 0.0 +4.82 0.0 0.0 +4.83 0.0 0.0 +4.84 0.0 0.0 +4.85 0.0 0.0 +4.86 0.0 0.0 +4.87 0.0 0.0 +4.88 0.0 0.0 +4.89 0.0 0.0 +4.9 0.0 0.0 +4.91 0.0 0.0 +4.92 0.0 0.0 +4.93 0.0 0.0 +4.94 0.0 0.0 +4.95 0.0 0.0 +4.96 0.0 0.0 +4.97 0.0 0.0 +4.98 0.0 0.0 +4.99 0.0 0.0 +5.0 0.0 0.0 +5.01 0.0 0.0 +5.02 0.0 0.0 +5.03 0.0 0.0 +5.04 0.0 0.0 +5.05 0.0 0.0 +5.06 0.0 0.0 +5.07 0.0 0.0 +5.08 0.0 0.0 +5.09 0.0 0.0 +5.1 0.0 0.0 +5.11 0.0 0.0 +5.12 0.0 0.0 +5.13 0.0 0.0 +5.14 0.0 0.0 +5.15 0.0 0.0 +5.16 0.0 0.0 +5.17 0.0 0.0 +5.18 0.0 0.0 +5.19 0.0 0.0 +5.2 0.0 0.0 +5.21 0.0 0.0 +5.22 0.0 0.0 +5.23 0.0 0.0 +5.24 0.0 0.0 +5.25 0.0 0.0 +5.26 0.0 0.0 +5.27 0.0 0.0 +5.28 0.0 0.0 +5.29 0.0 0.0 +5.3 0.0 0.0 +5.31 0.0 0.0 +5.32 0.0 0.0 +5.33 0.0 0.0 +5.34 0.0 0.0 +5.35 0.0 0.0 +5.36 0.0 0.0 +5.37 0.0 0.0 +5.38 0.0 0.0 +5.39 0.0 0.0 +5.4 0.0 0.0 +5.41 0.0 0.0 +5.42 0.0 0.0 +5.43 0.0 0.0 +5.44 0.0 0.0 +5.45 0.0 0.0 +5.46 0.0 0.0 +5.47 0.0 0.0 +5.48 0.0 0.0 +5.49 0.0 0.0 +5.5 0.0 0.0 +5.51 0.0 0.0 +5.52 0.0 0.0 +5.53 0.0 0.0 +5.54 0.0 0.0 +5.55 0.0 0.0 +5.56 0.0 0.0 +5.57 0.0 0.0 +5.58 0.0 0.0 +5.59 0.0 0.0 +5.6 0.0 0.0 +5.61 0.0 0.0 +5.62 0.0 0.0 +5.63 0.0 0.0 +5.64 0.0 0.0 +5.65 0.0 0.0 +5.66 0.0 0.0 +5.67 0.0 0.0 +5.68 0.0 0.0 +5.69 0.0 0.0 +5.7 0.0 0.0 +5.71 0.0 0.0 +5.72 0.0 0.0 +5.73 0.0 0.0 +5.74 0.0 0.0 +5.75 0.0 0.0 +5.76 0.0 0.0 +5.77 0.0 0.0 +5.78 0.0 0.0 +5.79 0.0 0.0 +5.8 0.0 0.0 +5.81 0.0 0.0 +5.82 0.0 0.0 +5.83 0.0 0.0 +5.84 0.0 0.0 +5.85 0.0 0.0 +5.86 0.0 0.0 +5.87 0.0 0.0 +5.88 0.0 0.0 +5.89 0.0 0.0 +5.9 0.0 0.0 +5.91 0.0 0.0 +5.92 0.0 0.0 +5.93 0.0 0.0 +5.94 0.0 0.0 +5.95 0.0 0.0 +5.96 0.0 0.0 +5.97 0.0 0.0 +5.98 0.0 0.0 +5.99 0.0 0.0 +6.0 0.0 0.0 +6.01 0.0 0.0 +# local +0.0 -8.0040357051E+00 +0.01 -1.0915313841E+01 +0.02 -1.1926634752E+01 +0.03 -1.1787164785E+01 +0.04 -1.1246070288E+01 +0.05 -1.0794315449E+01 +0.06 -1.0504229539E+01 +0.07 -1.0326824625E+01 +0.08 -1.0214366235E+01 +0.09 -1.0137727506E+01 +0.1 -1.0080874316E+01 +0.11 -1.0035003937E+01 +0.12 -9.9950701240E+00 +0.13 -9.9579682786E+00 +0.14 -9.9216175254E+00 +0.15 -9.8844988979E+00 +0.16 -9.8454251644E+00 +0.17 -9.8034308020E+00 +0.18 -9.7577256587E+00 +0.19 -9.7076822519E+00 +0.2 -9.6528393253E+00 +0.21 -9.5929109716E+00 +0.22 -9.5277938843E+00 +0.23 -9.4575680815E+00 +0.24 -9.3824880939E+00 +0.25 -9.3029638082E+00 +0.26 -9.2195315487E+00 +0.27 -9.1328179593E+00 +0.28 -9.0434999753E+00 +0.29 -8.9522650634E+00 +0.3 -8.8597754125E+00 +0.31 -8.7666390992E+00 +0.32 -8.6733899916E+00 +0.33 -8.5804766285E+00 +0.34 -8.4882589889E+00 +0.35 -8.3970122835E+00 +0.36 -8.3069340415E+00 +0.37 -8.2181545195E+00 +0.38 -8.1307470735E+00 +0.39 -8.0447379951E+00 +0.4 -7.9601154422E+00 +0.41 -7.8768366143E+00 +0.42 -7.7948336090E+00 +0.43 -7.7140181523E+00 +0.44 -7.6342855586E+00 +0.45 -7.5555182806E+00 +0.46 -7.4775893571E+00 +0.47 -7.4003659698E+00 +0.48 -7.3237131964E+00 +0.49 -7.2474979648E+00 +0.5 -7.1715931897E+00 +0.51 -7.0958817131E+00 +0.52 -7.0202598536E+00 +0.53 -6.9446410093E+00 +0.54 -6.8689575562E+00 +0.55 -6.7931628572E+00 +0.56 -6.7172317494E+00 +0.57 -6.6411601915E+00 +0.58 -6.5649643349E+00 +0.59 -6.4886785011E+00 +0.6 -6.4123528084E+00 +0.61 -6.3360506878E+00 +0.62 -6.2598455474E+00 +0.63 -6.1838184649E+00 +0.64 -6.1080550858E+00 +0.65 -6.0326433511E+00 +0.66 -5.9576713766E+00 +0.67 -5.8832256635E+00 +0.68 -5.8093897646E+00 +0.69 -5.7362431951E+00 +0.7 -5.6638605523E+00 +0.71 -5.5923111611E+00 +0.72 -5.5216587284E+00 +0.73 -5.4519612207E+00 +0.74 -5.3832708703E+00 +0.75 -5.3156340870E+00 +0.76 -5.2490917825E+00 +0.77 -5.1836794575E+00 +0.78 -5.1194273636E+00 +0.79 -5.0563606602E+00 +0.8 -4.9944995528E+00 +0.81 -4.9338594083E+00 +0.82 -4.8744508489E+00 +0.83 -4.8162797570E+00 +0.84 -4.7593456941E+00 +0.85 -4.7036404323E+00 +0.86 -4.6491488191E+00 +0.87 -4.5958503018E+00 +0.88 -4.5437202905E+00 +0.89 -4.4927313346E+00 +0.9 -4.4428541041E+00 +0.91 -4.3940581854E+00 +0.92 -4.3463127073E+00 +0.93 -4.2995868222E+00 +0.94 -4.2538500632E+00 +0.95 -4.2090726006E+00 +0.96 -4.1652254133E+00 +0.97 -4.1222803761E+00 +0.98 -4.0802104155E+00 +0.99 -4.0389894424E+00 +1.0 -3.9985923752E+00 +1.01 -3.9589951171E+00 +1.02 -3.9201745213E+00 +1.03 -3.8821083136E+00 +1.04 -3.8447751347E+00 +1.05 -3.8081544513E+00 +1.06 -3.7722264724E+00 +1.07 -3.7369721252E+00 +1.08 -3.7023729745E+00 +1.09 -3.6684112850E+00 +1.1 -3.6350699327E+00 +1.11 -3.6023323365E+00 +1.12 -3.5701824270E+00 +1.13 -3.5386046578E+00 +1.14 -3.5075840389E+00 +1.15 -3.4771060022E+00 +1.16 -3.4471563851E+00 +1.17 -3.4177215076E+00 +1.18 -3.3887881411E+00 +1.19 -3.3603434189E+00 +1.2 -3.3323747917E+00 +1.21 -3.3048702537E+00 +1.22 -3.2778180933E+00 +1.23 -3.2512068883E+00 +1.24 -3.2250256902E+00 +1.25 -3.1992638816E+00 +1.26 -3.1739110904E+00 +1.27 -3.1489573788E+00 +1.28 -3.1243931477E+00 +1.29 -3.1002089989E+00 +1.3 -3.0763959737E+00 +1.31 -3.0529454103E+00 +1.32 -3.0298488061E+00 +1.33 -3.0070981380E+00 +1.34 -2.9846855681E+00 +1.35 -2.9626034484E+00 +1.36 -2.9408445649E+00 +1.37 -2.9194017799E+00 +1.38 -2.8982682474E+00 +1.39 -2.8774374158E+00 +1.4 -2.8569027838E+00 +1.41 -2.8366583102E+00 +1.42 -2.8166979333E+00 +1.43 -2.7970157170E+00 +1.44 -2.7776059465E+00 +1.45 -2.7584628862E+00 +1.46 -2.7395813120E+00 +1.47 -2.7209559281E+00 +1.48 -2.7025814769E+00 +1.49 -2.6844529182E+00 +1.5 -2.6665654321E+00 +1.51 -2.6489143215E+00 +1.52 -2.6314948525E+00 +1.53 -2.6143026185E+00 +1.54 -2.5973331367E+00 +1.55 -2.5805822050E+00 +1.56 -2.5640455899E+00 +1.57 -2.5477192629E+00 +1.58 -2.5315992341E+00 +1.59 -2.5156816404E+00 +1.6 -2.4999627160E+00 +1.61 -2.4844387583E+00 +1.62 -2.4691062076E+00 +1.63 -2.4539615174E+00 +1.64 -2.4390013163E+00 +1.65 -2.4242222103E+00 +1.66 -2.4096210002E+00 +1.67 -2.3951944407E+00 +1.68 -2.3809394898E+00 +1.69 -2.3668530475E+00 +1.7 -2.3529322150E+00 +1.71 -2.3391740343E+00 +1.72 -2.3255757369E+00 +1.73 -2.3121345038E+00 +1.74 -2.2988476842E+00 +1.75 -2.2857125958E+00 +1.76 -2.2727266936E+00 +1.77 -2.2598874301E+00 +1.78 -2.2471923547E+00 +1.79 -2.2346390526E+00 +1.8 -2.2222251574E+00 +1.81 -2.2099483850E+00 +1.82 -2.1978064438E+00 +1.83 -2.1857971777E+00 +1.84 -2.1739183621E+00 +1.85 -2.1621679623E+00 +1.86 -2.1505438243E+00 +1.87 -2.1390440064E+00 +1.88 -2.1276664658E+00 +1.89 -2.1164093055E+00 +1.9 -2.1052706076E+00 +1.91 -2.0942485108E+00 +1.92 -2.0833412212E+00 +1.93 -2.0725469071E+00 +1.94 -2.0618638889E+00 +1.95 -2.0512903807E+00 +1.96 -2.0408247749E+00 +1.97 -2.0304653889E+00 +1.98 -2.0202106279E+00 +1.99 -2.0100589312E+00 +2.0 -2.0000087103E+00 +2.01 -1.9900585122E+00 +2.02 -1.9802067870E+00 +2.03 -1.9704521324E+00 +2.04 -1.9607931012E+00 +2.05 -1.9512282831E+00 +2.06 -1.9417563496E+00 +2.07 -1.9323758836E+00 +2.08 -1.9230856358E+00 +2.09 -1.9138842758E+00 +2.1 -1.9047705356E+00 +2.11 -1.8957432036E+00 +2.12 -1.8868009904E+00 +2.13 -1.8779427643E+00 +2.14 -1.8691673156E+00 +2.15 -1.8604734863E+00 +2.16 -1.8518601785E+00 +2.17 -1.8433262185E+00 +2.18 -1.8348705689E+00 +2.19 -1.8264921414E+00 +2.2 -1.8181898599E+00 +2.21 -1.8099627353E+00 +2.22 -1.8018097066E+00 +2.23 -1.7937298023E+00 +2.24 -1.7857220591E+00 +2.25 -1.7777854595E+00 +2.26 -1.7699191167E+00 +2.27 -1.7621220808E+00 +2.28 -1.7543934221E+00 +2.29 -1.7467322844E+00 +2.3 -1.7391377483E+00 +2.31 -1.7316089650E+00 +2.32 -1.7241451058E+00 +2.33 -1.7167452837E+00 +2.34 -1.7094087213E+00 +2.35 -1.7021346118E+00 +2.36 -1.6949221170E+00 +2.37 -1.6877705067E+00 +2.38 -1.6806789977E+00 +2.39 -1.6736468113E+00 +2.4 -1.6666732460E+00 +2.41 -1.6597575481E+00 +2.42 -1.6528989922E+00 +2.43 -1.6460969044E+00 +2.44 -1.6393505609E+00 +2.45 -1.6326592831E+00 +2.46 -1.6260224243E+00 +2.47 -1.6194392909E+00 +2.48 -1.6129092448E+00 +2.49 -1.6064316667E+00 +2.5 -1.6000058928E+00 +2.51 -1.5936313198E+00 +2.52 -1.5873073559E+00 +2.53 -1.5810333672E+00 +2.54 -1.5748087798E+00 +2.55 -1.5686330293E+00 +2.56 -1.5625055119E+00 +2.57 -1.5564256777E+00 +2.58 -1.5503929901E+00 +2.59 -1.5444068751E+00 +2.6 -1.5384668024E+00 +2.61 -1.5325722630E+00 +2.62 -1.5267227130E+00 +2.63 -1.5209176368E+00 +2.64 -1.5151565535E+00 +2.65 -1.5094389490E+00 +2.66 -1.5037643188E+00 +2.67 -1.4981322096E+00 +2.68 -1.4925421373E+00 +2.69 -1.4869936048E+00 +2.7 -1.4814861859E+00 +2.71 -1.4760194253E+00 +2.72 -1.4705928352E+00 +2.73 -1.4652060108E+00 +2.74 -1.4598585195E+00 +2.75 -1.4545499031E+00 +2.76 -1.4492797547E+00 +2.77 -1.4440476710E+00 +2.78 -1.4388532240E+00 +2.79 -1.4336960013E+00 +2.8 -1.4285756285E+00 +2.81 -1.4234917081E+00 +2.82 -1.4184438201E+00 +2.83 -1.4134316176E+00 +2.84 -1.4084547241E+00 +2.85 -1.4035127407E+00 +2.86 -1.3986053162E+00 +2.87 -1.3937321017E+00 +2.88 -1.3888927287E+00 +2.89 -1.3840868309E+00 +2.9 -1.3793140881E+00 +2.91 -1.3745741585E+00 +2.92 -1.3698666744E+00 +2.93 -1.3651913276E+00 +2.94 -1.3605477969E+00 +2.95 -1.3559357455E+00 +2.96 -1.3513548435E+00 +2.97 -1.3468047997E+00 +2.98 -1.3422853031E+00 +2.99 -1.3377960192E+00 +3.0 -1.3333366667E+00 +3.01 -1.3289069548E+00 +3.02 -1.3245065801E+00 +3.03 -1.3201352345E+00 +3.04 -1.3157926572E+00 +3.05 -1.3114785654E+00 +3.06 -1.3071926593E+00 +3.07 -1.3029346704E+00 +3.08 -1.2987043403E+00 +3.09 -1.2945013996E+00 +3.1 -1.2903255537E+00 +3.11 -1.2861765686E+00 +3.12 -1.2820541890E+00 +3.13 -1.2779581512E+00 +3.14 -1.2738881875E+00 +3.15 -1.2698440733E+00 +3.16 -1.2658255636E+00 +3.17 -1.2618324006E+00 +3.18 -1.2578643439E+00 +3.19 -1.2539211739E+00 +3.2 -1.2500026569E+00 +3.21 -1.2461085419E+00 +3.22 -1.2422386120E+00 +3.23 -1.2383926530E+00 +3.24 -1.2345704421E+00 +3.25 -1.2307717359E+00 +3.26 -1.2269963372E+00 +3.27 -1.2232440378E+00 +3.28 -1.2195146255E+00 +3.29 -1.2158078651E+00 +3.3 -1.2121235755E+00 +3.31 -1.2084615554E+00 +3.32 -1.2048216019E+00 +3.33 -1.2012034905E+00 +3.34 -1.1976070510E+00 +3.35 -1.1940320904E+00 +3.36 -1.1904784150E+00 +3.37 -1.1869458110E+00 +3.38 -1.1834341162E+00 +3.39 -1.1799431468E+00 +3.4 -1.1764727187E+00 +3.41 -1.1730226261E+00 +3.42 -1.1695927154E+00 +3.43 -1.1661828113E+00 +3.44 -1.1627927385E+00 +3.45 -1.1594223013E+00 +3.46 -1.1560713502E+00 +3.47 -1.1527397198E+00 +3.48 -1.1494272430E+00 +3.49 -1.1461337354E+00 +3.5 -1.1428590485E+00 +3.51 -1.1396030273E+00 +3.52 -1.1363655124E+00 +3.53 -1.1331463311E+00 +3.54 -1.1299453334E+00 +3.55 -1.1267623757E+00 +3.56 -1.1235973060E+00 +3.57 -1.1204499641E+00 +3.58 -1.1173201962E+00 +3.59 -1.1142078701E+00 +3.6 -1.1111128408E+00 +3.61 -1.1080349612E+00 +3.62 -1.1049740715E+00 +3.63 -1.1019300514E+00 +3.64 -1.0989027626E+00 +3.65 -1.0958920671E+00 +3.66 -1.0928978113E+00 +3.67 -1.0899198727E+00 +3.68 -1.0869581243E+00 +3.69 -1.0840124343E+00 +3.7 -1.0810826631E+00 +3.71 -1.0781686775E+00 +3.72 -1.0752703635E+00 +3.73 -1.0723875957E+00 +3.74 -1.0695202484E+00 +3.75 -1.0666681783E+00 +3.76 -1.0638312823E+00 +3.77 -1.0610094414E+00 +3.78 -1.0582025361E+00 +3.79 -1.0554104376E+00 +3.8 -1.0526330277E+00 +3.81 -1.0498702023E+00 +3.82 -1.0471218471E+00 +3.83 -1.0443878479E+00 +3.84 -1.0416680742E+00 +3.85 -1.0389624318E+00 +3.86 -1.0362708132E+00 +3.87 -1.0335931096E+00 +3.88 -1.0309292062E+00 +3.89 -1.0282789899E+00 +3.9 -1.0256423687E+00 +3.91 -1.0230192389E+00 +3.92 -1.0204094967E+00 +3.93 -1.0178130267E+00 +3.94 -1.0152297344E+00 +3.95 -1.0126595264E+00 +3.96 -1.0101023039E+00 +3.97 -1.0075579680E+00 +3.98 -1.0050264036E+00 +3.99 -1.0025075318E+00 +4.0 -1.0000012587E+00 +4.01 -9.9750749017E-01 +4.02 -9.9502612827E-01 +4.03 -9.9255707008E-01 +4.04 -9.9010023863E-01 +4.05 -9.8765554399E-01 +4.06 -9.8522289626E-01 +4.07 -9.8280219920E-01 +4.08 -9.8039336082E-01 +4.09 -9.7799630531E-01 +4.1 -9.7561094693E-01 +4.11 -9.7323719995E-01 +4.12 -9.7087497061E-01 +4.13 -9.6852417533E-01 +4.14 -9.6618474019E-01 +4.15 -9.6385658348E-01 +4.16 -9.6153962345E-01 +4.17 -9.5923376957E-01 +4.18 -9.5693894429E-01 +4.19 -9.5465507638E-01 +4.2 -9.5238208792E-01 +4.21 -9.5011990097E-01 +4.22 -9.4786842899E-01 +4.23 -9.4562759817E-01 +4.24 -9.4339734072E-01 +4.25 -9.4117758234E-01 +4.26 -9.3896824874E-01 +4.27 -9.3676925801E-01 +4.28 -9.3458053800E-01 +4.29 -9.3240202505E-01 +4.3 -9.3023364833E-01 +4.31 -9.2807533700E-01 +4.32 -9.2592701447E-01 +4.33 -9.2378860824E-01 +4.34 -9.2166005944E-01 +4.35 -9.1954130054E-01 +4.36 -9.1743226399E-01 +4.37 -9.1533287919E-01 +4.38 -9.1324307143E-01 +4.39 -9.1116278719E-01 +4.4 -9.0909196208E-01 +4.41 -9.0703053173E-01 +4.42 -9.0497843174E-01 +4.43 -9.0293558460E-01 +4.44 -9.0090194157E-01 +4.45 -8.9887744160E-01 +4.46 -8.9686202332E-01 +4.47 -8.9485562532E-01 +4.48 -8.9285817725E-01 +4.49 -8.9086962411E-01 +4.5 -8.8888991183E-01 +4.51 -8.8691898186E-01 +4.52 -8.8495677569E-01 +4.53 -8.8300323078E-01 +4.54 -8.8105828432E-01 +4.55 -8.7912188957E-01 +4.56 -8.7719399073E-01 +4.57 -8.7527453201E-01 +4.58 -8.7336345760E-01 +4.59 -8.7146070069E-01 +4.6 -8.6956621704E-01 +4.61 -8.6767995511E-01 +4.62 -8.6580186170E-01 +4.63 -8.6393188361E-01 +4.64 -8.6206996295E-01 +4.65 -8.6021604427E-01 +4.66 -8.5837008467E-01 +4.67 -8.5653203342E-01 +4.68 -8.5470183981E-01 +4.69 -8.5287945310E-01 +4.7 -8.5106481278E-01 +4.71 -8.4925787746E-01 +4.72 -8.4745860110E-01 +4.73 -8.4566693537E-01 +4.74 -8.4388283189E-01 +4.75 -8.4210624015E-01 +4.76 -8.4033710465E-01 +4.77 -8.3857538894E-01 +4.78 -8.3682104692E-01 +4.79 -8.3507403249E-01 +4.8 -8.3333429953E-01 +4.81 -8.3160179632E-01 +4.82 -8.2987647733E-01 +4.83 -8.2815830464E-01 +4.84 -8.2644723430E-01 +4.85 -8.2474322235E-01 +4.86 -8.2304622483E-01 +4.87 -8.2135618969E-01 +4.88 -8.1967307922E-01 +4.89 -8.1799685479E-01 +4.9 -8.1632747449E-01 +4.91 -8.1466489640E-01 +4.92 -8.1300907863E-01 +4.93 -8.1135996958E-01 +4.94 -8.0971753736E-01 +4.95 -8.0808174336E-01 +4.96 -8.0645254761E-01 +4.97 -8.0482991016E-01 +4.98 -8.0321379105E-01 +4.99 -8.0160413993E-01 +5.0 -8.0000092874E-01 +5.01 -7.9840411967E-01 +5.02 -7.9681367461E-01 +5.03 -7.9522955548E-01 +5.04 -7.9365172419E-01 +5.05 -7.9208013222E-01 +5.06 -7.9051475362E-01 +5.07 -7.8895555208E-01 +5.08 -7.8740249128E-01 +5.09 -7.8585553490E-01 +5.1 -7.8431464661E-01 +5.11 -7.8277978042E-01 +5.12 -7.8125091074E-01 +5.13 -7.7972800346E-01 +5.14 -7.7821102397E-01 +5.15 -7.7669993763E-01 +5.16 -7.7519470982E-01 +5.17 -7.7369529759E-01 +5.18 -7.7220167421E-01 +5.19 -7.7071380838E-01 +5.2 -7.6923166711E-01 +5.21 -7.6775521737E-01 +5.22 -7.6628442616E-01 +5.23 -7.6481925419E-01 +5.24 -7.6335967205E-01 +5.25 -7.6190565187E-01 +5.26 -7.6045716218E-01 +5.27 -7.5901417151E-01 +5.28 -7.5757664837E-01 +5.29 -7.5614455774E-01 +5.3 -7.5471786613E-01 +5.31 -7.5329654960E-01 +5.32 -7.5188057815E-01 +5.33 -7.5046992177E-01 +5.34 -7.4906455045E-01 +5.35 -7.4766443396E-01 +5.36 -7.4626953348E-01 +5.37 -7.4487982944E-01 +5.38 -7.4349529322E-01 +5.39 -7.4211589621E-01 +5.4 -7.4074160982E-01 +5.41 -7.3937240541E-01 +5.42 -7.3800824884E-01 +5.43 -7.3664911416E-01 +5.44 -7.3529497775E-01 +5.45 -7.3394581233E-01 +5.46 -7.3260159063E-01 +5.47 -7.3126228537E-01 +5.48 -7.2992786831E-01 +5.49 -7.2859830532E-01 +5.5 -7.2727357831E-01 +5.51 -7.2595366129E-01 +5.52 -7.2463852824E-01 +5.53 -7.2332815315E-01 +5.54 -7.2202251003E-01 +5.55 -7.2072156818E-01 +5.56 -7.1942530279E-01 +5.57 -7.1813369317E-01 +5.58 -7.1684671453E-01 +5.59 -7.1556434207E-01 +5.6 -7.1428655100E-01 +5.61 -7.1301331652E-01 +5.62 -7.1174460638E-01 +5.63 -7.1048040319E-01 +5.64 -7.0922068435E-01 +5.65 -7.0796542622E-01 +5.66 -7.0671460517E-01 +5.67 -7.0546819755E-01 +5.68 -7.0422617850E-01 +5.69 -7.0298851868E-01 +5.7 -7.0175520260E-01 +5.71 -7.0052620774E-01 +5.72 -6.9930151156E-01 +5.73 -6.9808109151E-01 +5.74 -6.9686492507E-01 +5.75 -6.9565298698E-01 +5.76 -6.9444525229E-01 +5.77 -6.9324170497E-01 +5.78 -6.9204232351E-01 +5.79 -6.9084708645E-01 +5.8 -6.8965597228E-01 +5.81 -6.8846895952E-01 +5.82 -6.8728602321E-01 +5.83 -6.8610714124E-01 +5.84 -6.8493229763E-01 +5.85 -6.8376147189E-01 +5.86 -6.8259464355E-01 +5.87 -6.8143179210E-01 +5.88 -6.8027289707E-01 +5.89 -6.7911793440E-01 +5.9 -6.7796688336E-01 +5.91 -6.7681972864E-01 +5.92 -6.7567645070E-01 +5.93 -6.7453703003E-01 +5.94 -6.7340144708E-01 +5.95 -6.7226968233E-01 +5.96 -6.7114171324E-01 +5.97 -6.7001751900E-01 +5.98 -6.6889708562E-01 +5.99 -6.6778039446E-01 +6.0 -6.6666742690E-01 +6.01 -6.6555816432E-01 diff --git a/scripts/build_quartz_libROM.sh b/scripts/build_quartz_libROM.sh new file mode 100644 index 00000000..458dd662 --- /dev/null +++ b/scripts/build_quartz_libROM.sh @@ -0,0 +1,61 @@ +#!/bin/bash +## An example script to build on LLNL Peloton systems. +## For now, this script assumes intel/ mkl libraries are being used. + +## load some modules +source scripts/modules.quartz + +## set some environment variables. Set them explicitly or use loaded module path (preferred) +## Here we use an explicit path for scalapack to be consistent with the path for the blas libraries and avoid +## benign cmake warnings +##setenv SCALAPACK_ROOT /usr/tce/packages/mkl/mkl-2020.0/lib +#setenv SCALAPACK_ROOT ${MKLROOT} +#setenv HDF5_ROOT /usr/tce/packages/hdf5/hdf5-1.14.0-mvapich2-2.3.6-intel-2022.1.0 +# +## We need to define the cmake blas vendor option here to find the right one. +#set BLAS_VENDOR = Intel10_64lp +# +## manually set the location of BLACS libraries for scalapack +#set BLACS_LIB = ${SCALAPACK_ROOT}/lib/intel64 + +MGMOL_ROOT="$(pwd)" + +INSTALL_DIR=${MGMOL_ROOT}/install_quartz +mkdir -p ${INSTALL_DIR} + +BUILD_DIR=${MGMOL_ROOT}/build_quartz +mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} + +# clone the libROM GitHub repo in BUILD_DIR +USE_LIBROM="On" +LIBROM_PATH=${BUILD_DIR}/libROM +git clone https://github.com/LLNL/libROM +cd libROM +#./scripts/compile.sh -t ./cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake +./scripts/compile.sh +cd ${BUILD_DIR} + +# call cmake +cmake -DCMAKE_TOOLCHAIN_FILE=${MGMOL_ROOT}/cmake_toolchains/quartz.default.cmake \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ + -DUSE_LIBROM=${USE_LIBROM} \ + -DLIBROM_PATH=${LIBROM_PATH} \ + .. + +# -DCMAKE_CXX_COMPILER=mpic++ \ +# -DCMAKE_Fortran_COMPILER=mpif77 \ +# -DMPIEXEC_NUMPROC_FLAG="-n" \ +# -DBLA_VENDOR=${BLAS_VENDOR} \ +# -DSCALAPACK_BLACS_LIBRARY=${BLACS_LIB}/libmkl_blacs_intelmpi_lp64.so \ +# -DCMAKE_BUILD_TYPE=DEBUG \ + +# call make install +make -j 16 +### Currently libROM does not have the installation procedure, +### so copying binary file to installation directory will disrupt the relative path to libROM.so, +### causing a run-time error. +#make install + +# -DBLAS_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ +# -DLAPACK_LIBRARIES=/usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0/lib/intel64/lib \ diff --git a/scripts/build_ubuntu22_openmpi.sh b/scripts/build_ubuntu22_openmpi.sh index 5825759c..421d16b4 100755 --- a/scripts/build_ubuntu22_openmpi.sh +++ b/scripts/build_ubuntu22_openmpi.sh @@ -23,7 +23,7 @@ cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \ -DMPIEXEC_PREFLAGS="-report-bindings;--map-by;core;-bind-to;core" \ -DMGMOL_WITH_CLANG_FORMAT=ON \ -DCMAKE_PREFIX_PATH=${HOME}/bin \ - -D CMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ + -DCMAKE_CXX_FLAGS="-Wall -pedantic -Wextra" \ .. # call make install diff --git a/scripts/modules.quartz b/scripts/modules.quartz new file mode 100644 index 00000000..82433acb --- /dev/null +++ b/scripts/modules.quartz @@ -0,0 +1,21 @@ +### choose either gcc or intel +#module load intel/2022.1.0 +module load gcc/11.2.1 + +module load cmake +module load hdf5-parallel/1.14.0 +module load boost + +### choose either one +module load mkl-interfaces +#module load mkl + +module load python + +### manually add boost path +#setenv LD_LIBRARY_PATH /usr/tce/packages/boost/boost-1.80.0-mvapich2-2.3.6-gcc-10.3.1/lib:$LD_LIBRARY_PATH + +#setenv MKLROOT $LIBRARY_PATH +#setenv MKLROOT /usr/tce/packages/mkl/mkl-2022.1.0/mkl/2022.1.0 +#setenv HDF5ROOT $LD_LIBRARY_PATH +#setenv HDF5ROOT diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3b0207ee..8eb9352b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,7 @@ +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h.in + ${CMAKE_CURRENT_SOURCE_DIR}/mgmol_config.h @ONLY) + add_subdirectory(DistMatrix) add_subdirectory(linear_algebra) add_subdirectory(local_matrices) @@ -8,7 +11,8 @@ add_subdirectory(radial) add_subdirectory(sparse_linear_algebra) add_subdirectory(tools) -set(link_libs mgmol_distmatrix +set(link_libs + mgmol_distmatrix mgmol_linear_algebra mgmol_local_matrices mgmol_numerical_kernels @@ -153,13 +157,23 @@ set(SOURCES magma_singleton.cc ChebyshevApproximation.cc ChebyshevApproximationInterface.cc + rom.cc ) +if(USE_LIBROM) + list(APPEND SOURCES + rom_workflows.cc) +endif(USE_LIBROM) + add_library(mgmol_src ${SOURCES}) target_include_directories(mgmol_src PRIVATE ${HDF5_INCLUDE_DIRS}) target_include_directories(mgmol_src PRIVATE ${Boost_INCLUDE_DIRS}) -target_include_directories (mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(mgmol_src PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +if(USE_LIBROM) + target_include_directories (mgmol_src PRIVATE ${LIBROM_INCLUDES}) + target_link_libraries(mgmol_src ${LIBROM_LIB}) +endif(USE_LIBROM) target_link_libraries(mgmol_src ${link_libs}) if(${MGMOL_WITH_MAGMA}) @@ -191,3 +205,23 @@ endif (${MGMOL_WITH_LIBXC}) install(TARGETS mgmol-opt DESTINATION bin) +# build ROM executable +if(USE_LIBROM) + add_executable(mgmol-rom rom_main.cc) + target_include_directories (mgmol-rom PRIVATE ${Boost_INCLUDE_DIRS}) + + target_link_libraries(mgmol-rom mgmol_src ${link_libs}) + target_link_libraries(mgmol-rom ${SCALAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_LIBRARIES}) + target_link_libraries(mgmol-rom ${HDF5_HL_LIBRARIES}) + target_link_libraries(mgmol-rom ${BLAS_LIBRARIES}) + target_link_libraries(mgmol-rom ${LAPACK_LIBRARIES}) + target_link_libraries(mgmol-rom ${Boost_LIBRARIES}) + if (${OPENMP_CXX_FOUND}) + target_link_libraries(mgmol-rom OpenMP::OpenMP_CXX) + endif() + if(${MGMOL_WITH_LIBXC}) + target_link_libraries(mgmol-rom ${LIBXC_DIR}/lib/libxc.a) + endif (${MGMOL_WITH_LIBXC}) + install(TARGETS mgmol-rom DESTINATION bin) +endif(USE_LIBROM) \ No newline at end of file diff --git a/src/Control.cc b/src/Control.cc index 7d8dc33f..ce3023e3 100644 --- a/src/Control.cc +++ b/src/Control.cc @@ -1917,6 +1917,10 @@ void Control::setOptions(const boost::program_options::variables_map& vm) // synchronize all processors sync(); + +#ifdef MGMOL_HAS_LIBROM + setROMOptions(vm); +#endif } int Control::checkOptions() @@ -2051,3 +2055,66 @@ void Control::printPoissonOptions(std::ostream& os) } os << std::endl; } + +void Control::setROMOptions(const boost::program_options::variables_map& vm) +{ + printWithTimeStamp("Control::setROMOptions()...", std::cout); + + if (onpe0) + { + std::string str = vm["ROM.stage"].as(); + if (str.compare("offline") == 0) + rom_pri_option.rom_stage = ROMStage::OFFLINE; + else if (str.compare("online") == 0) + rom_pri_option.rom_stage = ROMStage::ONLINE; + else if (str.compare("build") == 0) + rom_pri_option.rom_stage = ROMStage::BUILD; + else if (str.compare("none") == 0) + rom_pri_option.rom_stage = ROMStage::UNSUPPORTED; + + rom_pri_option.restart_file_fmt = vm["ROM.offline.restart_filefmt"].as(); + rom_pri_option.restart_file_minidx = vm["ROM.offline.restart_min_idx"].as(); + rom_pri_option.restart_file_maxidx = vm["ROM.offline.restart_max_idx"].as(); + rom_pri_option.basis_file = vm["ROM.offline.basis_file"].as(); + + rom_pri_option.save_librom_snapshot = vm["ROM.offline.save_librom_snapshot"].as(); + } // onpe0 + + // synchronize all processors + syncROMOptions(); +} + +void Control::syncROMOptions() +{ + if (onpe0 && verbose > 0) + (*MPIdata::sout) << "Control::syncROMOptions()" << std::endl; + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + + mmpi.bcast(rom_pri_option.restart_file_fmt, comm_global_); + mmpi.bcast(rom_pri_option.basis_file, comm_global_); + + auto bcast_check = [](int mpirc) { + if (mpirc != MPI_SUCCESS) + { + (*MPIdata::sout) << "MPI Bcast of Control failed!!!" << std::endl; + MPI_Abort(comm_global_, 2); + } + }; + + short rom_stage = (short)static_cast(rom_pri_option.rom_stage); + int mpirc; + mpirc = MPI_Bcast(&rom_stage, 1, MPI_SHORT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.restart_file_minidx, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.restart_file_maxidx, 1, MPI_INT, 0, comm_global_); + bcast_check(mpirc); + + mpirc = MPI_Bcast(&rom_pri_option.save_librom_snapshot, 1, MPI_C_BOOL, 0, comm_global_); + bcast_check(mpirc); + + rom_pri_option.rom_stage = static_cast(rom_stage); +} \ No newline at end of file diff --git a/src/Control.h b/src/Control.h index f1fd8779..8fb08fc5 100644 --- a/src/Control.h +++ b/src/Control.h @@ -13,6 +13,9 @@ #include "Species.h" #include "Timeout.h" +/* enumeration and option variables for libROM */ +#include "rom_Control.h" + #include #include #include @@ -220,6 +223,9 @@ class Control void printRestartLink(); + /* libROM related options */ + ROMPrivateOptions rom_pri_option; + public: static Control* instance() { @@ -707,6 +713,11 @@ class Control } bool AtomsMove() { return (atoms_dyn_ != 0); } + + /* ROM-related options */ + void setROMOptions(const boost::program_options::variables_map& vm); + void syncROMOptions(); + const ROMPrivateOptions getROMOptions() { return rom_pri_option; } }; #endif diff --git a/src/MGmol.cc b/src/MGmol.cc index 962bdf51..d885d6e6 100644 --- a/src/MGmol.cc +++ b/src/MGmol.cc @@ -106,7 +106,7 @@ extern Timer ions_setupInteractingIons_tm; extern Timer ions_setup_tm; extern Timer updateCenters_tm; -#include "Signal.h" +#include "mgmol_Signal.h" std::set Signal::recv_; template @@ -1143,6 +1143,19 @@ void MGmol::cleanup() if (ierr < 0) os_ << "WARNING: writing restart data failed!!!" << std::endl; + +#ifdef MGMOL_HAS_LIBROM + // Save orbital snapshots + if (ct.getROMOptions().save_librom_snapshot > 0 && ct.AtomsDynamic() == AtomsDynamicType::Quench) + { + ierr = save_orbital_snapshot( + filename, *current_orbitals_); + + if (ierr < 0) + os_ << "WARNING: writing ROM snapshot data failed!!!" << std::endl; + } +#endif + } MPI_Barrier(comm_); diff --git a/src/MGmol.h b/src/MGmol.h index 23b507c4..2a0d0783 100644 --- a/src/MGmol.h +++ b/src/MGmol.h @@ -10,6 +10,8 @@ #ifndef MGMOL_H #define MGMOL_H +#include "mgmol_config.h" + #include "Energy.h" #include "GridFuncVector.h" #include "Hamiltonian.h" @@ -102,7 +104,7 @@ class MGmol : public MGmolInterface double total_energy_; ConstraintSet* constraints_; - OrbitalsExtrapolation* orbitals_extrapol_; + OrbitalsExtrapolation* orbitals_extrapol_ = nullptr; float md_time_; int md_iteration_; @@ -301,6 +303,11 @@ class MGmol : public MGmolInterface { forces_->force(orbitals, ions); } + + OrbitalsType* loadOrbitalFromRestartFile(const std::string filename); +#ifdef MGMOL_HAS_LIBROM + int save_orbital_snapshot(std::string snapshot_dir, OrbitalsType& orbitals); +#endif }; // Instantiate static variables here to avoid clang warnings template diff --git a/src/MGmolInterface.h b/src/MGmolInterface.h index d5795733..d9ab1446 100644 --- a/src/MGmolInterface.h +++ b/src/MGmolInterface.h @@ -19,10 +19,12 @@ class MGmolInterface virtual ~MGmolInterface() {} + virtual void setup() = 0; virtual int setupFromInput(const std::string input_file) = 0; virtual int setupLRs(const std::string input_file) = 0; virtual int setupConstraintsFromInput(const std::string input_file) = 0; virtual void run() = 0; + }; #endif diff --git a/src/OrbitalsExtrapolation.h b/src/OrbitalsExtrapolation.h index 9f2348f7..47c1a21b 100644 --- a/src/OrbitalsExtrapolation.h +++ b/src/OrbitalsExtrapolation.h @@ -44,7 +44,7 @@ class OrbitalsExtrapolation virtual short getNumOrbitalExtrapolations() { return 0; } protected: - OrbitalsType* orbitals_minus1_; + OrbitalsType* orbitals_minus1_ = nullptr; }; #endif diff --git a/src/main.cc b/src/main.cc index 6f933ffe..3077aed9 100644 --- a/src/main.cc +++ b/src/main.cc @@ -58,8 +58,9 @@ using namespace std; #include #include -#include -namespace po = boost::program_options; +// #include +// namespace po = boost::program_options; +#include "OptionDescription.h" //#include "MemTrack.h" @@ -133,308 +134,21 @@ int main(int argc, char** argv) // Declare a group of options that will be // allowed only on command line po::options_description generic("Generic options"); - generic.add_options()("version,v", "print version string")("help,h", - "produce help message")("check", "check input")("config,c", - po::value(&config_file)->default_value("mgmol.cfg"), - "name of a file of a configuration.")("atomicCoordinates,i", - po::value>(), - "coordinates filename")("LRsFilename,l", - po::value(&lrs_filename), "LRs filename"); + setupGenericOption(generic, config_file, lrs_filename); // Declare a group of options (with default when appropriate) that // will be allowed in config file po::options_description config("Configuration"); - config.add_options()( - "spin", po::value(), "system total spin")("verbosity", - po::value()->default_value(1), "verbosity level")( - "constraintsFilename", po::value(&constraints_filename), - "Name of file with list of constraints")("xcFunctional", - po::value()->required(), "XC functional: LDA or PBE")( - "FDtype", po::value()->default_value("Mehrstellen"), - "Finite Difference scheme")("charge", - po::value()->default_value(0), "system total charge")( - "Domain.lx", po::value()->required(), - "domain dimension in x direction")("Domain.ly", - po::value()->required(), - "domain dimension in y direction")("Domain.lz", - po::value()->required(), - "domain dimension in z direction")("Domain.ox", - po::value()->required(), "domain origin in x direction")( - "Domain.oy", po::value()->required(), - "domain origin in y direction")("Domain.oz", - po::value()->required(), "domain origin in z direction")( - "Mesh.nx", po::value()->required(), - "mesh dimension in x direction")("Mesh.ny", - po::value()->required(), - "mesh dimension in y direction")("Mesh.nz", - po::value()->required(), - "mesh dimension in z direction")("Potentials.pseudopotential", - po::value>()->multitoken(), - "pseudopotentials list")("Potentials.external", - po::value>()->multitoken(), - "external potentials list")("Potentials.binExternal", - po::value()->default_value(true), - "binary external potential")("Restart.input_filename", - po::value()->default_value(""), - "Read restart filename/directory")("Restart.input_level", - po::value()->default_value(0), - "Read restart level")("Restart.input_type", - po::value()->default_value("distributed"), - "Read restart type: distributed or single_file")( - "Restart.output_filename", - po::value()->default_value("auto"), - "Dump restart filename/directory")("Restart.output_level", - po::value()->default_value(3), - "Write restart level")("Restart.output_type", - po::value()->default_value("distributed"), - "Write restart type: distributed or single_file")( - "Restart.interval", po::value()->default_value(1000), - "Restart frequency")("Restart.rescale_v", - po::value()->default_value(1.), - "rescaling factor velocity of all atoms")("Poisson.bcx", - po::value()->default_value("periodic"), - "boundary condition x")("Poisson.bcy", - po::value()->default_value("periodic"), - "boundary condition y")("Poisson.bcz", - po::value()->default_value("periodic"), - "boundary condition z")("Poisson.diel", - po::value()->default_value("off"), - "continuum solvent: on/off")("Run.type", - po::value()->default_value("QUENCH"), "Run type")( - "Quench.solver", po::value()->default_value("ABPG"), - "Iterative solver for quench")("Quench.max_steps", - po::value()->default_value(200), - "Max. steps in loose quench")("Quench.max_steps_tight", - po::value()->default_value(1000), - "Max. steps in tight quench")("Quench.atol", - po::value()->default_value(1.e-12), - "Abs. tol. in quench convergence")("Quench.rtol", - po::value()->default_value(-1.), - "Rel. tol. in quench convergence")("Quench.conv_criterion", - po::value()->default_value("deltaE"), - "Convergence criterion")("Quench.MLWC", po::value(), - "Compute MLWC in quench")("Quench.MLWF", - po::value()->default_value(false), - "Compute MLWF (apply rotation) in quench")( - "Quench.num_lin_iterations", - po::value()->default_value(0), - "Number of iterations without potential update in quench")( - "Quench.preconditioner_num_levels", - po::value()->default_value(2), - "Number of levels for MG preconditioner")( - "Quench.spread_penalty_damping", - po::value()->default_value(0.), - "Spread penalty damping factor")("Quench.spread_penalty_target", - po::value()->default_value(2.), - "Spread penalty target")("SpreadPenalty.type", - po::value()->default_value("individual"), - "Spread penalty type (individual,volume)")( - "SpreadPenalty.damping", po::value()->default_value(1.), - "Spread penalty damping factor")("SpreadPenalty.target", - po::value()->default_value(-1.), - "Spread penalty target")("SpreadPenalty.alpha", - po::value()->default_value(0.), "Spread penalty factor")( - "MD.num_steps", po::value()->default_value(1), - "number of MD steps")("MD.last_step", - po::value()->default_value(-1), "last MD step")( - "MD.dt", po::value(), "time step for MD (a.u.)")( - "MD.print_interval", po::value()->default_value(1), - "print intervale for MD data")("MD.print_directory", - po::value()->default_value("MD"), - "print directory for MD data")("MD.thermostat", - po::value()->default_value("OFF"), - "MD thermostat: ON or OFF")("MD.remove_mass_center_motion", - po::value()->default_value(true), - "Remove mass center motion")("MD.type", - po::value()->default_value("BOMD"), "MD type: BOMD")( - "GeomOpt.type", po::value()->default_value("LBFGS"), - "Geometry optimization algorithm")("GeomOpt.tol", - po::value()->default_value(4.e-4), - "Tolerance on forces for Geometry optimization")( - "GeomOpt.max_steps", po::value()->default_value(1), - "max. number of Geometry optimization steps")("GeomOpt.dt", - po::value(), "Delta t for trial pseudo-time steps")( - "atomicCoordinates", po::value>(), - "coordinates filename")("Thermostat.type", - po::value()->default_value("Langevin"), - "Thermostat type")("Thermostat.temperature", - po::value()->default_value(-1.), - "Thermostat temperature")("Thermostat.relax_time", - po::value()->default_value(-1.), - "Thermostat relaxation time")("Thermostat.width", - po::value()->default_value(-1.), - "Thermostat width (for SCALING)")("Orbitals.nempty", - po::value()->default_value(0), - "Number of empty orbitals")("Orbitals.initial_type", - po::value()->default_value("random"), - "initial orbitals type")("Orbitals.initial_width", - po::value()->default_value(10000.), - "initial orbitals radius")("Orbitals.temperature", - po::value()->default_value(0.), - "electronic temperature [K]")("Orbitals.bcx", - po::value()->default_value("periodic"), - "orbitals boundary condition x")("Orbitals.bcy", - po::value()->default_value("periodic"), - "orbitals boundary condition y")("Orbitals.bcz", - po::value()->default_value("periodic"), - "orbitals boundary condition z")("ProjectedMatrices.solver", - po::value()->default_value("exact"), - "solver for projected matrices")("ProjectedMatrices.printMM", - po::value()->default_value(false), - "print projected matrices in MM format")( - "LocalizationRegions.radius", - po::value()->default_value(1000.), - "Localization regions radius")("LocalizationRegions.adaptive", - po::value()->default_value(true), - "Localization regions adaptivity")( - "LocalizationRegions.move_tol", - po::value()->default_value(1000.), - "Localization regions move tolerance")( - "Parallel.atomic_info_radius", - po::value()->default_value(8.), - "Max. distance for atomic data communication")( - "LoadBalancing.alpha", po::value()->default_value(0.0), - "Parameter for computing bias for load balancing algo")( - "LoadBalancing.damping_tol", - po::value()->default_value(0.9), - "Damping parameter for computing bias for load balancing algo")( - "LoadBalancing.max_iterations", - po::value()->default_value(50), - "Maximum number of iterations for load balancing algo")( - "LoadBalancing.modulo", po::value()->default_value(1), - "Modulos or parameter to control how often clusters are " - "recomputed during md")("LoadBalancing.output_file", - po::value()->default_value(""), - "Output file for dumping cluster information in vtk format"); + setupConfigOption(config, constraints_filename); +#ifdef MGMOL_HAS_LIBROM + // ROM configuration options + setupROMConfigOption(config); +#endif // Hidden options, will be allowed in config file, but will not be // shown to the user. po::options_description hidden("Hidden options"); - hidden.add_options()("Quench.interval_print_residual", - po::value()->default_value(0), - "Print interval for residual in quench (0 for none)")( - "Quench.required_tol", po::value()->default_value(1000.), - "required tolerance (code stops if not reached)")( - "Quench.compute_cond_Gram", - po::value()->default_value(false), - "Compute condition number of S during quench")( - "Quench.min_Gram_eigenvalue", - po::value()->default_value(0.), - "min. eigenvalue for Gram matrix")("Quench.step_length", - po::value()->default_value(-1.), - "Step length for corrections in quench")("Quench.ortho_freq", - po::value()->default_value(1000), - "Orthonormalization frequency")( - "Quench.pair_mlwf_distance_threshold", - po::value()->default_value(4.), - "Max. distance between pairs for MLWF transform")( - "AOMM.kernel_radius", po::value()->default_value(-1.), - "Radius of kernel functions in AOMM algorithm")( - "AOMM.threshold_factor", po::value()->default_value(-1.), - "Multiplicative factor of kernel radius to set threshold in " - "AOMM projectors dropping")("Orbitals.type", - po::value()->default_value("NO"), - "orbital type")("Orbitals.dotProduct", - po::value()->default_value("diagonal"), - "orbital dot product type")("Orbitals.overallocate_factor", - po::value()->default_value(1.2), - "safety factor to use for static allocation of orbitals")( - "Potentials.filterPseudo", - po::value()->default_value('f'), - "filter")("Poisson.solver", - po::value()->default_value("CG"), "solver")( - "Poisson.e0", po::value()->default_value(78.36), - "continuum solvent: epsilon0")("Poisson.rho0", - po::value()->default_value(0.0004), - "continuum solvent: rho0")("Poisson.beta", - po::value()->default_value(1.3), - "continuum solvent: beta")("Poisson.FDtype", - po::value()->default_value("Mehrstellen"), "FDtype")( - "Poisson.nu1", po::value()->default_value(2), "nu_1")( - "Poisson.nu2", po::value()->default_value(2), "nu_2")( - "Poisson.max_steps", po::value()->default_value(20), - "max. nb. steps Poisson solver")("Poisson.max_steps_initial", - po::value()->default_value(20), - "max. nb. steps Poisson solver in first solve")( - "Poisson.max_levels", po::value()->default_value(10), - "max. nb. MG levels Poisson solver")("Poisson.reset", - po::value()->default_value(false), - "reset Hartree potential at each MD step")("ABPG.m", - po::value()->default_value(1), - "History length for Anderson extrapolation")("ABPG.beta", - po::value()->default_value(1.), - "beta for Anderson extrapolation")("NLCG.parallel_transport", - po::value()->default_value(true), - "Turn ON/OFF parallel transport algorithm")( - "MD.extrapolation_type", po::value()->default_value(1), - "MD extrapolation type")("MD.compute_cond_Gram", - po::value()->default_value(false), - "Compute condition number of S at end of quench")( - "MD.min_Gram_eigenvalue", po::value()->default_value(0.), - "min. eigenvalue for Gram matrix")( - "ShortSightedInverse.spread_factor", - po::value()->default_value(2.), - "Shortsighted spread factor")("ShortSightedInverse.tol", - po::value()->default_value(1.e-10), - "Shortsighted tolerance")("ShortSightedInverse.krylov_dim", - po::value()->default_value(10), - "Shortsighted Krylov space max. dimension")( - "ShortSightedInverse.max_iterations", - po::value()->default_value(10), - "Shortsighted max. number of GMRES iterations")( - "ShortSightedInverse.ilu_type", - po::value()->default_value("ILU"), - "Shortsighted ILU type: ILU or ILUT")( - "ShortSightedInverse.ilu_drop_tol", - po::value()->default_value(1.e-5), - "Shortsighted ILU dropping tolerance")( - "ShortSightedInverse.ilu_filling_level", - po::value()->default_value(0), - "Shortsighted ILU filling level")( - "ShortSightedInverse.ilut_max_fill", - po::value()->default_value(10000), - "Shortsighted max. filling for ILUT")("Coloring.algo", - po::value()->default_value("RLF"), - "Coloring algorithm: RLF or Greedy")("Coloring.scope", - po::value()->default_value("local"), - "Coloring scope: local or global")( - "LocalizationRegions.min_distance", - po::value()->default_value(0.), - "min. distance between Localization centers")( - "LocalizationRegions.extrapolation_scheme", - po::value()->default_value("linear"), - "Extrapolation order for localization centers")( - "LocalizationRegions.computation", - po::value()->default_value(0), - "Flag for computing new centers from extrapolated orbitals.")( - "DensityMatrix.mixing", po::value()->default_value(-1.), - "Mixing coefficient for Density Matrix")("DensityMatrix.solver", - po::value()->default_value("Mixing"), - "Algorithm for updating Density Matrix: Mixing, MVP, HMVP")( - "DensityMatrix.nb_inner_it", - po::value()->default_value(3), - "Max. number of inner iterations in DM optimization")( - "DensityMatrix.algo", - po::value()->default_value("Diagonalization"), - "Algorithm for computing Density Matrix. " - "Diagonalization or SP2 or Chebyshev.")("DensityMatrix.use_old", - po::value()->default_value(true), - "Start DM optimization with matrix of previous WF step")( - "DensityMatrix.approximation_order", - po::value()->default_value(500), - "Polynomial order for Chebyshev approximation " - "of density matrix. ")("DensityMatrix.approximation_ndigits", - po::value()->default_value(1), - "Number of digits of precision for Chebyshev " - "approximation of density matrix. ")( - "DensityMatrix.approximation_power_maxits", - po::value()->default_value(100), - "Maximum number of iterations for power method " - "to compute interval for Chebyshev " - "approximation of density matrix. ")("DensityMatrix.tol", - po::value()->default_value(1.e-7), - "tolerance, used in iterative DM computation convergence " - "criteria"); + setupHiddenOption(hidden); po::options_description cmdline_options; cmdline_options.add(generic); diff --git a/src/md.cc b/src/md.cc index 1adab46b..745383cf 100644 --- a/src/md.cc +++ b/src/md.cc @@ -31,7 +31,7 @@ #include "ProjectedMatricesMehrstellen.h" #include "ProjectedMatricesSparse.h" #include "Rho.h" -#include "Signal.h" +#include "mgmol_Signal.h" #include "SpreadsAndCenters.h" #include "tools.h" @@ -641,6 +641,18 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) count++; } +#ifdef MGMOL_HAS_LIBROM + // Save orbital snapshots + if (ct.getROMOptions().save_librom_snapshot > 0) + { + int ierr = save_orbital_snapshot( + ct.md_print_filename + "_mdstep" + std::to_string(mdstep), **orbitals); + + if (ierr < 0) + os_ << "WARNING md(): writing ROM snapshot data failed!!!" << std::endl; + } +#endif + printWithTimeStamp("dumped restart file...", std::cout); } @@ -674,5 +686,146 @@ void MGmol::md(OrbitalsType** orbitals, Ions& ions) delete orbitals_extrapol_; } +template +OrbitalsType* MGmol::loadOrbitalFromRestartFile(const std::string filename) +{ + MGmol_MPI& mmpi(*(MGmol_MPI::instance())); + Control& ct = *(Control::instance()); + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + /* For now, we only consider double-precision hdf5 I/O. */ + assert(ct.restart_info > 3); + assert((ct.AtomsDynamic() == AtomsDynamicType::MD) || (ct.AtomsDynamic() == AtomsDynamicType::Quench)); + + HDFrestart h5file(filename, myPEenv, ct.out_restart_file_type); + int ierr; + + OrbitalsType *restart_orbitals = nullptr; + + /* This corresponds to MGmol::initial */ + { + // If not an initial run read data from files + if (ct.isLocMode()) + { + std::string name = "ExtrapolatedFunction"; + if (ct.verbose > 0) + printWithTimeStamp( + "read LRs from ExtrapolatedFunction database...", os_); + int n = read_restart_lrs(h5file, name); + if (n == 0) + { + if (ct.verbose > 0) + printWithTimeStamp("read LRs from Function database...", os_); + name = "Function"; + n = read_restart_lrs(h5file, name); + } + if (n < 0) return nullptr; + + if (n > 0) lrs_->setup(); + } + + // Copy from current orbital, instead of constructing brand-new one + restart_orbitals = new OrbitalsType("ForLoading", *current_orbitals_, false); + + // TODO(kevin): do we need this? + // increaseMemorySlotsForOrbitals(); + + // If not an initial run read data from files + md_time_ = h5file.getMDTimeFromFile(); + md_iteration_ = h5file.getMDstepFromFile(); + + /* This corresponds to MGmol::read_restart_data */ + { + // TODO(kevin): do we need to load rho? + // read_rho_and_pot_hdf5(h5file, *rho_); + + ierr = restart_orbitals->read_func_hdf5(h5file); + } // read_restart_data + + // TODO(kevin): Do we need this routine? + restart_orbitals->applyMask(); + + } // initial() + + /* This corresponds to MGmol::md */ + { + // TODO(kevin): do we need this? + // if (!ct.override_restart) + // { + // if (onpe0) os_ << "Use restart file to initialize MD..." << std::endl; + // stepper->init(h5file); + // } + + int flag_extrapolated_data = 0; + if (onpe0) + { + flag_extrapolated_data + = h5file.dset_exists("ExtrapolatedFunction0000"); + if (flag_extrapolated_data == 0) + flag_extrapolated_data + = h5file.dset_exists("ExtrapolatedFunction0"); + } + MPI_Bcast(&flag_extrapolated_data, 1, MPI_INT, 0, comm_); + + /* + If extrapolated function exists, + then function is set as previous orbitals, + while the extrapolated function is set as the current orbitals. + This is how the restart file is saved via dumprestartFile. + */ + if (flag_extrapolated_data) + { + orbitals_extrapol_ = OrbitalsExtrapolationFactory::create( + ct.WFExtrapolation()); + + if (onpe0) os_ << "Create new orbitals_minus1..." << std::endl; + + orbitals_extrapol_->setupPreviousOrbitals(&restart_orbitals, + proj_matrices_, lrs_, local_cluster_, currentMasks_, + corrMasks_, h5file); + + // need to reset a few things as we just read new orbitals + restart_orbitals->computeGramAndInvS(); + dm_strategy_->update(); + } + + DFTsolver::setItCountLarge(); + + // check if we are restarting from an MD dump + // no extrapolated functions -> atomic positions were extrapolated + // in stepper->init() + if (!flag_extrapolated_data) + { + moveVnuc(*ions_); + } + + /* main workflow delete h5f_file_ here, meaning the loading is over. */ + // delete h5f_file_; + // h5f_file_ = nullptr; + } // md() + + ierr = h5file.close(); + mmpi.allreduce(&ierr, 1, MPI_MIN); + if (ierr < 0) + { + if (onpe0) + (*MPIdata::serr) + << "loadRestartFile: cannot close file..." << std::endl; + return nullptr; + } + + /* + In returning restart_orbitals, + we hope that the wavefunctions in restart_orbitals are all set. + At least the following functions should return proper data loaded from the file: + + restart_orbitals->getLocNumpt() + restart_orbitals->chromatic_number() + restart_orbitals->getPsi(idx) (for int idx) + */ + return restart_orbitals; +} + template class MGmol; template class MGmol; diff --git a/src/mgmol_config.h.in b/src/mgmol_config.h.in new file mode 100644 index 00000000..0b099f6c --- /dev/null +++ b/src/mgmol_config.h.in @@ -0,0 +1,19 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// Description: A configuration header for mgmol. +// + +#ifndef INCLUDED_MGMOL_CONFIG_H +#define INCLUDED_MGMOL_CONFIG_H + +/* Have google test library. */ +#cmakedefine MGMOL_HAS_LIBROM + +#endif diff --git a/src/rom.cc b/src/rom.cc new file mode 100644 index 00000000..f269f4b8 --- /dev/null +++ b/src/rom.cc @@ -0,0 +1,61 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "mgmol_config.h" +#ifdef MGMOL_HAS_LIBROM + +#include "LocGridOrbitals.h" +#include "MGmol.h" + +#include "librom.h" + +#include +#include +#include +#include + +// Save the wavefunction snapshots +template +int MGmol::save_orbital_snapshot(std::string file_path, OrbitalsType& orbitals) +{ + std::string snapshot_filename = file_path; + struct stat s; + if (stat(file_path.c_str(), &s) == 0) + { + if (s.st_mode & S_IFDIR) + snapshot_filename = file_path + "/orbital"; + else if (s.st_mode & S_IFREG) + { + snapshot_filename = file_path + "_orbital"; + } + else + { + std::cout << file_path << " exists but is not a directory or a file." << std::endl; + return 1; + } + } + + const int dim = orbitals.getLocNumpt(); + const int totalSamples = orbitals.chromatic_number(); + + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, snapshot_filename); + + for (int i = 0; i < totalSamples; ++i) + basis_generator.takeSample(orbitals.getPsi(i)); + + basis_generator.writeSnapshot(); + + return 0; +} + +template class MGmol; +template class MGmol; + +#endif // MGMOL_HAS_LIBROM diff --git a/src/rom_Control.h b/src/rom_Control.h new file mode 100644 index 00000000..d1fd6854 --- /dev/null +++ b/src/rom_Control.h @@ -0,0 +1,42 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef ROM_CONTROL_H +#define ROM_CONTROL_H + +#include +#include +#include +#include +#include + +enum class ROMStage +{ + OFFLINE, + ONLINE, + RESTORE, // TODO(kevin): what stage is this? + BUILD, + UNSUPPORTED +}; + +/* Stored as a private member variable of Control class */ +struct ROMPrivateOptions +{ + ROMStage rom_stage = ROMStage::UNSUPPORTED; + + std::string restart_file_fmt = ""; + int restart_file_minidx = -1; + int restart_file_maxidx = -1; + std::string basis_file = ""; + + /* save librom snapshot matrix at FOM simulation. */ + bool save_librom_snapshot = false; +}; + +#endif // ROM_CONTROL_H diff --git a/src/rom_main.cc b/src/rom_main.cc new file mode 100644 index 00000000..61435c9e --- /dev/null +++ b/src/rom_main.cc @@ -0,0 +1,341 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +// +// main.cc +// +// Description: +// Real grid, finite difference, molecular dynamics program +// for with nonorthogonal localized orbitals. +// +// Uses Mehrstellen operators, multigrid accelerations, and +// non-local pseudopotentials. +// +// Includes LDA and PBE exchange and correlation functionals. +// +// Units: +// Potentials, eigenvalues and operators in Rydberg +// Energies in Hartree +// +#include "rom_workflows.h" + +//#include "MemTrack.h" + +int main(int argc, char** argv) +{ + // change handling of memory allocation errors + set_new_handler(noMoreMemory); + + cout.sync_with_stdio(); + + int mpirc = MPI_Init(&argc, &argv); + if (mpirc != MPI_SUCCESS) + { + cerr << "MPI Initialization failed!!!" << endl; + MPI_Abort(MPI_COMM_WORLD, 0); + } + MPI_Comm_rank(MPI_COMM_WORLD, &mype); + assert(mype > -1); + onpe0 = (mype == 0); + +#ifdef HAVE_MAGMA + std::cout << "ROM does not support MAGMA!\n" << std::endl; + return 1; +#endif + + string input_file(""); + string lrs_filename; + string constraints_filename(""); + bool tcheck = false; + + float total_spin = 0.; + bool with_spin = false; + + po::variables_map vm; + + // use configure file if it can be found + // std::string config_filename("mgmol.cfg"); + + // read options from PE0 only + if (onpe0) try + { + string config_file; + + // Declare a group of options that will be + // allowed only on command line + po::options_description generic("Generic options"); + setupGenericOption(generic, config_file, lrs_filename); + + // Declare a group of options (with default when appropriate) that + // will be allowed in config file + po::options_description config("Configuration"); + setupConfigOption(config, constraints_filename); + // ROM configuration options + setupROMConfigOption(config); + + // Hidden options, will be allowed in config file, but will not be + // shown to the user. + po::options_description hidden("Hidden options"); + setupHiddenOption(hidden); + + po::options_description cmdline_options; + cmdline_options.add(generic); + + po::options_description config_file_options; + config_file_options.add(config).add(hidden); + + po::options_description visible("Allowed options"); + visible.add(generic).add(config); + + po::positional_options_description pd; + pd.add("atomicCoordinates", -1); + + store(po::command_line_parser(argc, argv) + .options(cmdline_options) + .positional(pd) + .run(), + vm); + notify(vm); + + ifstream ifs(config_file.c_str()); + if (!ifs) + { + cout << "can not open config file: " << config_file << "\n"; + return 0; + } + else + { + store(parse_config_file(ifs, config_file_options), vm); + notify(vm); + } + + if (vm.count("help")) + { + if (onpe0) cout << visible << "\n"; + return 0; + } + + if (vm.count("version")) + { + if (onpe0) + { +#ifdef GITHASH +#define xstr(x) #x +#define LOG(x) cout << " MGmol: git_hash " << xstr(x) << endl; + LOG(GITHASH); + cout << endl; +#endif + } + return 0; + } + + if (vm.count("check")) + { + tcheck = true; + } + if (vm.count("spin")) + { + total_spin = vm["spin"].as(); + with_spin = true; + if (onpe0) cout << "Spin was set to " << total_spin << endl; + } + if (vm.count("atomicCoordinates")) + { + if (onpe0) + cout << "Input files is: " + << vm["atomicCoordinates"].as>()[0] + << "\n"; + input_file = vm["atomicCoordinates"].as>()[0]; + } + else + { + if (vm["Restart.input_level"].as() == 0) + { + throw std::runtime_error( + "ERROR: No coordinates file provided!!!"); + } + } + + if (onpe0) cout << "Spin: " << total_spin << "\n"; + + } // try + catch (exception& e) + { + cerr << e.what() << "\n"; + return 1; + } + + MGmol_MPI::setup(MPI_COMM_WORLD, std::cout, with_spin); + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + MPI_Comm global_comm = mmpi.commGlobal(); + + Control::setup(global_comm, with_spin, total_spin); + Control& ct = *(Control::instance()); + + ct.setOptions(vm); + ct.sync(); + + int ret = ct.checkOptions(); + if (ret < 0) return ret; + + mmpi.bcastGlobal(input_file); + mmpi.bcastGlobal(lrs_filename); + +#ifdef _OPENMP + if (onpe0) + { + cout << " " << omp_get_max_threads() << " thread" + << (omp_get_max_threads() > 1 ? "s " : " "); + cout << "active" << endl << endl; + } + omp_set_nested(0); + if (omp_get_nested()) + { + cerr << "Nested parallelism not allowed" << endl; + return 1; + } +#endif + + // Enter main scope + { + // setup standard output and error + sout = &std::cout; + serr = &std::cerr; + + MGmolInterface* mgmol; + if (ct.isLocMode()) + mgmol = new MGmol(global_comm, *MPIdata::sout); + else + mgmol + = new MGmol(global_comm, *MPIdata::sout); + + unsigned ngpts[3] = { ct.ngpts_[0], ct.ngpts_[1], ct.ngpts_[2] }; + double origin[3] = { ct.ox_, ct.oy_, ct.oz_ }; + const double cell[3] = { ct.lx_, ct.ly_, ct.lz_ }; + Mesh::setup(mmpi.commSpin(), ngpts, origin, cell, ct.lap_type); + + mgmol->setupFromInput(input_file); + + if (ct.isLocMode() || ct.init_loc == 1) mgmol->setupLRs(lrs_filename); + + mgmol->setupConstraintsFromInput(constraints_filename); + + ct.checkNLrange(); + + LocGridOrbitals::setDotProduct(ct.dot_product_type); + + Mesh* mymesh = Mesh::instance(); + const pb::PEenv& myPEenv = mymesh->peenv(); + + if (!ct.short_sighted) + { + MatricesBlacsContext::instance().setup(mmpi.commSpin(), ct.numst); + + dist_matrix::DistMatrix::setBlockSize(64); + + dist_matrix::DistMatrix::setDefaultBlacsContext( + MatricesBlacsContext::instance().bcxt()); + + ReplicatedWorkSpace::instance().setup(ct.numst); + + dist_matrix::SparseDistMatrix< + DISTMATDTYPE>::setNumTasksPerPartitioning(128); + + MGmol_MPI& mmpi = *(MGmol_MPI::instance()); + int npes = mmpi.size(); + setSparseDistMatriConsolidationNumber(npes); + } +#ifdef HAVE_MAGMA + ReplicatedMatrix::setMPIcomm(mmpi.commSpin()); +#endif + + if (myPEenv.color() > 0) + { + cerr << "Code should be called with " << myPEenv.n_mpi_tasks() + << " MPI tasks only" << endl; + ct.global_exit(2); + } + + assert(myPEenv.color() == 0); + + if (myPEenv.color() == 0) + { + assert(ct.getMGlevels() >= -1); + if (ct.withPreconditioner()) + { + const pb::Grid& mygrid = mymesh->grid(); + + if ((mygrid.dim(0) % (1 << ct.getMGlevels())) != 0) + { + cerr << "main: mygrid.dim(0)=" << mygrid.dim(0) + << " not evenly divisible by " + << "2^(ct.getMGlevels())=" << (1 << ct.getMGlevels()) + << endl; + return -1; + } + if ((mygrid.dim(1) % (1 << ct.getMGlevels())) != 0) + { + cerr << "main: mygrid.dim(1)=" << mygrid.dim(1) + << " not evenly divisible by " + << "2^(ct.getMGlevels())=" << (1 << ct.getMGlevels()) + << endl; + return -1; + } + if ((mygrid.dim(2) % (1 << ct.getMGlevels())) != 0) + { + cerr << "main: mygrid.dim(2)=" << mygrid.dim(2) + << " not evenly divisible by " + << "2^(ct.getMGlevels())=" << (1 << ct.getMGlevels()) + << endl; + return -1; + } + } + + myPEenv.barrier(); // wait to see if everybody is OK before + // continuing... + + // mgmol->run(); + mgmol->setup(); + + if (ct.isLocMode()) + readRestartFiles(mgmol); + else + readRestartFiles(mgmol); + } + delete mgmol; + + if (!ct.short_sighted) + { + // need to destroy any MPI based object befor calling MPI_Finalize + MatricesBlacsContext::instance().clear(); + } + } // close main scope + + // release memory for static arrays + PackedCommunicationBuffer::deleteStorage(); + Mesh::deleteInstance(); + Control::deleteInstance(); + MGmol_MPI::deleteInstance(); + + mpirc = MPI_Finalize(); + if (mpirc != MPI_SUCCESS) + { + cerr << "MPI Finalize failed!!!" << endl; + } + + time_t tt; + time(&tt); + if (onpe0) cout << " Run ended at " << ctime(&tt) << endl; + + // MemTrack::TrackDumpBlocks(); + + // MemTrack::TrackListMemoryUsage(); + + return 0; +} diff --git a/src/rom_workflows.cc b/src/rom_workflows.cc new file mode 100644 index 00000000..4594dd3a --- /dev/null +++ b/src/rom_workflows.cc @@ -0,0 +1,71 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "rom_workflows.h" +#include +#include +#include + +template +std::string string_format( const std::string& format, Args ... args ) +{ + int size_s = std::snprintf( nullptr, 0, format.c_str(), args ... ) + 1; // Extra space for '\0' + if( size_s <= 0 ){ throw std::runtime_error( "Error during formatting." ); } + auto size = static_cast( size_s ); + std::unique_ptr buf( new char[ size ] ); + std::snprintf( buf.get(), size, format.c_str(), args ... ); + return std::string( buf.get(), buf.get() + size - 1 ); // We don't want the '\0' inside +} + +template +void readRestartFiles(MGmolInterface *mgmol_) +{ + Control& ct = *(Control::instance()); + ROMPrivateOptions rom_options = ct.getROMOptions(); + assert(rom_options.restart_file_minidx >= 0); + assert(rom_options.restart_file_maxidx >= 0); + const int minidx = rom_options.restart_file_minidx; + const int maxidx = rom_options.restart_file_maxidx; + const int num_restart = maxidx - minidx + 1; + + MGmol *mgmol = static_cast *>(mgmol_); + OrbitalsType *orbitals = nullptr; + std::string filename; + + /* Read the first snapshot to determin dimension and number of snapshots */ + filename = string_format(rom_options.restart_file_fmt, minidx); + orbitals = mgmol->loadOrbitalFromRestartFile(filename); + const int dim = orbitals->getLocNumpt(); + const int chrom_num = orbitals->chromatic_number(); + const int totalSamples = orbitals->chromatic_number() * num_restart; + delete orbitals; + + /* Initialize libROM classes */ + CAROM::Options svd_options(dim, totalSamples, 1); + CAROM::BasisGenerator basis_generator(svd_options, false, rom_options.basis_file); + + /* Collect the restart files */ + for (int k = minidx; k <= maxidx; k++) + { + filename = string_format(rom_options.restart_file_fmt, k); + orbitals = mgmol->loadOrbitalFromRestartFile(filename); + assert(dim == orbitals->getLocNumpt()); + assert(chrom_num == orbitals->chromatic_number()); + + for (int i = 0; i < chrom_num; ++i) + basis_generator.takeSample(orbitals->getPsi(i)); + + delete orbitals; + } + basis_generator.writeSnapshot(); + basis_generator.endSamples(); +} + +template void readRestartFiles(MGmolInterface *mgmol_); +template void readRestartFiles(MGmolInterface *mgmol_); \ No newline at end of file diff --git a/src/rom_workflows.h b/src/rom_workflows.h new file mode 100644 index 00000000..bf2e7941 --- /dev/null +++ b/src/rom_workflows.h @@ -0,0 +1,55 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef ROM_WORKFLOWS_H +#define ROM_WORKFLOWS_H + +#include +#include +#include +#include +using namespace std; + +#ifdef _OPENMP +#include +#endif + +#ifdef USE_CNR +#include +#endif + +#include + +#include "Control.h" +#include "DistMatrix.h" +#include "ExtendedGridOrbitals.h" +#include "LocGridOrbitals.h" +#include "MGmol.h" +#include "MGmol_MPI.h" +#include "MPIdata.h" +#include "MatricesBlacsContext.h" +#include "Mesh.h" +#include "PackedCommunicationBuffer.h" +#include "ReplicatedWorkSpace.h" +#include "SparseDistMatrix.h" +#include "magma_singleton.h" +#include "tools.h" + +#include +#include +#include + +#include "OptionDescription.h" + +#include "librom.h" + +template +void readRestartFiles(MGmolInterface *mgmol_); + +#endif // ROM_WORKFLOWS_H diff --git a/src/setup.cc b/src/setup.cc index 035d7c8c..16b09e08 100644 --- a/src/setup.cc +++ b/src/setup.cc @@ -39,8 +39,8 @@ int MGmol::setupFromInput(const std::string filename) const pb::PEenv& myPEenv = mymesh->peenv(); if (ct.restart_info > 0) - h5f_file_ - = new HDFrestart(ct.restart_file, myPEenv, ct.restart_file_type); + h5f_file_ = new HDFrestart(ct.restart_file, myPEenv, + ct.restart_file_type); int status = readCoordinates(filename, false); if (status == -1) return -1; diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 936a7ab6..274a0404 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -8,6 +8,7 @@ set(SOURCES mgmol_mpi_tools.cc random.cc coloring.cc SymmetricMatrix.cc + OptionDescription.cc ) add_library(mgmol_tools ${SOURCES}) diff --git a/src/tools/OptionDescription.cc b/src/tools/OptionDescription.cc new file mode 100644 index 00000000..a95e6a4a --- /dev/null +++ b/src/tools/OptionDescription.cc @@ -0,0 +1,335 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#include "OptionDescription.h" + +using namespace std; + +void setupGenericOption(po::options_description &generic, string &config_file, string &lrs_filename) +{ + generic.add_options()("version,v", "print version string")("help,h", + "produce help message")("check", "check input")("config,c", + po::value(&config_file)->default_value("mgmol.cfg"), + "name of a file of a configuration.")("atomicCoordinates,i", + po::value>(), + "coordinates filename")("LRsFilename,l", + po::value(&lrs_filename), "LRs filename"); +} + +void setupConfigOption(po::options_description &config, string &constraints_filename) +{ + config.add_options()( + "spin", po::value(), "system total spin")("verbosity", + po::value()->default_value(1), "verbosity level")( + "constraintsFilename", po::value(&constraints_filename), + "Name of file with list of constraints")("xcFunctional", + po::value()->required(), "XC functional: LDA or PBE")( + "FDtype", po::value()->default_value("Mehrstellen"), + "Finite Difference scheme")("charge", + po::value()->default_value(0), "system total charge")( + "Domain.lx", po::value()->required(), + "domain dimension in x direction")("Domain.ly", + po::value()->required(), + "domain dimension in y direction")("Domain.lz", + po::value()->required(), + "domain dimension in z direction")("Domain.ox", + po::value()->required(), "domain origin in x direction")( + "Domain.oy", po::value()->required(), + "domain origin in y direction")("Domain.oz", + po::value()->required(), "domain origin in z direction")( + "Mesh.nx", po::value()->required(), + "mesh dimension in x direction")("Mesh.ny", + po::value()->required(), + "mesh dimension in y direction")("Mesh.nz", + po::value()->required(), + "mesh dimension in z direction")("Potentials.pseudopotential", + po::value>()->multitoken(), + "pseudopotentials list")("Potentials.external", + po::value>()->multitoken(), + "external potentials list")("Potentials.binExternal", + po::value()->default_value(true), + "binary external potential")("Restart.input_filename", + po::value()->default_value(""), + "Read restart filename/directory")("Restart.input_level", + po::value()->default_value(0), + "Read restart level")("Restart.input_type", + po::value()->default_value("distributed"), + "Read restart type: distributed or single_file")( + "Restart.output_filename", + po::value()->default_value("auto"), + "Dump restart filename/directory")("Restart.output_level", + po::value()->default_value(3), + "Write restart level")("Restart.output_type", + po::value()->default_value("distributed"), + "Write restart type: distributed or single_file")( + "Restart.interval", po::value()->default_value(1000), + "Restart frequency")("Restart.rescale_v", + po::value()->default_value(1.), + "rescaling factor velocity of all atoms")("Poisson.bcx", + po::value()->default_value("periodic"), + "boundary condition x")("Poisson.bcy", + po::value()->default_value("periodic"), + "boundary condition y")("Poisson.bcz", + po::value()->default_value("periodic"), + "boundary condition z")("Poisson.diel", + po::value()->default_value("off"), + "continuum solvent: on/off")("Run.type", + po::value()->default_value("QUENCH"), "Run type")( + "Quench.solver", po::value()->default_value("ABPG"), + "Iterative solver for quench")("Quench.max_steps", + po::value()->default_value(200), + "Max. steps in loose quench")("Quench.max_steps_tight", + po::value()->default_value(1000), + "Max. steps in tight quench")("Quench.atol", + po::value()->default_value(1.e-12), + "Abs. tol. in quench convergence")("Quench.rtol", + po::value()->default_value(-1.), + "Rel. tol. in quench convergence")("Quench.conv_criterion", + po::value()->default_value("deltaE"), + "Convergence criterion")("Quench.MLWC", po::value(), + "Compute MLWC in quench")("Quench.MLWF", + po::value()->default_value(false), + "Compute MLWF (apply rotation) in quench")( + "Quench.num_lin_iterations", + po::value()->default_value(0), + "Number of iterations without potential update in quench")( + "Quench.preconditioner_num_levels", + po::value()->default_value(2), + "Number of levels for MG preconditioner")( + "Quench.spread_penalty_damping", + po::value()->default_value(0.), + "Spread penalty damping factor")("Quench.spread_penalty_target", + po::value()->default_value(2.), + "Spread penalty target")("SpreadPenalty.type", + po::value()->default_value("individual"), + "Spread penalty type (individual,volume)")( + "SpreadPenalty.damping", po::value()->default_value(1.), + "Spread penalty damping factor")("SpreadPenalty.target", + po::value()->default_value(-1.), + "Spread penalty target")("SpreadPenalty.alpha", + po::value()->default_value(0.), "Spread penalty factor")( + "MD.num_steps", po::value()->default_value(1), + "number of MD steps")("MD.last_step", + po::value()->default_value(-1), "last MD step")( + "MD.dt", po::value(), "time step for MD (a.u.)")( + "MD.print_interval", po::value()->default_value(1), + "print intervale for MD data")("MD.print_directory", + po::value()->default_value("MD"), + "print directory for MD data")("MD.thermostat", + po::value()->default_value("OFF"), + "MD thermostat: ON or OFF")("MD.remove_mass_center_motion", + po::value()->default_value(true), + "Remove mass center motion")("MD.type", + po::value()->default_value("BOMD"), "MD type: BOMD")( + "GeomOpt.type", po::value()->default_value("LBFGS"), + "Geometry optimization algorithm")("GeomOpt.tol", + po::value()->default_value(4.e-4), + "Tolerance on forces for Geometry optimization")( + "GeomOpt.max_steps", po::value()->default_value(1), + "max. number of Geometry optimization steps")("GeomOpt.dt", + po::value(), "Delta t for trial pseudo-time steps")( + "atomicCoordinates", po::value>(), + "coordinates filename")("Thermostat.type", + po::value()->default_value("Langevin"), + "Thermostat type")("Thermostat.temperature", + po::value()->default_value(-1.), + "Thermostat temperature")("Thermostat.relax_time", + po::value()->default_value(-1.), + "Thermostat relaxation time")("Thermostat.width", + po::value()->default_value(-1.), + "Thermostat width (for SCALING)")("Orbitals.nempty", + po::value()->default_value(0), + "Number of empty orbitals")("Orbitals.initial_type", + po::value()->default_value("random"), + "initial orbitals type")("Orbitals.initial_width", + po::value()->default_value(10000.), + "initial orbitals radius")("Orbitals.temperature", + po::value()->default_value(0.), + "electronic temperature [K]")("Orbitals.bcx", + po::value()->default_value("periodic"), + "orbitals boundary condition x")("Orbitals.bcy", + po::value()->default_value("periodic"), + "orbitals boundary condition y")("Orbitals.bcz", + po::value()->default_value("periodic"), + "orbitals boundary condition z")("ProjectedMatrices.solver", + po::value()->default_value("exact"), + "solver for projected matrices")("ProjectedMatrices.printMM", + po::value()->default_value(false), + "print projected matrices in MM format")( + "LocalizationRegions.radius", + po::value()->default_value(1000.), + "Localization regions radius")("LocalizationRegions.adaptive", + po::value()->default_value(true), + "Localization regions adaptivity")( + "LocalizationRegions.move_tol", + po::value()->default_value(1000.), + "Localization regions move tolerance")( + "Parallel.atomic_info_radius", + po::value()->default_value(8.), + "Max. distance for atomic data communication")( + "LoadBalancing.alpha", po::value()->default_value(0.0), + "Parameter for computing bias for load balancing algo")( + "LoadBalancing.damping_tol", + po::value()->default_value(0.9), + "Damping parameter for computing bias for load balancing algo")( + "LoadBalancing.max_iterations", + po::value()->default_value(50), + "Maximum number of iterations for load balancing algo")( + "LoadBalancing.modulo", po::value()->default_value(1), + "Modulos or parameter to control how often clusters are " + "recomputed during md")("LoadBalancing.output_file", + po::value()->default_value(""), + "Output file for dumping cluster information in vtk format"); +} + +void setupHiddenOption(po::options_description &hidden) +{ + hidden.add_options()("Quench.interval_print_residual", + po::value()->default_value(0), + "Print interval for residual in quench (0 for none)")( + "Quench.required_tol", po::value()->default_value(1000.), + "required tolerance (code stops if not reached)")( + "Quench.compute_cond_Gram", + po::value()->default_value(false), + "Compute condition number of S during quench")( + "Quench.min_Gram_eigenvalue", + po::value()->default_value(0.), + "min. eigenvalue for Gram matrix")("Quench.step_length", + po::value()->default_value(-1.), + "Step length for corrections in quench")("Quench.ortho_freq", + po::value()->default_value(1000), + "Orthonormalization frequency")( + "Quench.pair_mlwf_distance_threshold", + po::value()->default_value(4.), + "Max. distance between pairs for MLWF transform")( + "AOMM.kernel_radius", po::value()->default_value(-1.), + "Radius of kernel functions in AOMM algorithm")( + "AOMM.threshold_factor", po::value()->default_value(-1.), + "Multiplicative factor of kernel radius to set threshold in " + "AOMM projectors dropping")("Orbitals.type", + po::value()->default_value("NO"), + "orbital type")("Orbitals.dotProduct", + po::value()->default_value("diagonal"), + "orbital dot product type")("Orbitals.overallocate_factor", + po::value()->default_value(1.2), + "safety factor to use for static allocation of orbitals")( + "Potentials.filterPseudo", + po::value()->default_value('f'), + "filter")("Poisson.solver", + po::value()->default_value("CG"), "solver")( + "Poisson.e0", po::value()->default_value(78.36), + "continuum solvent: epsilon0")("Poisson.rho0", + po::value()->default_value(0.0004), + "continuum solvent: rho0")("Poisson.beta", + po::value()->default_value(1.3), + "continuum solvent: beta")("Poisson.FDtype", + po::value()->default_value("Mehrstellen"), "FDtype")( + "Poisson.nu1", po::value()->default_value(2), "nu_1")( + "Poisson.nu2", po::value()->default_value(2), "nu_2")( + "Poisson.max_steps", po::value()->default_value(20), + "max. nb. steps Poisson solver")("Poisson.max_steps_initial", + po::value()->default_value(20), + "max. nb. steps Poisson solver in first solve")( + "Poisson.max_levels", po::value()->default_value(10), + "max. nb. MG levels Poisson solver")("Poisson.reset", + po::value()->default_value(false), + "reset Hartree potential at each MD step")("ABPG.m", + po::value()->default_value(1), + "History length for Anderson extrapolation")("ABPG.beta", + po::value()->default_value(1.), + "beta for Anderson extrapolation")("NLCG.parallel_transport", + po::value()->default_value(true), + "Turn ON/OFF parallel transport algorithm")( + "MD.extrapolation_type", po::value()->default_value(1), + "MD extrapolation type")("MD.compute_cond_Gram", + po::value()->default_value(false), + "Compute condition number of S at end of quench")( + "MD.min_Gram_eigenvalue", po::value()->default_value(0.), + "min. eigenvalue for Gram matrix")( + "ShortSightedInverse.spread_factor", + po::value()->default_value(2.), + "Shortsighted spread factor")("ShortSightedInverse.tol", + po::value()->default_value(1.e-10), + "Shortsighted tolerance")("ShortSightedInverse.krylov_dim", + po::value()->default_value(10), + "Shortsighted Krylov space max. dimension")( + "ShortSightedInverse.max_iterations", + po::value()->default_value(10), + "Shortsighted max. number of GMRES iterations")( + "ShortSightedInverse.ilu_type", + po::value()->default_value("ILU"), + "Shortsighted ILU type: ILU or ILUT")( + "ShortSightedInverse.ilu_drop_tol", + po::value()->default_value(1.e-5), + "Shortsighted ILU dropping tolerance")( + "ShortSightedInverse.ilu_filling_level", + po::value()->default_value(0), + "Shortsighted ILU filling level")( + "ShortSightedInverse.ilut_max_fill", + po::value()->default_value(10000), + "Shortsighted max. filling for ILUT")("Coloring.algo", + po::value()->default_value("RLF"), + "Coloring algorithm: RLF or Greedy")("Coloring.scope", + po::value()->default_value("local"), + "Coloring scope: local or global")( + "LocalizationRegions.min_distance", + po::value()->default_value(0.), + "min. distance between Localization centers")( + "LocalizationRegions.extrapolation_scheme", + po::value()->default_value("linear"), + "Extrapolation order for localization centers")( + "LocalizationRegions.computation", + po::value()->default_value(0), + "Flag for computing new centers from extrapolated orbitals.")( + "DensityMatrix.mixing", po::value()->default_value(-1.), + "Mixing coefficient for Density Matrix")("DensityMatrix.solver", + po::value()->default_value("Mixing"), + "Algorithm for updating Density Matrix: Mixing, MVP, HMVP")( + "DensityMatrix.nb_inner_it", + po::value()->default_value(3), + "Max. number of inner iterations in DM optimization")( + "DensityMatrix.algo", + po::value()->default_value("Diagonalization"), + "Algorithm for computing Density Matrix. " + "Diagonalization or SP2 or Chebyshev.")("DensityMatrix.use_old", + po::value()->default_value(true), + "Start DM optimization with matrix of previous WF step")( + "DensityMatrix.approximation_order", + po::value()->default_value(500), + "Polynomial order for Chebyshev approximation " + "of density matrix. ")("DensityMatrix.approximation_ndigits", + po::value()->default_value(1), + "Number of digits of precision for Chebyshev " + "approximation of density matrix. ")( + "DensityMatrix.approximation_power_maxits", + po::value()->default_value(100), + "Maximum number of iterations for power method " + "to compute interval for Chebyshev " + "approximation of density matrix. ")("DensityMatrix.tol", + po::value()->default_value(1.e-7), + "tolerance, used in iterative DM computation convergence " + "criteria"); +} + +void setupROMConfigOption(po::options_description &rom_cfg) +{ + rom_cfg.add_options() + ("ROM.stage", po::value()->default_value("none"), + "ROM workflow stage: offline; build; online; none.") + ("ROM.offline.restart_filefmt", po::value(), + "File name format to read for snapshots.") + ("ROM.offline.restart_min_idx", po::value(), + "Minimum index for snapshot file format.") + ("ROM.offline.restart_max_idx", po::value(), + "Maximum index for snapshot file format.") + ("ROM.offline.basis_file", po::value(), + "File name for libROM snapshot/POD matrices.") + ("ROM.offline.save_librom_snapshot", po::value()->default_value(false), + "Save libROM snapshot file at FOM simulation."); +} diff --git a/src/tools/OptionDescription.h b/src/tools/OptionDescription.h new file mode 100644 index 00000000..8213030f --- /dev/null +++ b/src/tools/OptionDescription.h @@ -0,0 +1,26 @@ +// Copyright (c) 2017, Lawrence Livermore National Security, LLC and +// UT-Battelle, LLC. +// Produced at the Lawrence Livermore National Laboratory and the Oak Ridge +// National Laboratory. +// LLNL-CODE-743438 +// All rights reserved. +// This file is part of MGmol. For details, see https://github.com/llnl/mgmol. +// Please also read this link https://github.com/llnl/mgmol/LICENSE + +#ifndef OPTION_DESCRIPTION_H +#define OPTION_DESCRIPTION_H + +#include +namespace po = boost::program_options; + +using namespace std; + +void setupGenericOption(po::options_description &generic, string &config_file, string &lrs_filename); + +void setupConfigOption(po::options_description &config, string &constraints_filename); + +void setupHiddenOption(po::options_description &hidden); + +void setupROMConfigOption(po::options_description &rom_cfg); + +#endif diff --git a/src/tools/Timeout.h b/src/tools/Timeout.h index 18940a78..25751b2a 100644 --- a/src/tools/Timeout.h +++ b/src/tools/Timeout.h @@ -17,7 +17,7 @@ #include #include "MPIdata.h" -#include "Signal.h" +#include "mgmol_Signal.h" #if PCS #include diff --git a/src/tools/Signal.h b/src/tools/mgmol_Signal.h similarity index 98% rename from src/tools/Signal.h rename to src/tools/mgmol_Signal.h index 8e8d0a21..33c47b3e 100644 --- a/src/tools/Signal.h +++ b/src/tools/mgmol_Signal.h @@ -15,7 +15,7 @@ // access the flag set, reset flags, or interrogate flags. // The Signal class can be used in an application by declaring // -// #include "Signal.h" +// #include "mgmol_Signal.h" // set Signal::recv_; // // A signal can be registered using, e.g. diff --git a/util/analyzeMDwithDistanceDiffConstraint.py b/util/analyzeMDwithDistanceDiffConstraint.py index 5a70a9a4..91c5e31f 100644 --- a/util/analyzeMDwithDistanceDiffConstraint.py +++ b/util/analyzeMDwithDistanceDiffConstraint.py @@ -5,8 +5,12 @@ # This file is part of MGmol. For details, see https://github.com/llnl/mgmol. # Please also read this link https://github.com/llnl/mgmol/LICENSE # -#how to run in local directory: -#python analyzeMDwithDistanceDiffConstraint.py . O8203 H3041 N6678 > forces.dat +# How to run in a python script: +# sys.path.append('/home/q8j/Documents/q8j/GIT/MGmol/util') +# import analyzeMDwithDistanceDiffConstraint as analyze +# ... +# analyze.runAnalyzeForces(idir,name0,name1,name2) + import sys, string, os from math import sqrt, acos from numpy import * @@ -34,12 +38,11 @@ time=0. def getMassAtom(name): - one=name[0] mass=0. - if one[0:2] in spmass.keys(): - mass=spmass[one[0:0+2]] + if name[0:2] in spmass.keys(): + mass=spmass[name[0:0+2]] else: - mass=spmass[one[0]] + mass=spmass[name[0]] return mass def analyzeForces(filename,name0,name1,name2,dt): @@ -48,14 +51,18 @@ def analyzeForces(filename,name0,name1,name2,dt): global corrected_avg_force global avg_invsqrtz global forces + global times + global running_forces global nsteps global nravg global time - + + print("analyzeForces with arguments {}, {}, {}, {}".format(filename, name0,name1,name2,dt)) + m0=getMassAtom(name0) m1=getMassAtom(name1) m2=getMassAtom(name2) - #print '#Masses:',m0,m1,m2 + print("#Masses: {}, {}, {}".format(m0,m1,m2)) coords=[] found=0 @@ -66,10 +73,10 @@ def analyzeForces(filename,name0,name1,name2,dt): r1=zeros(3) r2=zeros(3) - file=open(filename,'r') - L1=file.readlines() - for line in L1: ## loop over lines of file - words=string.split(line) + f=open(filename,'r') + lines=f.readlines() + for line in lines: ## loop over lines of file + words=line.split() if len(words)>1: #find force on constraint if 'force' in words and 'constraint' in words and '=' in words: @@ -86,7 +93,7 @@ def analyzeForces(filename,name0,name1,name2,dt): coords.append(name+'\t'+x+'\t'+y+'\t'+z) if found==3 and found_force: for i in range(3): - swords=string.split(coords[i]) + swords=coords[i].split() name=swords[0] x=eval(swords[1]) y=eval(swords[2]) @@ -121,8 +128,12 @@ def analyzeForces(filename,name0,name1,name2,dt): times.append(time) forces.append(eval(force)) - - print filename,'time=',time,' Q=',q,' force=',force,' corrected force=',corrected_force,' Z=',z,' running avg.= ',running_avg_force + + tol = 1.e-4 + delta = corrected_force-eval(force) + if abs(delta)>tol: + print("{}, time={}, Q={}, force={}, corrected force={}, Z={}, running avg.= {}".format( \ + filename,time,q,force,corrected_force,z,running_avg_force)) #print r01, r12 time=time+dt*au2ps @@ -131,7 +142,6 @@ def analyzeForces(filename,name0,name1,name2,dt): if time>1.: avg_force=avg_force+eval(force) - running_forces.append(running_avg_force) invsqrtz=1./sqrt(z) @@ -140,96 +150,137 @@ def analyzeForces(filename,name0,name1,name2,dt): nsteps=nsteps+1 - file.close() - -#main -filesdir=sys.argv[1] -filenames=os.listdir(filesdir) - -name0=sys.argv[2] -name1=sys.argv[3] -name2=sys.argv[4] -#print name0, name1, name2 - -inputs=[] -for filename in filenames: - if 'md_run' in filename: - inputs.append(filename) - -inputs.sort() - -inputs[0] -file=open(inputs[0],'r') -L1=file.readlines() -for line in L1: ## loop over lines of file - word=string.split(line) - if len(word)>1: - if word[0]=='Timestep': - dt=eval(word[5]) - -for filename in inputs: - analyzeForces(filename,name0,name1,name2,dt) - -nf=int(ceil(1./(dt*au2ps))) -ff=running_forces[-nf:] -maxF=max(float(v) for v in ff) -minF=min(float(v) for v in ff) -print '#maxF=',maxF -print '#minF=',minF - -if nsteps>0: - print '#Average force =',avg_force/nsteps - print '#Average corrected force=',corrected_avg_force/avg_invsqrtz - print '#running_forces, spread last ',nf,' steps=',maxF-minF + f.close() + +############################################################################### +# main script +############################################################################### + + +def runAnalyzeForces(filesdir,name0,name1,name2): + + global avg_force + global running_avg_force + global corrected_avg_force + global avg_invsqrtz + global forces + global times + global running_forces + global nsteps + global nravg + global time + + avg_force=0. + running_avg_force=0. + corrected_avg_force=0. + avg_invsqrtz=0. + + forces=[] + times=[] + running_forces=[] + + nsteps=0 + nravg=0 + time=0. + + + filenames=os.listdir(filesdir) + + #screen out files that are not mgmol outputs + inputs=[] + for filename in filenames: + if '.out' in filename: + inputs.append(filesdir+"/"+filename) + + inputs.sort() + + #extract dt from first output file in MD run + inputs[0] + f=open(inputs[0],'r') + lines=f.readlines() + for line in lines: ## loop over lines of file + word=line.split() + if len(word)>1: + if word[0]=='Timestep': + dt=eval(word[5]) + + #analyze all outputs + for filename in inputs: + analyzeForces(filename,name0,name1,name2,dt) + + #number of steps/ps + nf=int(ceil(1./(dt*au2ps))) + print("nf = {}".format(nf)) + + ff=running_forces[-nf:] + maxF=max(float(v) for v in ff) + minF=min(float(v) for v in ff) + print("#maxF={}".format(maxF)) + print("#minF={}".format(minF)) + + if nsteps>0: + print("#Average force ={}".format(avg_force/nsteps)) + print("#Average corrected force={}".format(corrected_avg_force/avg_invsqrtz)) + print("#running_forces, spread last {}, steps={}".format(nf,maxF-minF)) -skip=5 -forcep = [ forces[i] for i in range(0, len(forces), skip)] -timesp = [ times[i] for i in range(0, len(forces), skip)] - -skip=25 -aves1 = [sum(forces[i-nf:i])/nf for i in range(nf, len(forces), skip)] -aves2 = [sum(forces[i-2*nf:i])/(2*nf) for i in range(2*nf, len(forces), skip)] -aves3 = [sum(forces[i-3*nf:i])/(3*nf) for i in range(3*nf, len(forces), skip)] - -times1 = [ times[i] for i in range(nf, len(forces), skip)] -times2 = [ times[i] for i in range(2*nf, len(forces), skip)] -times3 = [ times[i] for i in range(3*nf, len(forces), skip)] - -print '#time force' -for i in range(len(forces)): - print times[i],forces[i] - -print '#Running average over 2 ps, last value: ',aves2[-1] -print '#Running average over 3 ps, last value: ',aves3[-1] - - -xmax=len(forces)*dt*au2ps -xmax=max(xmax,4.) -ymin=min(forces) -ymax=max(forces) - -plt.figure(1) -plt.subplot(211) -plt.axis([0.,xmax,ymin,ymax]) -plt.plot(timesp, forcep, 'go') -plt.ylabel('force (Ha/Bohr)') -#plt.show() - -plt.subplot(212) -ymin=min(aves2) -ymax=max(aves2) -plt.axis([0.,xmax,ymin,ymax]) -#plt.plot(times1, aves1, 'ro') -plt.plot(times2, aves2, 'ro') -plt.plot(times3, aves3, 'bo') -plt.ylabel('force (Ha/Bohr)') -plt.xlabel('time (ps)') -#plt.show() -plt.savefig('aves.png') - -nn=nf/skip -max3=max(aves3[-1-nn:-1]) -min3=min(aves3[-1-nn:-1]) -print 'min aves3 over lat ps = ',min3 -print 'max aves3 over lat ps = ',max3 -print 'spread aves3 over lat ps = ',max3-min3 + skip=5 + forcep = [ forces[i] for i in range(0, len(forces), skip)] + timesp = [ times[i] for i in range(0, len(forces), skip)] + plt.figure(1) + xmax=len(forces)*dt*au2ps + ymin=min(forces) + ymax=max(forces) + plt.axis([0.,xmax,ymin,ymax]) + + fig=plt.figure(1) + plt.subplot(211) + plt.plot(timesp, forcep, 'go') + plt.ylabel('force (Ha/Bohr)') + colors=['ro','bo','go','yo','mo'] + icolor=0 + #compute running averages + skip=25 + for interval in range(1,6): + aves = [sum(forces[i-interval*nf:i])/(interval*nf) for i in range(interval*nf,len(forces), skip)] + + timesp = [ times[i] for i in range(interval*nf, len(forces), skip)] + forcep = [ forces[i] for i in range(interval*nf, len(forces), skip)] + + print("#Average over last {} ps: {}".format(interval,aves[-1])) + + sigma = [sum((forces[i-interval*nf:i]-aves[-1])*(forces[i-interval*nf:i]-aves[-1]))/(interval*nf-1) for i in range(interval*nf, len(forces), skip)] + + print("#Sigma over last {} ps: {}".format(interval,math.sqrt(sigma[-1]))) + + plt.subplot(212) + plt.plot(timesp, aves, colors[icolor]) + icolor=icolor+1 + plt.ylabel('running average (Ha/Bohr)') + plt.xlabel('time (ps)') + plt.axis([0.,xmax,ymin,ymax]) + + plt.savefig(filesdir+'_aves.png') + plt.close(fig) + + nn=int(nf/skip) + print(nn) + max3=max(aves[-1-nn:-1]) + min3=min(aves[-1-nn:-1]) + print("min aves over last ps = {}".format(min3)) + print("max aves over last ps = {}".format(max3)) + print("min-max spread aves3 over last ps = {}".format(max3-min3)) + +############################################################################### +# main script +############################################################################### + +#target directory containing multiple segments of a single run +#filesdir=sys.argv[1] +#filenames=os.listdir(filesdir) + +#read names of 3 atoms involved in constraint +#name0=sys.argv[2] +#name1=sys.argv[3] +#name2=sys.argv[4] + +#runAnalyzeForces(filesdir,name0,name1,name2)