GitHub Issue NOAA-EMC#158. Updates for community users to adopt the N…

…OAA hpc-stack structure.
AndrewEichmann-NOAA · May 14, 2021 · 77a729e · 77a729e
1 parent dcb735a
commit 77a729e
Show file tree

Hide file tree

Showing 7 changed files with 96 additions and 70 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -175,8 +175,8 @@ project(GSI)
     find_package( LAPACK )
   endif()
 # build the WRF I/O libraries
-  if(DEFINED ENV{GSIWRF_LIB})
-    set(wrflib "$ENV{GSIWRF_LIB}" CACHE INTERNAL "WRFIO library" )
+  if(DEFINED ENV{GSIWRFIO_LIB})
+    set(wrflib "$ENV{GSIWRFIO_LIB}" CACHE INTERNAL "WRFIO library" )
   elseif(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/libsrc/wrflib)
     add_subdirectory(libsrc/wrflib)
   else()

diff --git a/cmake/Modules/platforms/Generic.cmake b/cmake/Modules/platforms/Generic.cmake
@@ -20,5 +20,6 @@ macro (setGeneric)
   set(BUILD_SP "OFF" CACHE INTERNAL "Build the SP library" )
   set(BUILD_CRTM "OFF" CACHE INTERNAL "Build the CRTM library" )
   set(BUILD_W3EMC "OFF" CACHE INTERNAL "Build the EMC library" )
+  set(BUILD_W3NCO "OFF" CACHE INTERNAL "Build the EMC library" )
   set(BUILD_NCO "OFF" CACHE INTERNAL "Build the NCO library" )
 endmacro()
diff --git a/cmake/Modules/setHOST.cmake b/cmake/Modules/setHOST.cmake
@@ -33,8 +33,7 @@ macro( setHOST )
   string(REGEX MATCH "discover" HOST-Discover ${HOSTNAME} )
   string(REGEX MATCH "cheyenne" HOST-Cheyenne ${HOSTNAME}  )
   message("done figuring out host--${HOSTNAME}")
-  if ( BUILD_CORELIBS )
-    MESSAGE(STATUS "BUILD_CORELIBS manually-specified as ON")
+  if ( COMGSI )
     set( host "GENERIC"  )
     set( HOST-Generic "TRUE"  )
     setGeneric()

diff --git a/ush/build.comgsi b/ush/build.comgsi
@@ -5,9 +5,9 @@
 #   Cheyenne: source /glade/p/ral/jntp/gge/modulefiles/modulefile.cheyenne.GSI_UPP_WRF
 #
 #  build commands:
-#    cmake -DENKF_MODE=WRF -DBUILD_CORELIBS=ON -DBUILD_GSDCLOUD_ARW=ON path_to_GSI
-#    cmake -DENKF_MODE=WRF -DBUILD_CORELIBS=ON -DBUILD_GSDCLOUD_ARW=ON -DBUILD_UTIL_COM=ON -DBUILD_ENKF_PREPROCESS_ARW=ON path_to_GSI"
-#    (for global: cmake -D-DENKF_MODE=GFS -DBUILD_CORELIBS=ON path_to_GSI)
+#    cmake -DCOMGSI=ON -DENKF_MODE=WRF -DBUILD_GSDCLOUD_ARW=ON path_to_GSI
+#    cmake -DCOMGSI=ON -DENKF_MODE=WRF -DBUILD_GSDCLOUD_ARW=ON -DBUILD_UTIL_COM=ON -DBUILD_ENKF_PREPROCESS_ARW=ON path_to_GSI"
+#    (for global: cmake -DCOMGSI=ON -DENKF_MODE=GFS path_to_GSI)
 #    make -j8
 #
 
@@ -16,80 +16,118 @@ dir_root=$(pwd)
 ################# Hera ####################
 if [[ "`grep -i "hera" /etc/hosts | head -n1`" != "" ]] ; then
     source /etc/profile.d/modules.sh
-    modulefile="/home/rtrr/PARM_EXEC/modulefiles/modulefile.hera.GSI_UPP_WRF"
-    NCEPLIBS="/scratch1/BMC/comgsi/precompiled/NCEPLIBS/b_intel18.0.5.274_impi2018.0.4/install"
-    GSILIBS="/scratch1/BMC/comgsi/precompiled/GSILIBS/b_intel18.0.5.274_impi2018.0.4"
+    module purge
+    module load cmake/3.20.1
+    module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+    module load hpc/1.1.0
+    module load hpc-intel/18.0.5.274
+    module load hpc-impi/2018.0.4
+    module load bufr/11.4.0
+    module load ip/3.3.3
+    module load nemsio/2.5.2
+    module load sfcio/1.4.1
+    module load sigio/2.3.2
+    module load sp/2.3.3
+    module load w3nco/2.4.1
+    module load w3emc/2.7.3
+    module load bacio/2.4.1
+    module load crtm/2.3.0
+    module load netcdf/4.7.0
+    module use /scratch1/BMC/comgsi/hpc-stacks/modulefiles/gsiwrfio
+    module load gsiwrfio/1.0.0
 
 ################# Jet ####################
 elif [[ -d /jetmon ]] ; then
     source /etc/profile.d/modules.sh
-    modulefile="/home/rtrr/PARM_EXEC/modulefiles/modulefile.jet.GSI_UPP_WRF"
-    NCEPLIBS="/lfs4/BMC/wrfruc/gge/precompiled/NCEPLIBS/b_intel18.0.5.274_impi2018.4.274/install"
-    GSILIBS="/lfs4/BMC/wrfruc/gge/precompiled/GSILIBS/b_intel18.0.5.274_impi2018.4.274"
+    module purge
+    module load cmake/3.16.1
+    module load intel/18.0.5.274
+    module load impi/2018.4.274
+
+    module use /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack
+    module load hpc/1.1.0
+    module load hpc-intel/18.0.5.274
+    module load hpc-impi/2018.4.274
+
+    module load netcdf/4.7.0 #don't load netcdf/4.7.4 from hpc-stack, GSI does not compile with it.
+    module load bufr/11.4.0
+    module load bacio/2.4.1
+    module load crtm/2.3.0
+    module load ip/3.3.3
+    module load nemsio/2.5.2
+    module load sp/2.3.3
+    module load w3emc/2.7.3
+    module load w3nco/2.4.1
+    module load sfcio/1.4.1
+    module load sigio/2.3.2
+
+    module use /mnt/lfs4/BMC/wrfruc/gge/hpc-stacks/modulefiles/gsiwrfio
+    module load gsiwrfio/1.0.0
 
 ################# Cheyenne ####################
 elif [[ -d /glade ]] ; then
     source /etc/profile.d/modules.sh
-    modulefile="/glade/p/ral/jntp/gge/modulefiles/modulefile.cheyenne.GSI_UPP_WRF"
-    NCEPLIBS="/glade/p/ral/jntp/gge/precompiled/NCEPLIBS/b_intel18.0.5_impi2018.4.274/install"
-    GSILIBS="/glade/p/ral/jntp/gge/precompiled/GSILIBS/b_intel18.0.5_impi2018.4.274"
-    #modulefile="/glade/p/ral/jntp/gge/precompiled/NCEPLIBS/modulefile.cheyenne.GSI_UPP_WRF.gnu"
-    #NCEPLIBS="/glade/p/ral/jntp/gge/precompiled/NCEPLIBS/b_gnu8.3.0_openmpi3.1.4/install"
-    #GSILIBS="/glade/p/ral/jntp/gge/precompiled/GSILIBS/b_gnu8.3.0_openmpi3.1.4/"
+    module purge
+    module use /glade/p/ral/jntp/gge/hpc-stacks/modulefiles/stack
+    module load intel/18.0.5 ncarenv ncarcompilers
+    module load impi/2018.4.274
+    module load mkl/2018.0.5
+    module load netcdf/4.7.4
+    module load hpc/1.1.0
+    module load hpc-intel/18.0.5
+    module load hpc-impi/2018.4.274
+    module load bufr/11.4.0
+    module load ip/3.3.3
+    module load nemsio/2.5.2
+    module load sfcio/1.4.1
+    module load sigio/2.3.2
+    module load sp/2.3.3
+    module load w3nco/2.4.1
+    module load w3emc/2.7.3
+    module load bacio/2.4.1
+    module load crtm/2.3.0
+    module load gsiwrfio/1.0.0
+    module load cmake/3.18.2
 
 ################# Orion ####################
 elif [[ -d /work/noaa ]] ; then  ### orion
-    modulefile="/home/gge/modulefiles/modulefile.orion.GSI_UPP_WRF"
-    NCEPLIBS="/work/noaa/wrfruc/gge/precompiled/NCEPLIBS/b_intel2018.4_impi2018.4/install"
-    GSILIBS="/work/noaa/wrfruc/gge/precompiled/GSILIBS/b_intel2018.4_impi2018.4"
+    module purge
+    module use /work/noaa/wrfruc/gge/hpc-stacks/modulefiles/stack
+    module load intel/2018.4
+    module load impi/2018.4
+    module load netcdf/4.7.2
+    module load hpc/1.1.0
+    module load hpc-intel/2018.4
+    module load hpc-impi/2018.4
+    module load bufr/11.4.0
+    module load ip/3.3.3
+    module load nemsio/2.5.2
+    module load sfcio/1.4.1
+    module load sigio/2.3.2
+    module load sp/2.3.3
+    module load w3nco/2.4.1
+    module load w3emc/2.7.3
+    module load bacio/2.4.1
+    module load crtm/2.3.0
+    module load gsiwrfio/1.0.0
+    module load cmake #any version as long as it is later than v3.16
 
 ################# Generic ####################
 else
     echo -e "\nunknown machine"
     echo "Please modify build.comgsi at this location"
-    echo "to load required modules and setup NCEPLIBS and GSILIBS"
+    echo "to load required modules"
     ##follow the above examples and delete the following "exit 9" to go forward
     exit 9
     source /etc/profile.d/modules.sh
-    modulefile="/my/modulefile.me.GSI_UPP_WRF"
-    NCEPLIBS="/my/NCEPLIBS/b_intel18.0.5_impi2018.4.274/install"
-    GSILIBS="/my/GSILIBS/b_intel18.0.5_impi2018.4.274/"
+    #load modules from the NOAA hpc-stack
 fi
 
-if [ ! -f $modulefile ]; then
-    echo "modulefiles $modulefile does not exist"
-    exit 10
-fi
-source $modulefile
-
 ## if NETCDF4 is set to 0 or 1, unset it
 if [[ "$NETCDF4" == "1" ]] || [[ "$NETCDF4" == "0" ]]; then
   unset NETCDF4
 fi
 
-export BACIO_LIB4=${NCEPLIBS}/lib/libbacio_4.a
-#export  BUFR_LIBd=${NCEPLIBS}/lib/libbufr_d.a
-  export  BUFR_LIBd=${GSILIBS}/lib/libbufr_v.a
-  export GSIWRF_LIB=${GSILIBS}/lib/libWRFLIB.a
-export   CRTM_LIB=${NCEPLIBS}/lib/libcrtm.a
-export   CRTM_INC=${NCEPLIBS}/include
-export NEMSIO_LIB=${NCEPLIBS}/lib/libnemsio.a
-export NEMSIO_INC=${NCEPLIBS}/include
-export SFCIO_LIB4=${NCEPLIBS}/lib/libsfcio_4.a
-export SFCIO_INC4=${NCEPLIBS}/include_4
-export SIGIO_LIB4=${NCEPLIBS}/lib/libsigio_4.a
-export SIGIO_INC4=${NCEPLIBS}/include_4
-export    SP_LIBd=${NCEPLIBS}/lib/libsp_d.a
-export    SP_LIB4=${NCEPLIBS}/lib/libsp_4.a
-export W3EMC_LIBd=${NCEPLIBS}/lib/libw3emc_d.a
-export W3EMC_LIB4=${NCEPLIBS}/lib/libw3emc_4.a
-export W3EMC_INCd=${NCEPLIBS}/include_d
-export W3EMC_INC4=${NCEPLIBS}/include_4
-export W3NCO_LIBd=${NCEPLIBS}/lib/libw3nco_d.a
-export W3NCO_LIB4=${NCEPLIBS}/lib/libw3nco_4.a
-export    IP_LIBd=${NCEPLIBS}/lib/libip_d.a
-export    IP_LIB4=${NCEPLIBS}/lib/libip_4.a
-
 set -x
 rm -rf $dir_root/build
 mkdir -p $dir_root/build
@@ -107,12 +145,12 @@ git log -1 | head -n1 >> output.log
 echo -e "\ngit status:" >> output.log
 git status >> output.log
 echo -e "\nCompiling commands:" >> output.log
-echo "  cmake -DENKF_MODE=WRF -DBUILD_CORELIBS=ON -DBUILD_GSDCLOUD_ARW=ON -DBUILD_ENKF_PREPROCESS_ARW=ON -DBUILD_UTIL_COM=ON -Wno-dev .." >> output.log
+echo "  cmake -DCOMGSI=ON -DENKF_MODE=WRF -DBUILD_GSDCLOUD_ARW=ON -DBUILD_ENKF_PREPROCESS_ARW=ON -DBUILD_UTIL_COM=ON .." >> output.log
 echo "  make -j8" >> output.log
 cat output.log
 
 
-cmake -DENKF_MODE=WRF -DBUILD_CORELIBS=ON -DBUILD_GSDCLOUD_ARW=ON -DBUILD_ENKF_PREPROCESS_ARW=ON -DBUILD_UTIL_COM=ON -Wno-dev ..  2>&1  | tee output.cmake
+cmake -DCOMGSI=ON -DENKF_MODE=WRF -DBUILD_GSDCLOUD_ARW=ON -DBUILD_ENKF_PREPROCESS_ARW=ON -DBUILD_UTIL_COM=ON ..  2>&1  | tee output.cmake
 make -j 8 2>&1 | tee output.compile
 
 ###aftermath

diff --git a/util/bufr_tools/CMakeLists.txt b/util/bufr_tools/CMakeLists.txt
@@ -6,56 +6,45 @@ cmake_minimum_required(VERSION 2.6)
   set_source_files_properties( ${LOCAL_SRC} PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
 
   add_executable(bufr_append_sample.x  bufr_append_sample.f90 )
-  add_dependencies( bufr_append_sample.x ${BUFR_LIBRARY} )
   set_target_properties( bufr_append_sample.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( bufr_append_sample.x ${BUFR_LIBRARY} ) 
 
   add_executable(bufr_decode_sample.x  bufr_decode_sample.f90 )
-  add_dependencies( bufr_decode_sample.x ${BUFR_LIBRARY} )
   set_target_properties( bufr_decode_sample.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( bufr_decode_sample.x ${BUFR_LIBRARY} ) 
 
   add_executable(bufr_encode_sample.x  bufr_encode_sample.f90 )
-  add_dependencies( bufr_encode_sample.x ${BUFR_LIBRARY} )
   set_target_properties( bufr_encode_sample.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( bufr_encode_sample.x ${BUFR_LIBRARY} ) 
 
   add_executable(bufr_decode_radiance.x  bufr_decode_radiance.f90 )
-  add_dependencies( bufr_decode_radiance.x ${BUFR_LIBRARY} )
   set_target_properties( bufr_decode_radiance.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( bufr_decode_radiance.x ${BUFR_LIBRARY} ) 
 
   add_executable(prepbufr_decode_all.x  prepbufr_decode_all.f90 )
-  add_dependencies( prepbufr_decode_all.x ${BUFR_LIBRARY} )
   set_target_properties( prepbufr_decode_all.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( prepbufr_decode_all.x ${BUFR_LIBRARY} ) 
 
   add_executable(prepbufr_encode_surface.x  prepbufr_encode_surface.f90 )
-  add_dependencies( prepbufr_encode_surface.x ${BUFR_LIBRARY} )
   set_target_properties( prepbufr_encode_surface.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( prepbufr_encode_surface.x ${BUFR_LIBRARY} ) 
 
   add_executable(prepbufr_encode_upperair.x  prepbufr_encode_upperair.f90 )
-  add_dependencies( prepbufr_encode_upperair.x ${BUFR_LIBRARY} )
   set_target_properties( prepbufr_encode_upperair.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( prepbufr_encode_upperair.x ${BUFR_LIBRARY} ) 
 
   add_executable(prepbufr_inventory.x  prepbufr_inventory.f90 )
-  add_dependencies( prepbufr_inventory.x ${BUFR_LIBRARY} )
   set_target_properties( prepbufr_inventory.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( prepbufr_inventory.x ${BUFR_LIBRARY} ) 
 
   add_executable(prepbufr_append_retrieve.x  prepbufr_append_retrieve.f90 )
-  add_dependencies( prepbufr_append_retrieve.x ${BUFR_LIBRARY} )
   set_target_properties( prepbufr_append_retrieve.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( prepbufr_append_retrieve.x ${BUFR_LIBRARY} ) 
 
   add_executable(prepbufr_append_surface.x  prepbufr_append_surface.f90 )
-  add_dependencies( prepbufr_append_surface.x ${BUFR_LIBRARY} )
   set_target_properties( prepbufr_append_surface.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( prepbufr_append_surface.x ${BUFR_LIBRARY} ) 
 
   add_executable(prepbufr_append_upperair.x  prepbufr_append_upperair.f90 )
-  add_dependencies( prepbufr_append_upperair.x ${BUFR_LIBRARY} )
   set_target_properties( prepbufr_append_upperair.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( prepbufr_append_upperair.x ${BUFR_LIBRARY} ) 
diff --git a/util/ndate/CMakeLists.txt b/util/ndate/CMakeLists.txt
@@ -4,6 +4,5 @@ cmake_minimum_required(VERSION 2.6)
   set_source_files_properties( ${NDATE_SRC} PROPERTIES COMPILE_FLAGS ${NDATE_Fortran_FLAGS} )
 
   add_executable( ndate.x  ${NDATE_SRC} )
-  add_dependencies( ndate.x ${W3NCO_4_LIBRARY} )
   target_link_libraries( ndate.x ${W3NCO_4_LIBRARY} )
 
diff --git a/util/radar_process/reflectivity/CMakeLists.txt b/util/radar_process/reflectivity/CMakeLists.txt
@@ -4,6 +4,6 @@ cmake_minimum_required(VERSION 2.6)
   include_directories( ${NETCDF_INCLUDE_DIRS}  ${MPI_Fortran_INCLUDE_PATH} )
 
   add_executable(process_NSSL_mosaic.x ${LOCAL_SRC} )
-  add_dependencies(process_NSSL_mosaic.x ${GSILIB} ${GSISHAREDLIB} ${BUFR_LIBRARY} )
+  add_dependencies(process_NSSL_mosaic.x ${GSILIB} ${GSISHAREDLIB} )
   set_target_properties( process_NSSL_mosaic.x PROPERTIES COMPILE_FLAGS ${UTIL_COM_Fortran_FLAGS} )
   target_link_libraries( process_NSSL_mosaic.x ${GSISHAREDLIB} ${BUFR_LIBRARY} ${NETCDF_Fortran_LIBRARIES} ${NETCDF_C_LIBRARIES} ${MPI_Fortran_LIBRARIES} )