Merge branch 'develop' of https://github.com/ufs-community/ufs-weathe…

…r-model into feature/rad-fix-new

.github/pull_request_template.md

@@ -146,10 +146,11 @@ Please delete what is not needed.
   - [ ] Orion
   - [ ] Hercules
   - [ ] Jet
-  - [ ] Gaea
+  - [ ] GaeaC5
+  - [ ] GaeaC6
   - [ ] Derecho
 - WCOSS2
   - [ ] Dogwood/Cactus
   - [ ] Acorn
 - [ ] CI
-- [ ] opnReqTest (complete task if unnecessary)
+- [ ] opnReqTest (complete task if unnecessary)

CDEPS-interface/CMakeLists.txt

@@ -95,6 +95,14 @@ target_include_directories(dwav PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY
 target_link_libraries(dwav PRIVATE dshr share streams)
 target_link_libraries(dwav PRIVATE esmf)
 
+# CDEPS/dglc
+add_library(dglc OBJECT ${cdeps_dglc_files})
+add_dependencies(dglc share streams dshr)
+set_target_properties(dglc PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_include_directories(dglc PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
+target_link_libraries(dglc PRIVATE dshr share streams)
+target_link_libraries(dglc PRIVATE esmf PIO::PIO_Fortran)
+
 # CDEPS
 add_library(cdeps STATIC $<TARGET_OBJECTS:share>
                          $<TARGET_OBJECTS:streams>
@@ -104,7 +112,8 @@ add_library(cdeps STATIC $<TARGET_OBJECTS:share>
                          $<TARGET_OBJECTS:dlnd>
                          $<TARGET_OBJECTS:docn>
                          $<TARGET_OBJECTS:drof>
-                         $<TARGET_OBJECTS:dwav>)
+                         $<TARGET_OBJECTS:dwav>
+                         $<TARGET_OBJECTS:dglc>)
 add_library(cdeps::cdeps ALIAS cdeps)
 target_include_directories(cdeps PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
                                         $<INSTALL_INTERFACE:mod>)

CDEPS-interface/cdeps_files.cmake

@@ -25,6 +25,7 @@ list(APPEND cdeps_share_files
   CDEPS/share/shr_timer_mod.F90
   CDEPS/share/shr_file_mod.F90
   CDEPS/share/shr_nl_mod.F90
+  CDEPS/share/nuopc_shr_methods.F90
 )
 
 list(APPEND cdeps_streams_files
@@ -68,6 +69,8 @@ list(APPEND cdeps_docn_files
   CDEPS/docn/docn_datamode_iaf_mod.F90
   CDEPS/docn/docn_datamode_som_mod.F90
   CDEPS/docn/docn_datamode_cplhist_mod.F90
+  CDEPS/docn/docn_datamode_multilev_mod.F90
+  CDEPS/docn/docn_datamode_multilev_dom_mod.F90
   CDEPS/docn/docn_import_data_mod.F90
   CDEPS/docn/ocn_comp_nuopc.F90
 )
@@ -79,3 +82,8 @@ list(APPEND cdeps_drof_files
 list(APPEND cdeps_dwav_files
   CDEPS/dwav/wav_comp_nuopc.F90
 )
+
+list(APPEND cdeps_dglc_files
+  CDEPS/dglc/dglc_datamode_noevolve_mod.F90
+  CDEPS/dglc/glc_comp_nuopc.F90
+)

CMEPS-interface/CMakeLists.txt

@@ -43,6 +43,7 @@ list(APPEND _ufs_util_files
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/shr_kind_mod.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_assert_mod.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_infnan_mod.F90
+  ${PROJECT_SOURCE_DIR}/CDEPS-interface/CDEPS/share/nuopc_shr_methods.F90
   ${PROJECT_SOURCE_DIR}/CDEPS-interface/ufs/cdeps_share/shr_is_restart_fh_mod.F90)
 
 list(APPEND _mediator_files
@@ -66,7 +67,6 @@ list(APPEND _mediator_files
   CMEPS/mediator/med_phases_prep_ocn_mod.F90
   CMEPS/mediator/esmFlds.F90
   CMEPS/mediator/med.F90
-  CMEPS/mediator/med_time_mod.F90
   CMEPS/mediator/med_phases_ocnalb_mod.F90
   CMEPS/mediator/med_phases_prep_wav_mod.F90
   CMEPS/mediator/med_utils_mod.F90

cmake/configure_gaeac6.intel.cmake

@@ -0,0 +1,4 @@
+set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
+set(MOM6_Extra_FORTRAN_FLAGS "-xsse2")
+set(HYCOM_Extra_FORTRAN_FLAGS "-xSSE4.2")
+set(HYCOM_Extra_C_FLAGS "-xSSE4.2")

cmake/configure_gaeac6.intelllvm.cmake

@@ -0,0 +1,2 @@
+set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
+set(AVX2            OFF CACHE BOOL "Enable AVX2 instruction set" FORCE)

modulefiles/ufs_gaea.intel.lua → modulefiles/ufs_gaeac5.intel.lua

@@ -3,7 +3,7 @@ help([[
   on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0.
 ]])
 
-whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===])
+whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C5 ]===])
 
 prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core")
 
@@ -30,4 +30,4 @@ unload("cray-libsci")
 setenv("CC","cc")
 setenv("CXX","CC")
 setenv("FC","ftn")
-setenv("CMAKE_Platform","gaea.intel")
+setenv("CMAKE_Platform","gaeac5.intel")

modulefiles/ufs_gaea.intelllvm.lua → modulefiles/ufs_gaeac5.intelllvm.lua

@@ -3,7 +3,7 @@ help([[
   on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0.
 ]])
 
-whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===])
+whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C5]===])
 
 prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core")
 
@@ -36,4 +36,4 @@ setenv("I_MPI_F90", "ifx")
 setenv("CC","cc")
 setenv("CXX","CC")
 setenv("FC","ftn")
-setenv("CMAKE_Platform","gaea.intelllvm")
+setenv("CMAKE_Platform","gaeac5.intelllvm")

modulefiles/ufs_gaeac6.intel.lua

@@ -0,0 +1,33 @@
+help([[
+  This module loads libraries required for building and running UFS Weather Model 
+  on the NOAA RDHPC machine Gaea C6 using Intel-2023.2.0.
+]])
+
+whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C6]===])
+
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+stack_python_ver=os.getenv("stack_python_ver") or "3.10.13"
+load(pathJoin("stack-python", stack_python_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+load("ufs_common")
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+unload("darshan-runtime")
+unload("cray-libsci")
+
+setenv("CC","cc")
+setenv("CXX","CC")
+setenv("FC","ftn")
+setenv("CMAKE_Platform","gaeac6.intel")

modulefiles/ufs_gaeac6.intelllvm.lua

@@ -0,0 +1,40 @@
+help([[
+  This module loads libraries required for building and running UFS Weather Model 
+  on the NOAA RDHPC machine Gaea C6 using Intel-2023.2.0.
+]])
+
+whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C6]===])
+
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
+
+stack_python_ver=os.getenv("stack_python_ver") or "3.10.13"
+load(pathJoin("stack-python", stack_python_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+load("ufs_common")
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+unload("darshan-runtime")
+unload("cray-libsci")
+
+unload("intel-classic/2023.2.0")
+load("intel-oneapi/2023.2.0")
+
+setenv("I_MPI_CC", "icx")
+setenv("I_MPI_CXX", "icpx")
+setenv("I_MPI_F90", "ifx")
+
+setenv("CC","cc")
+setenv("CXX","CC")
+setenv("FC","ftn")
+setenv("CMAKE_Platform","gaeac6.intelllvm")

tests-dev/ufs_test_utils.sh

@@ -5,8 +5,13 @@ function set_run_task() {
     source default_vars.sh
     source rt_utils.sh
     source "${PATHRT}"/tests/"${TEST_NAME}"
-    compute_petbounds_and_tasks
-
+    #jkim compute_petbounds_and_tasks
+    if [[ ${ESMF_THREADING} == true ]]; then
+        compute_petbounds_and_tasks_esmf_threading
+    else
+        compute_petbounds_and_tasks_traditional_threading
+    fi
+
     TPN=$(( TPN / THRD ))
     NODES=$(( TASKS / TPN ))
     if (( NODES * TPN < TASKS )); then

tests/bl_date.conf

@@ -1 +1 @@
-export BL_DATE=20241206
+export BL_DATE=20241230

tests/compile.sh

@@ -61,7 +61,10 @@ case ${MACHINE_ID} in
     ;;
   *)
     # Activate lua environment for gaea c5
-    if [[ ${MACHINE_ID} == gaea ]]; then
+    if [[ ${MACHINE_ID} == gaeac5 ]]; then
+      module reset
+    fi
+    if [[ ${MACHINE_ID} == gaeac6 ]]; then
       module reset
     elif [[ ${MACHINE_ID} == hercules ]]; then
       module purge
@@ -97,17 +100,6 @@ SUITES=$(grep -Po "\-DCCPP_SUITES=\K[^ ]*" <<< "${MAKE_OPT}")
 export SUITES
 set -ex
 
-# Valid applications
-if [[ ${MACHINE_ID} != gaea ]] || [[ ${RT_COMPILER} != intelllvm ]]; then # skip MOM6SOLO on gaea with intelllvm
-  if [[ "${MAKE_OPT}" == *"-DAPP=S2S"* ]]; then
-      CMAKE_FLAGS+=" -DMOM6SOLO=ON"
-  fi
-
-  if [[ "${MAKE_OPT}" == *"-DAPP=NG-GODAS"* ]]; then
-      CMAKE_FLAGS+=" -DMOM6SOLO=ON"
-  fi
-fi
-
 CMAKE_FLAGS=$(set -e; trim "${CMAKE_FLAGS}")
 echo "CMAKE_FLAGS = ${CMAKE_FLAGS}"