UFS-dev PR#53

.gitmodules

@@ -1,7 +1,9 @@
 [submodule "FV3"]
   path = FV3
-  url = https://github.com/NCAR/fv3atm
-  branch = main
+  #url = https://github.com/NCAR/fv3atm
+  #branch = main
+  url = https://github.com/grantfirl/fv3atm
+  branch = ufs-dev-PR53
 [submodule "WW3"]
   path = WW3
   url = https://github.com/NOAA-EMC/WW3

CMakeLists.txt

@@ -63,6 +63,7 @@ set(AVX             OFF CACHE BOOL "Enable AVX-I instruction set")
 set(SIMDMULTIARCH   OFF CACHE BOOL "Enable multi-target SIMD instruction sets")
 set(DEBUG           OFF CACHE BOOL "Enable DEBUG mode")
 set(DEBUG_LINKMPI   ON  CACHE BOOL "Enable linkmpi option when DEBUG mode is on")
+set(DISABLE_FMA     OFF CACHE BOOL "Disable Fused Multiply-Add instructions (workaround needed for AMD EPYC)" FORCE)
 set(INLINE_POST     ON  CACHE BOOL "Enable inline post")
 set(MULTI_GASES     OFF CACHE BOOL "Enable MULTI_GASES")
 set(MOVING_NEST     OFF CACHE BOOL "Enable moving nest code")
@@ -137,7 +138,7 @@ if(FMS)
 endif()
 
 if(CMEPS)
-  find_package(PIO 2.5.3 REQUIRED COMPONENTS C Fortran STATIC)
+  find_package(PIO 2.5.3 REQUIRED COMPONENTS C Fortran)
 endif()
 
 find_package(bacio 2.4.0 REQUIRED)

cmake/Intel.cmake

@@ -43,3 +43,9 @@ if(APPLE)
   # Passing the -c flag includes them and fixes an error with undefined symbols
   set(CMAKE_Fortran_ARCHIVE_FINISH "<CMAKE_RANLIB> -c <TARGET>")
 endif()
+
+# This must be last, to override all other optimization settings.
+if(DISABLE_FMA)
+  set(CMAKE_Fortran_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -no-fma")
+  set(CMAKE_C_FLAGS_RELEASE "${CMAKE_Fortran_FLAGS_RELEASE} -no-fma")
+endif()

cmake/configure_acorn.cmake

@@ -1,3 +1,7 @@
 set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
 set(DEBUG_LINKMPI   OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE)
 set(AVX2            OFF CACHE BOOL "Enable AVX2 instruction set" FORCE)
+
+if(FASTER)
+  set(DISABLE_FMA   ON  CACHE BOOL "Disable Fused Multiply-Add instructions (workaround needed for AMD EPYC)" FORCE)
+endif()

cmake/configure_wcoss2.cmake

@@ -1,3 +1,7 @@
 set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
 set(DEBUG_LINKMPI   OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE)
 set(AVX2            OFF CACHE BOOL "Enable AVX2 instruction set" FORCE)
+
+if(FASTER)
+  set(DISABLE_FMA   ON  CACHE BOOL "Disable Fused Multiply-Add instructions (workaround needed for AMD EPYC)" FORCE)
+endif()

modulefiles/ufs_gaea.intel.lua

@@ -0,0 +1,35 @@
+help([[
+  This module loads libraries required for building and running UFS Weather Model 
+  on the NOAA RDHPC machine Gaea using Intel-2022.1.2
+]])
+
+whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===])
+
+prepend_path("MODULEPATH", "/lustre/f2/dev/role.epic/contrib/modulefiles")
+load(pathJoin("miniconda3",os.getenv("miniconda_ver") or "4.12.0"))
+
+load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"))
+
+prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-2021.3.0_noarch/modulefiles/stack")
+load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
+load(pathJoin("intel", os.getenv("intel_ver") or "2021.3.0"))
+load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2021.3.0"))
+load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.11"))
+load(pathJoin("gcc", os.getenv("gcc_ver") or "8.3.0"))
+load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37"))
+
+-- needed for WW3 build
+load(pathJoin("gcc", os.getenv("gcc_ver") or "8.3.0"))
+-- Needed at runtime:
+load("alps")
+load("rocoto")
+
+load("ufs_common")
+
+setenv("CC","cc")
+setenv("FC","ftn")
+setenv("CXX","CC")
+setenv("CMAKE_C_COMPILER","cc")
+setenv("CMAKE_CXX_COMPILER","CC")
+setenv("CMAKE_Fortran_COMPILER","ftn")
+setenv("CMAKE_Platform","gaea.intel")

modulefiles/ufs_gaea.intel_debug.lua

@@ -0,0 +1,35 @@
+help([[
+  This module loads libraries required for building and running UFS Weather Model 
+  on the NOAA RDHPC machine Gaea using Intel-2022.1.2
+]])
+
+whatis([===[Loads libraries needed for building the UFS Weather Model and debug on Gaea ]===])
+
+prepend_path("MODULEPATH", "/lustre/f2/dev/role.epic/contrib/modulefiles")
+load(pathJoin("miniconda3",os.getenv("miniconda_ver") or "4.12.0"))
+
+load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"))
+
+prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-2021.3.0_noarch/modulefiles/stack")
+load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
+load(pathJoin("intel", os.getenv("intel_ver") or "2021.3.0"))
+load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2021.3.0"))
+load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.11"))
+load(pathJoin("gcc", os.getenv("gcc_ver") or "8.3.0"))
+load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37"))
+
+-- needed for WW3 build
+load(pathJoin("gcc", os.getenv("gcc_ver") or "8.3.0"))
+-- Needed at runtime:
+load("alps")
+load("rocoto")
+
+load("ufs_common_debug")
+
+setenv("CC","cc")
+setenv("FC","ftn")
+setenv("CXX","CC")
+setenv("CMAKE_C_COMPILER","cc")
+setenv("CMAKE_CXX_COMPILER","CC")
+setenv("CMAKE_Fortran_COMPILER","ftn")
+setenv("CMAKE_Platform","gaea.intel")