Generalize UFS atmospheric component. Bring in MPAS dycore.

.gitmodules

@@ -1,5 +1,5 @@
-[submodule "atmos_cubed_sphere"]
-  path = atmos_cubed_sphere
+[submodule "fv3/atmos_cubed_sphere"]
+  path = fv3/atmos_cubed_sphere
   url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere
   branch = dev/emc
 [submodule "ccpp/framework"]
@@ -14,3 +14,7 @@
   path = upp
   url = https://github.com/NOAA-EMC/UPP
   branch = develop
+[submodule "mpas/MPAS-Model"]
+  path = mpas/MPAS-Model
+  url = https://github.com/ufs-community/MPAS-Model.git
+  branch = feature/mpas-in-ufs

CMakeLists.txt

@@ -1,6 +1,6 @@
-# This is the main CMake file for fv3atm.
+# This is the main CMake file for ufsatm.
 #
-# Dusan Jovic, Alex Richert
+# Dusan Jovic, Alex Richert, Dustin Swales
 cmake_minimum_required(VERSION 3.19)
 
 # Handle user build options.
@@ -14,171 +14,233 @@ endif()
 
 # Enable CI build & unit testing:
 if(BUILD_TESTING)
-  project(fv3atm VERSION 1.0 LANGUAGES C CXX Fortran)
+  if (FV3)
+    project(ufsatm_fv3 VERSION 1.0 LANGUAGES C CXX Fortran)
+  endif()
+  if (MPAS)
+    project(ufsatm_mpas VERSION 1.0 LANGUAGES C CXX Fortran)
+  endif()
   include(ci/CMakeLists.txt)
 endif()
 
+# Set variables for all dycore build options in UFSATM.
+set(DYCORE_TARGET_MPAS ufsatm_mpas)
+set(DYCORE_TARGET_FV3  ufsatm_fv3)
+
 ###############################################################################
-### CCPP
+### All dynamical cores
 ###############################################################################
-
+# CCPP
 set(MPI ON)
 add_subdirectory(ccpp)
 
-###############################################################################
-### fv3 dynamical core
-###############################################################################
-
-# These ifdefs need to be turned ON in the dycore.
-set(use_WRTCOMP ON)
-set(GFS_PHYS    ON)
-set(GFS_TYPES   ON)
-set(USE_GFSL63  ON)
-if(MOVING_NEST)
-  set(MOVING_NEST ON)
-endif()
-if(HYDRO)
-  set(HYDRO ON)
-endif()
-add_subdirectory(atmos_cubed_sphere)
-target_compile_definitions(fv3 PRIVATE BYPASS_BREED_SLP_INLINE)
-
-###############################################################################
-### fv3atm
-###############################################################################
-if(INLINE_POST)
-  set(BUILD_POSTEXEC OFF)
-  add_subdirectory(upp)
-  set(POST_SRC io/post_nems_routines.F90 io/post_fv3.F90)
-  list(APPEND _fv3atm_defs_private INLINE_POST)
-endif()
-
 if(CCPP_32BIT)
   add_definitions(-DCCPP_32BIT)
 else()
   remove_definitions(-DCCPP_32BIT)
 endif()
 
+# Pnetcdf
 if(NOT PARALLEL_NETCDF)
-  list(APPEND _fv3atm_defs_private NO_PARALLEL_NETCDF)
+  list(APPEND _ufsatm_defs_private NO_PARALLEL_NETCDF)
 endif()
 
-if(MOVING_NEST)
-  list(APPEND _fv3atm_defs_private MOVING_NEST)
-  if(NOT HYDRO)
-    list(APPEND _fv3atm_defs_private MOIST_CAPPA USE_COND)
-  endif()
+if(BUILD_TESTING)
+  include(CTest)
+  add_subdirectory(tests)
+endif()
 
-  if(DEBUG)
-    list(APPEND _fv3atm_defs_private DEBUG)
-  endif()
-  if(GFS_PHYS)
-    list(APPEND _fv3atm_defs_private GFS_PHYS)
-  endif()
-  if(GFS_TYPES)
-    list(APPEND _fv3atm_defs_private GFS_TYPES)
-  endif()
-  if(USE_GFSL63)
-    list(APPEND _fv3atm_defs_private USE_GFSL63)
+###############################################################################
+### UFSATM with FV3 dynamical core
+###############################################################################
+if (FV3)
+  set(DYCORE_TARGET ${DYCORE_TARGET_FV3})
+
+  # These ifdefs need to be turned ON in the dycore.
+  set(use_WRTCOMP ON)
+  set(GFS_PHYS    ON)
+  set(GFS_TYPES   ON)
+  set(USE_GFSL63  ON)
+  if(MOVING_NEST)
+    set(MOVING_NEST ON)
   endif()
-  if(INTERNAL_FILE_NML)
-    list(APPEND _fv3atm_defs_private INTERNAL_FILE_NML)
+
+  if(HYDRO)
+    set(HYDRO ON)
   endif()
-  if(ENABLE_QUAD_PRECISION)
-    list(APPEND _fv3atm_defs_private ENABLE_QUAD_PRECISION)
+  add_subdirectory(fv3/atmos_cubed_sphere)
+  target_compile_definitions(fv3 PRIVATE BYPASS_BREED_SLP_INLINE)
+
+  if(INLINE_POST)
+    set(BUILD_POSTEXEC OFF)
+    add_subdirectory(upp)
+    set(POST_SRC io/post_nems_routines.F90 io/post_fv3.F90)
+    list(APPEND _ufsatm_defs_private INLINE_POST)
   endif()
-  if(32BIT)
-    list(APPEND _fv3atm_defs_private OVERLOAD_R4 OVERLOAD_R8)
+
+  if(MOVING_NEST)
+    list(APPEND _ufsatm_defs_private MOVING_NEST)
+    if(NOT HYDRO)
+      list(APPEND _ufsatm_defs_private MOIST_CAPPA USE_COND)
+    endif()
+
+    if(DEBUG)
+      list(APPEND _ufsatm_defs_private DEBUG)
+    endif()
+    if(GFS_PHYS)
+      list(APPEND _ufsatm_defs_private GFS_PHYS)
+    endif()
+    if(GFS_TYPES)
+      list(APPEND _ufsatm_defs_private GFS_TYPES)
+    endif()
+    if(USE_GFSL63)
+      list(APPEND _ufsatm_defs_private USE_GFSL63)
+    endif()
+    if(INTERNAL_FILE_NML)
+      list(APPEND _ufsatm_defs_private INTERNAL_FILE_NML)
+    endif()
+    if(ENABLE_QUAD_PRECISION)
+      list(APPEND _ufsatm_defs_private ENABLE_QUAD_PRECISION)
+    endif()
+    if(32BIT)
+      list(APPEND _ufsatm_defs_private OVERLOAD_R4 OVERLOAD_R8)
+    endif()
+
+    list(APPEND moving_nest_srcs
+      fv3/moving_nest/bounding_box.F90
+      fv3/moving_nest/fv_tracker.F90
+      fv3/moving_nest/fv_moving_nest.F90
+      fv3/moving_nest/fv_moving_nest_main.F90
+      fv3/moving_nest/fv_moving_nest_physics.F90
+      fv3/moving_nest/fv_moving_nest_types.F90
+      fv3/moving_nest/fv_moving_nest_utils.F90
+      )
+  else()
+    list(APPEND moving_nest_srcs "")
   endif()
 
-  list(APPEND moving_nest_srcs
-    moving_nest/bounding_box.F90
-    moving_nest/fv_tracker.F90
-    moving_nest/fv_moving_nest.F90
-    moving_nest/fv_moving_nest_main.F90
-    moving_nest/fv_moving_nest_physics.F90
-    moving_nest/fv_moving_nest_types.F90
-    moving_nest/fv_moving_nest_utils.F90
-  )
-else()
-   list(APPEND moving_nest_srcs "")
+  if(MULTI_GASES)
+    list(APPEND _ufsatm_defs_private MULTI_GASES)
+  endif()
+
+  # FV3 drivers and dependencies
+  add_library(${DYCORE_TARGET}
+    fv3/atmos_model.F90
+    fv3/fv3_cap.F90
+    fv3/module_fv3_config.F90
+    fv3/module_fcst_grid_comp.F90
+    stochastic_physics/stochastic_physics_wrapper.F90
+    cpl/module_block_data.F90
+    cpl/module_cplfields.F90
+    cpl/module_cap_cpl.F90
+    cpl/module_cplscalars.F90
+    io/fv3atm_common_io.F90
+    io/fv3atm_clm_lake_io.F90
+    io/fv3atm_rrfs_sd_io.F90
+    io/fv3atm_sfc_io.F90
+    io/fv3atm_oro_io.F90
+    io/fv3atm_history_io.F90
+    io/fv3atm_restart_io.F90
+    io/module_write_netcdf.F90
+    io/module_write_restart_netcdf.F90
+    io/module_fv3_io_def.F90
+    io/module_write_internal_state.F90
+    io/module_wrt_grid_comp.F90
+    ${moving_nest_srcs}
+    ${POST_SRC}
+    )
+  add_dependencies(${DYCORE_TARGET} fv3 fv3ccpp stochastic_physics)
+
+  list(APPEND _ufsatm_defs_private GFS_PHYS
+                                   INTERNAL_FILE_NML
+                                   use_WRTCOMP)
 endif()
 
-add_library(fv3atm
-  atmos_model.F90
-  fv3_cap.F90
-  module_fv3_config.F90
-  module_fcst_grid_comp.F90
-  stochastic_physics/stochastic_physics_wrapper.F90
-  cpl/module_block_data.F90
-  cpl/module_cplfields.F90
-  cpl/module_cap_cpl.F90
-  cpl/module_cplscalars.F90
-  io/fv3atm_common_io.F90
-  io/fv3atm_clm_lake_io.F90
-  io/fv3atm_rrfs_sd_io.F90
-  io/fv3atm_sfc_io.F90
-  io/fv3atm_oro_io.F90
-  io/fv3atm_history_io.F90
-  io/fv3atm_restart_io.F90
-  io/module_write_netcdf.F90
-  io/module_write_restart_netcdf.F90
-  io/module_fv3_io_def.F90
-  io/module_write_internal_state.F90
-  io/module_wrt_grid_comp.F90
-  ${moving_nest_srcs}
-  ${POST_SRC}
-)
-
-add_dependencies(fv3atm fv3 fv3ccpp stochastic_physics)
-
-list(APPEND _fv3atm_defs_private GFS_PHYS
-                                 INTERNAL_FILE_NML
-                                 use_WRTCOMP)
-
-target_compile_definitions(fv3atm PRIVATE "${_fv3atm_defs_private}")
-
-set_target_properties(fv3atm PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
-target_include_directories(fv3atm INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
-                                            $<INSTALL_INTERFACE:mod>)
-
-if(MULTI_GASES)
-  list(APPEND _fv3atm_defs_private MULTI_GASES)
+###############################################################################
+### UFSATM with MPAS dynamical core.
+###############################################################################
+if (MPAS)
+  set(DYCORE_TARGET ${DYCORE_TARGET_MPAS})
+
+  # Include MPAS Cmake tools.
+  include(${CMAKE_CURRENT_SOURCE_DIR}/mpas/MPAS-Model/cmake/Functions/MPAS_Functions.cmake)
+
+  # Set any pre-processor directive needed in MPAS dycore.
+  get_mpas_version(MPAS_VERSION)
+  set(MPAS_ALL_CORES atmosphere)
+  set(MPAS_CORES atmosphere CACHE STRING "MPAS cores to build. Options: ${MPAS_ALL_CORES}")
+  if(MPAS_CORES MATCHES " ") #Convert strings separated with spaces to CMake list separated with ';'
+    string(REPLACE " " ";" MPAS_CORES ${MPAS_CORES})
+    set(MPAS_CORES ${MPAS_CORES} CACHE STRING "MPAS cores to build. Options: ${MPAS_ALL_CORES}" FORCE)
+  endif()
+  set(DO_MPASDA OFF)
+  set(DO_PHYSICS FALSE)
+
+  # Source files for MPAS dynamical core drivers.
+  set(MPAS_MAIN_SRC      ${CMAKE_CURRENT_SOURCE_DIR}/mpas/ufs_mpas.F90)
+  set(MPAS_SUBDRIVER_SRC ${CMAKE_CURRENT_SOURCE_DIR}/mpas/ufs_mpas_subdriver.F90)
+
+  # MPAS NUOPC cap.
+  set(MPAS_NUOPC_CAP     ${CMAKE_CURRENT_SOURCE_DIR}/mpas/mpas_cap.F90)
+
+  # MPAS dynamical core
+  add_subdirectory(mpas)
+
+  # MPAS drivers and dependencies
+  add_library(${DYCORE_TARGET}
+    ${MPAS_MAIN_SRC}
+    ${MPAS_SUBDRIVER_SRC}
+    ${MPAS_NUOPC_CAP}
+    )
+  add_dependencies(${DYCORE_TARGET} mpas)
 endif()
 
-target_link_libraries(fv3atm PUBLIC fv3
-                                    fv3ccpp
-                                    stochastic_physics
-                                    fms)
-
-target_link_libraries(fv3atm PUBLIC w3emc::w3emc_d
-                                    sp::sp_d
-                                    bacio::bacio_4
-                                    esmf)
-
-if(INLINE_POST)
-  target_link_libraries(fv3atm PUBLIC upp::upp)
+###############################################################################
+### Link libraries
+###############################################################################
+target_compile_definitions(${DYCORE_TARGET} PRIVATE "${_ufsatm_defs_private}")
+
+set_target_properties(${DYCORE_TARGET} PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_include_directories(${DYCORE_TARGET} INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
+                                                  $<INSTALL_INTERFACE:mod>)
+
+if (FV3)
+  target_link_libraries(${DYCORE_TARGET} PUBLIC fv3
+                                         fv3ccpp
+					 stochastic_physics
+					 fms)
+  if(INLINE_POST)
+    target_link_libraries(${DYCORE_TARGET} PUBLIC upp::upp)
+  endif()				       
 endif()
 
-if(OPENMP)
-  target_link_libraries(fv3atm PUBLIC OpenMP::OpenMP_Fortran)
+if (MPAS)
+  target_link_libraries(${DYCORE_TARGET} PUBLIC mpas)
+#                                       mpasccpp
+#  				       stochastic_physics
+# 				       fms)
 endif()
 
-if(BUILD_TESTING)
-  include(CTest)
-  add_subdirectory(tests)
+# Always include EMC libraries in dycore install
+target_link_libraries(${DYCORE_TARGET} PUBLIC w3emc::w3emc_d
+                                              sp::sp_d
+					      bacio::bacio_4
+					      esmf)
+# OpenMP
+if(OPENMP)
+  target_link_libraries(${DYCORE_TARGET} PUBLIC OpenMP::OpenMP_Fortran)
 endif()
 
 ###############################################################################
-### Install
+### Install UFSATM
 ###############################################################################
 install(
-  TARGETS fv3atm
-  EXPORT fv3atm-config
+  TARGETS ${DYCORE_TARGET}
+  EXPORT ufsatm-config
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)
 
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
 
-install(EXPORT fv3atm-config
+install(EXPORT ufsatm-config
   DESTINATION lib/cmake)