Merge remote-tracking branch 'gfdl/dev/gfdl' into mle_spatial_cr

.github/workflows/macos-regression.yml

@@ -11,7 +11,6 @@ jobs:
       CC: gcc
       FC: gfortran
       FMS_COMMIT: 2019.01.03
-      FRAMEWORK: fms1
 
     defaults:
       run:

.github/workflows/macos-stencil.yml

@@ -11,7 +11,6 @@ jobs:
       CC: gcc
       FC: gfortran
       FMS_COMMIT: 2019.01.03
-      FRAMEWORK: fms1
 
     defaults:
       run:

.testing/Makefile

@@ -20,7 +20,6 @@
 #
 # General test configuration:
 #   MPIRUN                  MPI job launcher (mpirun, srun, etc)
-#   FRAMEWORK               Model framework (fms1 or fms2)
 #   DO_REPRO_TESTS          Enable production ("repro") testing equivalence
 #   DO_REGRESSION_TESTS     Enable regression tests (usually dev/gfdl)
 #   DO_COVERAGE             Enable code coverage and generate .gcov reports
@@ -74,8 +73,11 @@ AC_SRCDIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))../ac
 # User-defined configuration
 -include config.mk
 
-# Set the infra framework
-FRAMEWORK ?= fms2
+# Set the FMS library
+FMS_COMMIT ?= 2023.03
+FMS_URL ?= https://github.com/NOAA-GFDL/FMS.git
+export FMS_COMMIT
+export FMS_URL
 
 # Set the MPI launcher here
 # TODO: This needs more automated configuration
@@ -133,9 +135,6 @@ TIME ?= time
 WORKSPACE ?= .
 
 # Set directories for build/ and work/
-#BUILD ?= $(WORKSPACE)build
-#DEPS ?= $(BUILD)/deps
-#WORK ?= $(WORKSPACE)work
 BUILD ?= $(WORKSPACE)/build
 DEPS ?= $(BUILD)/deps
 WORK ?= $(WORKSPACE)/work
@@ -207,34 +206,6 @@ else
 endif
 
 
-# List of source files to link this Makefile's dependencies to model Makefiles
-# Assumes a depth of two, and the following extensions: F90 inc c h
-# (1): Root directory
-# NOTE: extensions could be a second variable
-SOURCE = \
-  $(foreach ext,F90 inc c h,$(wildcard $(1)/*/*.$(ext) $(1)/*/*/*.$(ext)))
-
-MOM_SOURCE = \
-  $(call SOURCE,../src) \
-  $(wildcard ../config_src/drivers/solo_driver/*.F90) \
-  $(wildcard ../config_src/ext*/*/*.F90)
-
-TARGET_SOURCE = \
-  $(call SOURCE,$(BUILD)/target_codebase/src) \
-  $(wildcard $(BUILD)/target_codebase/config_src/drivers/solo_driver/*.F90) \
-  $(wildcard $(BUILD)target_codebase/config_src/ext*/*.F90)
-
-ifeq ($(FRAMEWORK), fms1)
-  MOM_SOURCE += $(wildcard ../config_src/infra/FMS1/*.F90)
-  TARGET_SOURCE += $(wildcard $(BUILD)/target_codebase/config_src/infra/FMS1/*.F90)
-else
-  MOM_SOURCE +=$(wildcard ../config_src/infra/FMS2/*.F90)
-  TARGET_SOURCE += $(wildcard $(BUILD)/target_codebase/config_src/infra/FMS2/*.F90)
-endif
-
-FMS_SOURCE = $(call SOURCE,$(DEPS)/fms/src)
-
-
 ## Rules
 
 .PHONY: all build.regressions build.prof
@@ -286,7 +257,6 @@ $(BUILD)/unit/Makefile: MOM_ENV += $(COV_FCFLAGS) $(COV_LDFLAGS)
 $(BUILD)/timing/Makefile: MOM_ENV += $(OPT_FCFLAGS) $(MOM_LDFLAGS)
 
 # Configure script flags
-MOM_ACFLAGS := --with-framework=$(FRAMEWORK)
 $(BUILD)/openmp/Makefile: MOM_ACFLAGS += --enable-openmp
 $(BUILD)/coupled/Makefile: MOM_ACFLAGS += --with-driver=FMS_cap
 $(BUILD)/nuopc/Makefile: MOM_ACFLAGS += --with-driver=nuopc_cap
@@ -298,11 +268,21 @@ $(BUILD)/timing/Makefile: MOM_ACFLAGS += --with-driver=timing_tests
 $(BUILD)/unit/test_%: $(BUILD)/unit/Makefile FORCE
 	cd $(@D) && $(TIME) $(MAKE) $(@F) -j
 $(BUILD)/unit/Makefile: $(foreach e,$(UNIT_EXECS),../config_src/drivers/unit_tests/$(e).F90)
+
 $(BUILD)/timing/time_%: $(BUILD)/timing/Makefile FORCE
 	cd $(@D) && $(TIME) $(MAKE) $(@F) -j
 $(BUILD)/timing/Makefile: $(foreach e,$(TIMING_EXECS),../config_src/drivers/timing_tests/$(e).F90)
+
 $(BUILD)/%/MOM6: $(BUILD)/%/Makefile FORCE
 	cd $(@D) && $(TIME) $(MAKE) $(@F) -j
+
+# Target codebase should use its own build system
+$(BUILD)/target/MOM6: $(BUILD)/target FORCE | $(TARGET_CODEBASE)
+	$(MAKE) -C $(TARGET_CODEBASE)/.testing build/symmetric/MOM6
+
+$(BUILD)/target: | $(TARGET_CODEBASE)
+	ln -s $(abspath $(TARGET_CODEBASE))/.testing/build/symmetric $@
+
 FORCE:
 
 
@@ -334,27 +314,12 @@ $(BUILD)/%/configure.ac: ../ac/configure.ac | $(BUILD)/%/
 $(BUILD)/%/m4/: ../ac/m4/ | $(BUILD)/%/
 	cp -r ../ac/m4 $(@D)
 
-ALL_EXECS = symmetric asymmetric repro openmp target opt opt_target coupled \
-  nuopc cov unit timing
+ALL_EXECS = symmetric asymmetric repro openmp opt opt_target coupled nuopc \
+  cov unit timing
 $(foreach b,$(ALL_EXECS),$(BUILD)/$(b)/):
 	mkdir -p $@
 
 # Fetch the regression target codebase
-
-$(BUILD)/target/config.status: $(BUILD)/target/configure $(DEPS)/lib/libFMS.a
-	cd $(@D) && $(MOM_ENV) ./configure -n \
-	  --srcdir=$(abspath $(BUILD))/target_codebase/ac $(MOM_ACFLAGS) \
-	|| (cat config.log && false)
-
-$(BUILD)/target/Makefile.in: | $(TARGET_CODEBASE) $(BUILD)/target/
-	cp $(TARGET_CODEBASE)/ac/Makefile.in $(@D)
-
-$(BUILD)/target/configure.ac: | $(TARGET_CODEBASE) $(BUILD)/target/
-	cp $(TARGET_CODEBASE)/ac/configure.ac $(@D)
-
-$(BUILD)/target/m4/: | $(TARGET_CODEBASE) $(BUILD)/target/
-	cp -r $(TARGET_CODEBASE)/ac/m4 $(@D)
-
 $(TARGET_CODEBASE):
 	git clone --recursive $(MOM_TARGET_URL) $@
 	cd $@ && git checkout --recurse-submodules $(MOM_TARGET_BRANCH)

.testing/README.rst

@@ -47,9 +47,11 @@ Several of the following may require configuration for particular systems.
    Name of the MPI launcher.  Often this is ``mpirun`` or ``mpiexec`` but may
    all need to run through a scheduler, e.g. ``srun`` if using Slurm.
 
-``FRAMEWORK`` (*default:* ``fms1``)
-   Select either the legacy FMS framework (``fms1``) or an FMS2 I/O compatible
-   version (``fms2``).
+``FMS_COMMIT`` (*default:* ``2023.03``)
+   Set the FMS version, either by tag or commit (as defined in ``FMS_URL``).
+
+``FMS_URL`` (*default*: ``https://github.com/NOAA-GFDL/FMS.git``)
+   Set the URL of the FMS repository.
 
 ``DO_REPRO_TESTS`` (*default:* *none*)
    Set to ``true`` to test the REPRO build and confirm equivalence of DEBUG and

ac/configure.ac

@@ -79,18 +79,6 @@ AS_IF([test "x$with_driver" != "x"],
 # used to configure a header based on a template.
 #AC_CONFIG_HEADERS(["$MEM_LAYOUT/MOM_memory.h"])
 
-# Select the model framework (default: FMS1)
-# NOTE: We can phase this out after the FMS1 I/O has been removed from FMS and
-#   replace with a detection test.  For now, it is a user-defined switch.
-MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS2
-AC_ARG_WITH([framework],
-  AS_HELP_STRING([--with-framework=fms1|fms2], [Select the model framework]))
-AS_CASE(["$with_framework"],
-  [fms1], [MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS1],
-  [fms2], [MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS2],
-  [MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS2]
-)
-
 
 # Explicitly assume free-form Fortran
 AC_LANG(Fortran)
@@ -220,7 +208,6 @@ AX_FC_CHECK_LIB([FMS], [fms_init], [fms_mod],
   ]
 )
 
-
 # Verify that FMS is at least 2019.01.02
 # NOTE: 2019.01.02 introduced two changes:
 #   - diag_axis_init supports an optional domain_position argument
@@ -236,6 +223,14 @@ AC_COMPILE_IFELSE(
   ]
 )
 
+# Determine the FMS IO implementation.
+AX_FC_CHECK_MODULE([fms2_io_mod], [
+  MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS2
+],[
+  MODEL_FRAMEWORK=${srcdir}/config_src/infra/FMS1
+])
+
+
 # Python interpreter test
 
 # Declare the Python interpreter variable

config_src/drivers/ice_solo_driver/ice_shelf_driver.F90

@@ -26,6 +26,7 @@ program Shelf_main
   use MOM_debugging,       only : MOM_debugging_init
   use MOM_diag_mediator,   only : diag_mediator_init, diag_mediator_infrastructure_init, set_axes_info
   use MOM_diag_mediator,   only : diag_mediator_end, diag_ctrl, diag_mediator_close_registration
+  use MOM_diag_manager_infra, only : diag_manager_set_time_end_infra
   use MOM_domains,         only : MOM_infra_init, MOM_infra_end
   use MOM_domains,         only : MOM_domains_init, clone_MOM_domain, pass_var
   use MOM_dyn_horgrid,     only : dyn_horgrid_type, create_dyn_horgrid, destroy_dyn_horgrid
@@ -325,6 +326,8 @@ program Shelf_main
     Time_end = daymax
   endif
 
+  call diag_manager_set_time_end_infra (Time_end)
+
   if (Time >= Time_end) call MOM_error(FATAL, &
     "Shelf_driver: The run has been started at or after the end time of the run.")
 

config_src/drivers/nuopc_cap/mom_cap.F90

@@ -136,6 +136,7 @@ module MOM_cap_mod
 logical              :: grid_attach_area = .false.
 logical              :: use_coldstart = .true.
 logical              :: use_mommesh = .true.
+logical              :: restart_eor = .false.
 character(len=128)   :: scalar_field_name = ''
 integer              :: scalar_field_count = 0
 integer              :: scalar_field_idx_grid_nx = 0
@@ -381,6 +382,13 @@ subroutine InitializeP0(gcomp, importState, exportState, clock, rc)
     geomtype = ESMF_GEOMTYPE_GRID
   endif
 
+  ! Read end of run restart config option
+  call NUOPC_CompAttributeGet(gcomp, name="write_restart_at_endofrun", value=value, &
+                              isPresent=isPresent, isSet=isSet, rc=rc)
+  if (ChkErr(rc,__LINE__,u_FILE_u)) return
+  if (isPresent .and. isSet) then
+     if (trim(value) .eq. '.true.') restart_eor = .true.
+  end if
 
 end subroutine
 
@@ -1637,6 +1645,8 @@ subroutine ModelAdvance(gcomp, rc)
   real(8)                                :: MPI_Wtime, timers
   logical                                :: write_restart
   logical                                :: write_restartfh
+  logical                                :: write_restart_eor
+
 
   rc = ESMF_SUCCESS
   if(profile_memory) call ESMF_VMLogMemInfo("Entering MOM Model_ADVANCE: ")
@@ -1776,7 +1786,6 @@ subroutine ModelAdvance(gcomp, rc)
   !---------------
   ! Get the stop alarm
   !---------------
-
   call ESMF_ClockGetAlarm(clock, alarmname='stop_alarm', alarm=stop_alarm, rc=rc)
   if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
@@ -1807,7 +1816,18 @@ subroutine ModelAdvance(gcomp, rc)
       if (ChkErr(rc,__LINE__,u_FILE_u)) return
     end if
 
-    if (write_restart .or. write_restartfh) then
+    write_restart_eor = .false.
+    if (restart_eor) then
+      if (ESMF_AlarmIsRinging(stop_alarm, rc=rc)) then
+         if (ChkErr(rc,__LINE__,u_FILE_u)) return
+         write_restart_eor = .true.
+         ! turn off the alarm
+         call ESMF_AlarmRingerOff(stop_alarm, rc=rc )
+         if (ChkErr(rc,__LINE__,u_FILE_u)) return
+       end if
+    end if
+
+    if (write_restart .or. write_restartfh .or. write_restart_eor) then
       ! determine restart filename
       call ESMF_ClockGetNextTime(clock, MyTime, rc=rc)
       if (ChkErr(rc,__LINE__,u_FILE_u)) return

config_src/drivers/solo_driver/MOM_driver.F90

@@ -28,6 +28,7 @@ program MOM6
   use MOM_cpu_clock,       only : CLOCK_COMPONENT
   use MOM_data_override,   only : data_override_init
   use MOM_diag_mediator,   only : diag_mediator_end, diag_ctrl, diag_mediator_close_registration
+  use MOM_diag_manager_infra, only : diag_manager_set_time_end_infra
   use MOM,                 only : initialize_MOM, step_MOM, MOM_control_struct, MOM_end
   use MOM,                 only : extract_surface_state, finish_MOM_initialization
   use MOM,                 only : get_MOM_state_elements, MOM_state_is_synchronized
@@ -375,6 +376,8 @@ program MOM6
     Time_end = daymax
   endif
 
+  call diag_manager_set_time_end_infra(Time_end)
+
   call get_param(param_file, mod_name, "SINGLE_STEPPING_CALL", single_step_call, &
                  "If true, advance the state of MOM with a single step "//&
                  "including both dynamics and thermodynamics.  If false "//&

config_src/infra/FMS1/MOM_diag_manager_infra.F90

@@ -57,6 +57,8 @@ module MOM_diag_manager_infra
 public MOM_diag_manager_init
 public MOM_diag_manager_end
 public send_data_infra
+public diag_send_complete_infra
+public diag_manager_set_time_end_infra
 public MOM_diag_field_add_attribute
 public register_diag_field_infra
 public register_static_field_infra
@@ -451,4 +453,13 @@ subroutine MOM_diag_field_add_attribute_i1d(diag_field_id, att_name, att_value)
 
 end subroutine MOM_diag_field_add_attribute_i1d
 
+!> Needed for backwards compatibility, does nothing
+subroutine diag_send_complete_infra ()
+end subroutine diag_send_complete_infra
+
+!> Needed for backwards compatibility, does nothing
+subroutine diag_manager_set_time_end_infra(time)
+  type(time_type), intent(in) :: time !< The model time that simulation ends
+end subroutine diag_manager_set_time_end_infra
+
 end module MOM_diag_manager_infra

config_src/infra/FMS2/MOM_diag_manager_infra.F90

@@ -14,13 +14,15 @@ module MOM_diag_manager_infra
 use diag_data_mod,    only : null_axis_id
 use diag_manager_mod, only : fms_diag_manager_init => diag_manager_init
 use diag_manager_mod, only : fms_diag_manager_end => diag_manager_end
+use diag_manager_mod, only : diag_send_complete
+use diag_manager_mod, only : diag_manager_set_time_end
 use diag_manager_mod, only : send_data_fms => send_data
 use diag_manager_mod, only : fms_diag_field_add_attribute => diag_field_add_attribute
 use diag_manager_mod, only : DIAG_FIELD_NOT_FOUND
 use diag_manager_mod, only : register_diag_field_fms => register_diag_field
 use diag_manager_mod, only : register_static_field_fms => register_static_field
 use diag_manager_mod, only : get_diag_field_id_fms => get_diag_field_id
-use MOM_time_manager, only : time_type
+use MOM_time_manager, only : time_type, set_time
 use MOM_domain_infra, only : MOM_domain_type
 use MOM_error_infra,  only : MOM_error => MOM_err, FATAL, WARNING
 
@@ -57,6 +59,8 @@ module MOM_diag_manager_infra
 public MOM_diag_manager_init
 public MOM_diag_manager_end
 public send_data_infra
+public diag_send_complete_infra
+public diag_manager_set_time_end_infra
 public MOM_diag_field_add_attribute
 public register_diag_field_infra
 public register_static_field_infra
@@ -451,4 +455,19 @@ subroutine MOM_diag_field_add_attribute_i1d(diag_field_id, att_name, att_value)
 
 end subroutine MOM_diag_field_add_attribute_i1d
 
+!> Finishes the diag manager reduction methods as needed for the time_step
+subroutine diag_send_complete_infra ()
+  !! The time_step in the diag_send_complete call is a dummy argument, needed for backwards compatibility
+  !! It won't be used at all when diag_manager_nml::use_modern_diag=.true.
+  !! It won't have any impact when diag_manager_nml::use_modern_diag=.false.
+  call diag_send_complete (set_time(0))
+end subroutine diag_send_complete_infra
+
+!> Sets the time that the simulation ends in the diag manager
+subroutine diag_manager_set_time_end_infra(time)
+  type(time_type),           optional, intent(in) :: time  !< The time the simulation ends
+
+  call diag_manager_set_time_end(time)
+end subroutine diag_manager_set_time_end_infra
+
 end module MOM_diag_manager_infra

src/core/MOM_forcing_type.F90

@@ -126,9 +126,8 @@ module MOM_forcing_type
   real, pointer, dimension(:,:) :: &
     netMassIn     => NULL(), & !< Sum of water mass fluxes into the ocean integrated over a
                                !! forcing timestep [H ~> m or kg m-2]
-    netMassOut    => NULL(), & !< Net water mass flux out of the ocean integrated over a forcing timestep,
+    netMassOut    => NULL()    !< Net water mass flux out of the ocean integrated over a forcing timestep,
                                !! with negative values for water leaving the ocean [H ~> m or kg m-2]
-    KPP_salt_flux => NULL()    !< KPP effective salt flux [ppt m s-1]
 
   ! heat associated with water crossing ocean surface
   real, pointer, dimension(:,:) :: &