Merge pull request #12 from climbfuji/feature/move_exchange_for_jedi

Use JEDI_DRIVER to trigger ESMF exchanges in different places

CMakeLists.txt

@@ -35,6 +35,10 @@ else()
   remove_definitions(-DCCPP_32BIT)
 endif()
 
+if(JEDI_DRIVER)
+  add_definitions(-DJEDI_DRIVER)
+endif()
+
 if(NOT PARALLEL_NETCDF)
   list(APPEND _fv3atm_defs_private NO_PARALLEL_NETCDF)
 endif()

atmos_model.F90

@@ -122,6 +122,7 @@ module atmos_model_mod
 public get_atmos_model_ungridded_dim
 public atmos_model_get_nth_domain_info
 public addLsmask2grid
+public assign_importdata
 public setup_exportdata
 !-----------------------------------------------------------------------
 
@@ -292,9 +293,11 @@ subroutine update_atmos_radiation_physics (Atmos)
         if (ierr/=0)  call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed')
       endif
 
+#ifndef JEDI_DRIVER
 !--- if coupled, assign coupled fields
       call assign_importdata(jdat(:),rc)
       if (rc/=0)  call mpp_error(FATAL, 'Call to assign_importdata failed')
+#endif
 
       ! Currently for FV3ATM, it is only enabled for parent domain coupling
       ! with other model components. In this case, only the parent domain
@@ -998,10 +1001,12 @@ subroutine update_atmos_model_state (Atmos, rc)
     !--- get bottom layer data from dynamical core for coupling
     call atmosphere_get_bottom_layer (Atm_block, DYCORE_Data)
 
+#ifndef JEDI_DRIVER
     !--- if in coupled mode, set up coupled fields
     call setup_exportdata(rc=localrc)
     if (ESMF_LogFoundError(rcToCheck=localrc, msg=ESMF_LOGERR_PASSTHRU, &
       line=__LINE__, file=__FILE__, rcToReturn=rc)) return
+#endif
 
     !--- conditionally update the coordinate arrays for moving domains
     if (Atmos%is_moving_nest) then
@@ -1780,14 +1785,20 @@ subroutine update_atmos_chemistry(state, rc)
 end subroutine update_atmos_chemistry
 ! </SUBROUTINE>
 
+#ifdef JEDI_DRIVER
+  subroutine assign_importdata(rc)
+#else
   subroutine assign_importdata(jdat, rc)
-
+#endif
     use module_cplfields,  only: importFields, nImportFields, queryImportFields, &
                                  importFieldsValid
     use ESMF
 !
     implicit none
+    character(len=*),parameter :: subname='(atmos_model:assign_importdata)'
+#ifndef JEDI_DRIVER
     integer, intent(in)  :: jdat(8)
+#endif
     integer, intent(out) :: rc
 
     !--- local variables
@@ -1819,6 +1830,8 @@ subroutine assign_importdata(jdat, rc)
 !
     rc  = -999
 
+    if(mpp_pe() == mpp_root_pe()) print *, trim(subname) // ' start'
+
 ! set up local dimension
     isc = GFS_control%isc
     iec = GFS_control%isc+GFS_control%nx-1
@@ -1848,7 +1861,7 @@ subroutine assign_importdata(jdat, rc)
         mergeflg = .false.
         call ESMF_FieldGet(importFields(n), dimCount=dimCount ,typekind=datatype, &
                            name=impfield_name, rc=rc)
-          if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
+        if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
 
         if ( dimCount == 2) then
           if ( datatype == ESMF_TYPEKIND_R8) then
@@ -1872,6 +1885,7 @@ subroutine assign_importdata(jdat, rc)
         endif
 !
         if (found) then
+         if(mpp_pe() == mpp_root_pe()) print *, trim(subname) // ' / ' // trim(impfield_name)
          if (datar8(isc,jsc) > -99998.0) then
 !
         ! get sea land mask: in order to update the coupling fields over the ocean/ice
@@ -2736,6 +2750,7 @@ subroutine assign_importdata(jdat, rc)
           endif
         endif
 
+#ifndef JEDI_DRIVER
           ! write post merge import data to NetCDF file.
           if (GFS_control%cpl_imp_dbg) then
             call ESMF_FieldGet(importFields(n), grid=grid, rc=rc)
@@ -2753,6 +2768,7 @@ subroutine assign_importdata(jdat, rc)
             call ESMF_FieldDestroy(dbgField, rc=rc)
             if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
           endif
+#endif
 
         endif ! if (found) then
       endif   ! if (isFieldCreated) then
@@ -2828,6 +2844,8 @@ subroutine assign_importdata(jdat, rc)
 !
 
     rc=0
+
+    if(mpp_pe() == mpp_root_pe()) print *, trim(subname) // ' end'
 !
   end subroutine assign_importdata
 
@@ -2842,6 +2860,7 @@ subroutine setup_exportdata(rc)
     integer, optional, intent(out) :: rc
 
     !--- local variables
+    character(len=*),parameter :: subname='(atmos_model:setup_exportdata)'
     integer                :: i, j, k, ix
     integer                :: isc, iec, jsc, jec
     integer                :: nb, nk
@@ -2863,6 +2882,8 @@ subroutine setup_exportdata(rc)
     !--- begin
     if (present(rc)) rc = ESMF_SUCCESS
 
+    if(mpp_pe() == mpp_root_pe()) print *, trim(subname) // ' start'
+
     isc = Atm_block%isc
     iec = Atm_block%iec
     jsc = Atm_block%jsc
@@ -2915,6 +2936,7 @@ subroutine setup_exportdata(rc)
       if (isFound .and. GFS_control%cplchm) isFound = .not.any(trim(fieldname) == chemistryFieldNames)
 
       if (isFound) then
+        if(mpp_pe() == mpp_root_pe()) print *, trim(subname) // ' / ' // trim(fieldname)
 !$omp parallel do default(shared) private(nb) reduction(max:localrc)
         do nb = 1, Atm_block%nblks
           select case (trim(fieldname))
@@ -3356,6 +3378,8 @@ subroutine setup_exportdata(rc)
       if (mpp_pe() == mpp_root_pe()) print *,'zeroing coupling accumulated fields at kdt= ',GFS_control%kdt
     endif !cplflx or cpllnd
 
+    if(mpp_pe() == mpp_root_pe()) print *, trim(subname) // ' end'
+
   end subroutine setup_exportdata
 
   subroutine addLsmask2grid(fcstGrid, rc)

fv3_cap.F90

@@ -1228,7 +1228,7 @@ subroutine fv3_checkimport(gcomp, rc)
     date(1:6) = 0
     call ESMF_TimeGet(time=currTime,yy=date(1),mm=date(2),dd=date(3),h=date(4), &
                       m=date(5),s=date(6),rc=rc)
-    if(mype == 0) print *,'in fv3_checkimport, currtime=',date(1:6)
+    !if(mype == 0) print *,'in fv3_checkimport, currtime=',date(1:6)
 
     ! set up invalid time (by convention)
     call ESMF_TimeSet(invalidTime, yy=99999999, mm=01, dd=01, &
@@ -1248,12 +1248,12 @@ subroutine fv3_checkimport(gcomp, rc)
       do n = 1,size(fieldList)
         call ESMF_FieldGet(fieldList(n), name=fldname, rc=rc)
         if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
-        if(mype == 0) print *,'in fv3_checkimport, in fieldList loop for field ' // trim(fldname)
+        !if(mype == 0) print *,'in fv3_checkimport, in fieldList loop for field ' // trim(fldname)
 
         ! check if import field carries a valid timestamp
         call NUOPC_GetTimestamp(fieldList(n), isValid=isValid, rc=rc)
         if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
-        if(mype==0) print *,'in fv3_checkimport, NUOPC_GetTimestamp result for field ' // trim(fldname) // ' is ', isValid
+        !if(mype==0) print *,'in fv3_checkimport, NUOPC_GetTimestamp result for field ' // trim(fldname) // ' is ', isValid
 
         if (isValid) then
           ! if timestamp is set, check if it is valid

module_fcst_grid_comp.F90

@@ -72,7 +72,7 @@ module module_fcst_grid_comp
                                     nImportFields, importFields, importFieldsInfo
   use module_cplfields,       only: realizeConnectedCplFields
 
-  use atmos_model_mod,        only: setup_exportdata
+  use atmos_model_mod,        only: assign_importdata, setup_exportdata
 !
 !-----------------------------------------------------------------------
 !
@@ -1145,7 +1145,7 @@ subroutine fcst_run_phase_1(fcst_comp, importState, exportState,clock,rc)
 !
       logical,save               :: first=.true.
       integer,save               :: dt_cap=0
-      type(ESMF_Time)            :: currTime,stopTime
+      type(ESMF_Time)            :: currTime,stopTime,startTime
       integer                    :: mype, seconds
       real(kind=8)               :: mpi_wtime, tbeg1
 !
@@ -1165,7 +1165,7 @@ subroutine fcst_run_phase_1(fcst_comp, importState, exportState,clock,rc)
       n_atmsteps = seconds/dt_atmos
 
       if (first) then
-        call ESMF_ClockGet(clock, currTime=currTime, stopTime=stopTime, rc=rc)
+        call ESMF_ClockGet(clock, startTime=startTime, currTime=currTime, stopTime=stopTime, rc=rc)
         if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
 
         call ESMF_TimeIntervalGet(stopTime-currTime, s=dt_cap, rc=rc)
@@ -1179,6 +1179,16 @@ subroutine fcst_run_phase_1(fcst_comp, importState, exportState,clock,rc)
       else
         Atmos%isAtCapTime = .false.
       endif
+
+! DH*
+#ifdef JEDI_DRIVER
+      if (Atmos%isAtCapTime) then
+        call assign_importdata(rc)
+        if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+          line=__LINE__, file=__FILE__, rcToReturn=rc)) return
+      endif
+#endif
+! *DH
 !
 !-----------------------------------------------------------------------
 ! *** call fcst integration subroutines
@@ -1214,6 +1224,8 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc)
 !
 !***  local variables
 !
+      type(ESMF_Time)            :: currTime,stopTime,startTime
+      type(ESMF_TimeInterval)    :: timeStep
       integer                    :: mype, date(6), seconds
       character(len=64)          :: timestamp
       integer                    :: unit
@@ -1266,6 +1278,18 @@ subroutine fcst_run_phase_2(fcst_comp, importState, exportState,clock,rc)
           endif
       endif
 
+#ifdef JEDI_DRIVER
+      call ESMF_ClockGet(clock, startTime=startTime, currTime=currTime, stopTime=stopTime, &
+        timeStep=timeStep, rc=rc)
+      if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, line=__LINE__, file=__FILE__)) return
+      !
+      if (currTime+timeStep==stopTime) then
+        call setup_exportdata(rc)
+        if (ESMF_LogFoundError(rcToCheck=rc, msg=ESMF_LOGERR_PASSTHRU, &
+          line=__LINE__, file=__FILE__, rcToReturn=rc)) return
+      endif
+#endif
+
       if (mype == 0) write(*,'(A,I16,A,F16.6)')'PASS: fcstRUN phase 2, n_atmsteps = ', &
                                                n_atmsteps,' time is ',mpi_wtime()-tbeg1
 !