Add netCDF PIO capability for restarts and run-time history for dev/u…

…fs-weather-model (NOAA-EMC#1303) (#2)

Co-authored-by: Denise Worthen <denise.worthen@noaa.gov>

model/src/CMakeLists.txt

@@ -1,4 +1,3 @@
-
 # Open switch file
 file(STRINGS ${CMAKE_BINARY_DIR}/switch switch_strings)
 separate_arguments(switches UNIX_COMMAND ${switch_strings})
@@ -85,7 +84,7 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$")
   set(compile_flags -g -fno-second-underscore -ffree-line-length-none)
   set(compile_flags_release -O3)
   set(compile_flags_debug -Wall -fcheck=all -ffpe-trap=invalid,zero,overflow -frecursive -fbacktrace)
-  
+
   if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10)
     target_compile_options(ww3_lib PUBLIC -fallow-argument-mismatch)
   endif()
@@ -94,7 +93,7 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "PGI")
   set(compile_flags -g -i4 -r4 -Kieee)
   set(compile_flags_release -O3)
   set(compile_flags_debug -O0 -Mbounds -Mchkfpstk -Mchkstk -Mdalign -Mdclchk -Mdepchk -Miomutex -Ktrap=fp -Mrecursive -traceback)
-  
+
   if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10)
     target_compile_options(ww3_lib PUBLIC -fallow-argument-mismatch -fallow-invalid-boz)
   endif()
@@ -152,7 +151,7 @@ if(UFS_CAP)
   elseif(UFS_CAP STREQUAL "NUOPC_MESH")
     set(cap_src ${nuopc_mesh_cap_src})
   endif()
-  
+
   target_sources(ww3_lib PRIVATE ${cap_src})
   target_link_libraries(ww3_lib PUBLIC esmf)
   # Don't build executables when building WW3 ESMF library
@@ -231,7 +230,7 @@ install(
 
 install(FILES ${CMAKE_BINARY_DIR}/switch DESTINATION ${CMAKE_INSTALL_PREFIX})
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
-  
+
 
 export(EXPORT WW3Exports
   NAMESPACE WW3::

model/src/cmake/src_list.cmake

@@ -55,8 +55,7 @@ set(ftn_src
   wmupdtmd.F90
   wmwavemd.F90
   w3tidemd.F90
-  wav_grdout.F90
-  w3iogoncdmd.F90
+  wav_history_mod.F90
   wav_shr_flags.F90
   )
 
@@ -67,6 +66,8 @@ set(nuopc_mesh_cap_src
   wav_comp_nuopc.F90
   wav_import_export.F90
   wav_wrapper_mod.F90
+  wav_pio_mod.F90
+  wav_restart_mod.F90
   )
 
 set(esmf_multi_cap_src

model/src/w3gridmd.F90

@@ -836,7 +836,7 @@ MODULE W3GRIDMD
   !
 #ifdef W3_ST4
   INTEGER                 :: SWELLFPAR, SDSISO, SDSBRFDF
-  REAL 		   :: SDSBCHOICE
+  REAL                    :: SDSBCHOICE
   REAL                    :: ZWND, ALPHA0, Z0MAX, BETAMAX, SINTHP,&
        ZALP, Z0RAT, TAUWSHELTER, SWELLF,    &
        SWELLF2,SWELLF3,SWELLF4, SWELLF5,    &
@@ -3280,7 +3280,7 @@ SUBROUTINE W3GRID()
            JGS_TERMINATE_DIFFERENCE,                   &
            JGS_TERMINATE_NORM,                         &
            JGS_LIMITER,                                &
-           JGS_LIMITER_FUNC,                           & 
+           JGS_LIMITER_FUNC,                           &
            JGS_USE_JACOBI,                             &
            JGS_BLOCK_GAUSS_SEIDEL,                     &
            JGS_MAXITER,                                &
@@ -3617,7 +3617,7 @@ SUBROUTINE W3GRID()
       END SELECT
 
       IF (FSTOTALIMP .or. FSTOTALEXP) THEN
-        LPDLIB = .TRUE. 
+        LPDLIB = .TRUE.
       ENDIF
       !
       IF (SUM(UNSTSCHEMES).GT.1) WRITE(NDSO,1035)

model/src/w3initmd.F90

@@ -445,6 +445,10 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD,  FLGR2, FLGD,
 #ifdef W3_UOST
     USE W3UOSTMD, ONLY: UOST_SETGRID
 #endif
+    use w3timemd,        only : set_user_timestring
+    use w3odatmd,        only : runtype, restart_from_binary, use_restartnc, user_restfname
+    use w3odatmd,        only : logfile_is_assigned
+    use wav_restart_mod, only : read_restart
     !/
 #ifdef W3_MPI
     INCLUDE "mpif.h"
@@ -512,7 +516,10 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD,  FLGR2, FLGD,
 #ifdef W3_PDLIB
     INTEGER                 :: IScal(1), IPROC
 #endif
+    logical                 :: exists
     integer                 :: memunit
+    character(len=16)       :: user_timestring    !YYYY-MM-DD-SSSSS
+    character(len=1024)     :: fname
     !/
     !/ ------------------------------------------------------------------- /
     !
@@ -639,53 +646,52 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD,  FLGR2, FLGD,
     IF (FSTOTALIMP .and. .NOT. LPDLIB) THEN
       WRITE(NDSE,*) 'IMPTOTAL is selected'
       WRITE(NDSE,*) 'But PDLIB is not'
-      CALL FLUSH(NDSE) 
-      STOP 
+      CALL FLUSH(NDSE)
+      STOP
     ELSE IF (FSTOTALEXP .and. .NOT. LPDLIB) THEN
       WRITE(NDSE,*) 'EXPTOTAL is selected'
       WRITE(NDSE,*) 'But PDLIB is not'
-      CALL FLUSH(NDSE) 
-      STOP 
+      CALL FLUSH(NDSE)
+      STOP
     END IF
 #ifdef W3_PDLIB
     IF (B_JGS_BLOCK_GAUSS_SEIDEL .AND. .NOT. B_JGS_USE_JACOBI) THEN
       WRITE(NDSE,*) 'B_JGS_BLOCK_GAUSS_SEIDEL is used but the Jacobi solver is not choosen'
       WRITE(NDSE,*) 'Please set JGS_USE_JACOBI .eqv. .true.'
-      CALL FLUSH(NDSE) 
-      STOP 
+      CALL FLUSH(NDSE)
+      STOP
     ENDIF
 #endif
-
     !
     ! 1.c Open files without unpacking MDS ,,,
     !
-    IE     = LEN_TRIM(FEXT)
-    LFILE  = 'log.' // FEXT(:IE)
-    IFL    = LEN_TRIM(LFILE)
+    if (.not. logfile_is_assigned) then
+      IE     = LEN_TRIM(FEXT)
+      LFILE  = 'log.' // FEXT(:IE)
+      IFL    = LEN_TRIM(LFILE)
 #ifdef W3_SHRD
-    TFILE  = 'test.' // FEXT(:IE)
+      TFILE  = 'test.' // FEXT(:IE)
 #endif
 #ifdef W3_DIST
-    IW     = 1 + INT ( LOG10 ( REAL(NAPROC) + 0.5 ) )
-    IW     = MAX ( 3 , MIN ( 9 , IW ) )
-    WRITE (FORMAT,'(A5,I1.1,A1,I1.1,A4)')                    &
-         '(A4,I', IW, '.', IW, ',2A)'
-    WRITE (TFILE,FORMAT) 'test',                             &
-         OUTPTS(IMOD)%IAPROC, '.', FEXT(:IE)
-#endif
-    IFT    = LEN_TRIM(TFILE)
-    J      = LEN_TRIM(FNMPRE)
-    !
-#ifndef W3_CESMCOUPLED
-    IF ( OUTPTS(IMOD)%IAPROC .EQ. OUTPTS(IMOD)%NAPLOG )             &
-         OPEN (MDS(1),FILE=FNMPRE(:J)//LFILE(:IFL),ERR=888,IOSTAT=IERR)
-#endif
-    !
-    IF ( MDS(3).NE.MDS(1) .AND. MDS(3).NE.MDS(4) .AND. TSTOUT ) THEN
-      INQUIRE (MDS(3),OPENED=OPENED)
-      IF ( .NOT. OPENED ) OPEN (MDS(3),FILE=FNMPRE(:J)//TFILE(:IFT), ERR=889, &
-           IOSTAT=IERR)
-    END IF
+      IW     = 1 + INT ( LOG10 ( REAL(NAPROC) + 0.5 ) )
+      IW     = MAX ( 3 , MIN ( 9 , IW ) )
+      WRITE (FORMAT,'(A5,I1.1,A1,I1.1,A4)')                    &
+           '(A4,I', IW, '.', IW, ',2A)'
+      WRITE (TFILE,FORMAT) 'test',                             &
+           OUTPTS(IMOD)%IAPROC, '.', FEXT(:IE)
+#endif
+      IFT    = LEN_TRIM(TFILE)
+      J      = LEN_TRIM(FNMPRE)
+      !
+      IF ( OUTPTS(IMOD)%IAPROC .EQ. OUTPTS(IMOD)%NAPLOG )             &
+           OPEN (MDS(1),FILE=FNMPRE(:J)//LFILE(:IFL),ERR=888,IOSTAT=IERR)
+      !
+      IF ( MDS(3).NE.MDS(1) .AND. MDS(3).NE.MDS(4) .AND. TSTOUT ) THEN
+        INQUIRE (MDS(3),OPENED=OPENED)
+        IF ( .NOT. OPENED ) OPEN (MDS(3),FILE=FNMPRE(:J)//TFILE(:IFT), ERR=889, &
+             IOSTAT=IERR)
+      END IF
+    end if ! if (.not. logfile_is_assigned)
     !
     ! 1.d Dataset unit numbers
     !
@@ -725,6 +731,7 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD,  FLGR2, FLGD,
     ! 2.a Read model definition file
     !
     CALL W3IOGR ( 'READ', NDS(5), IMOD, FEXT )
+
     IF (GTYPE .eq. UNGTYPE) THEN
       CALL SPATIAL_GRID
       CALL NVECTRI
@@ -952,40 +959,64 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD,  FLGR2, FLGD,
     ! 3.a Read restart file
     !
     VA(:,:) = 0.
+    if (use_restartnc) then
+      if (runtype == 'continue' )then
+        call set_user_timestring(time,user_timestring)
+        if (restart_from_binary) then
+          fname = trim(user_restfname)//trim(user_timestring)
+        else
+          fname = trim(user_restfname)//trim(user_timestring)//'.nc'
+        endif
+        inquire(file=trim(fname), exist=exists)
+        if (exists) then
+          if (restart_from_binary) then
+            call w3iors('READ', nds(6), sig(nk), imod, filename=trim(fname))
+          else
+            call read_restart(trim(fname), va=va, mapsta=mapsta, mapst2=mapst2)
+          end if
+        else
+          call extcde (60, msg="required restart file " // trim(fname) // " does not exist")
+        end if
+      else
+        call read_restart('none')
+        ! mapst2 is module variable defined in read of mod_def; maptst is from 2.b above
+        flcold = .true.
+      end if
+    else
 #ifdef W3_DEBUGCOH
-    CALL ALL_VA_INTEGRAL_PRINT(IMOD, "Before W3IORS call", 1)
+      CALL ALL_VA_INTEGRAL_PRINT(IMOD, "Before W3IORS call", 1)
 #endif
 #ifdef W3_TIMINGS
-    CALL PRINT_MY_TIME("Before W3IORS")
+      CALL PRINT_MY_TIME("Before W3IORS")
 #endif
-    CALL W3IORS ( 'READ', NDS(6), SIG(NK), IMOD)
+      CALL W3IORS ( 'READ', NDS(6), SIG(NK), IMOD)
 #ifdef W3_TIMINGS
-    CALL PRINT_MY_TIME("After W3IORS")
+      CALL PRINT_MY_TIME("After W3IORS")
 #endif
-    call print_memcheck(memunit, 'memcheck_____:'//' WW3_INIT SECTION 3a')
+      call print_memcheck(memunit, 'memcheck_____:'//' WW3_INIT SECTION 3a')
 
 #ifdef W3_DEBUGCOH
-    CALL ALL_VA_INTEGRAL_PRINT(IMOD, "After W3IORS call", 1)
-#endif
-    FLCOLD = RSTYPE.LE.1  .OR. RSTYPE.EQ.4
-    IF ( IAPROC .EQ. NAPLOG ) THEN
-      IF (RSTYPE.EQ.0) THEN
-        WRITE (NDSO,930) 'cold start (idealized).'
-      ELSE IF ( RSTYPE .EQ. 1 ) THEN
-        WRITE (NDSO,930) 'cold start (wind).'
-      ELSE IF ( RSTYPE .EQ. 4 ) THEN
-        WRITE (NDSO,930) 'cold start (calm).'
-      ELSE
-        WRITE (NDSO,930) 'full restart.'
+      CALL ALL_VA_INTEGRAL_PRINT(IMOD, "After W3IORS call", 1)
+#endif
+      FLCOLD = RSTYPE.LE.1  .OR. RSTYPE.EQ.4
+      IF ( IAPROC .EQ. NAPLOG ) THEN
+        IF (RSTYPE.EQ.0) THEN
+          WRITE (NDSO,930) 'cold start (idealized).'
+        ELSE IF ( RSTYPE .EQ. 1 ) THEN
+          WRITE (NDSO,930) 'cold start (wind).'
+        ELSE IF ( RSTYPE .EQ. 4 ) THEN
+          WRITE (NDSO,930) 'cold start (calm).'
+        ELSE
+          WRITE (NDSO,930) 'full restart.'
+        END IF
       END IF
-    END IF
 #ifdef W3_DEBUGCOH
-    CALL ALL_VA_INTEGRAL_PRINT(IMOD, "W3INIT, step 4.2", 1)
+      CALL ALL_VA_INTEGRAL_PRINT(IMOD, "W3INIT, step 4.2", 1)
 #endif
 #ifdef W3_TIMINGS
-    CALL PRINT_MY_TIME("After restart inits")
+      CALL PRINT_MY_TIME("After restart inits")
 #endif
-
+    end if ! if (use_restartnc)
     !
     ! 3.b Compare MAPSTA from grid and restart
     !
@@ -1263,7 +1294,6 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD,  FLGR2, FLGD,
     !
     MAPTST = MOD(MAPST2/2,2)
     MAPST2 = MAPST2 - 2*MAPTST
-
     !
     !Li   For multi-resolution SMC grid, these 1-NX and 1-NY nested loops
     !Li   may miss the refined cells as they are not 1-1 corresponding to
@@ -1337,12 +1367,10 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD,  FLGR2, FLGD,
       CALL SET_IOBDP_PDLIB
     ENDIF
 #endif
-
     !
 #ifdef W3_DEBUGCOH
     CALL ALL_VA_INTEGRAL_PRINT(IMOD, "W3INIT, step 8.2", 1)
 #endif
-
     !
     MAPST2 = MAPST2 + 2*MAPTST
     !
@@ -1396,7 +1424,6 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD,  FLGR2, FLGD,
         !
       END DO
     END DO
-
     !
     ! 6.  Initialize arrays ---------------------------------------------- /
     !     Some initialized in W3IORS
@@ -1413,7 +1440,7 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD,  FLGR2, FLGD,
     !
     ! 7.  Write info to log file ----------------------------------------- /
     !
-    IF ( IAPROC .EQ. NAPLOG ) THEN
+    IF ( IAPROC .EQ. NAPLOG) THEN
       !
       WRITE (NDSO,970) GNAME
       IF (   FLLEV    ) WRITE (NDSO,971) 'Prescribed'
@@ -1500,7 +1527,9 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD,  FLGR2, FLGD,
         WRITE (NDSO,990) DTME21
       END IF
       !
-      WRITE (NDSO,984)
+      if (.not. logfile_is_assigned) then
+        WRITE (NDSO,984)
+      end if
       !
     END IF
     !
@@ -2171,6 +2200,7 @@ SUBROUTINE W3MPIO ( IMOD )
 #endif
     USE W3GDATMD, ONLY: GTYPE, UNGTYPE
     USE CONSTANTS, ONLY: LPDLIB
+    use w3odatmd, only : restart_from_binary, use_restartnc, use_historync
     !/
 #ifdef W3_MPI
     INCLUDE "mpif.h"
@@ -2202,6 +2232,7 @@ SUBROUTINE W3MPIO ( IMOD )
 #ifdef W3_MPIT
     CHARACTER(LEN=5)      :: STRING
 #endif
+    logical               :: do_rstsetup
     !/
     !/ ------------------------------------------------------------------- /
     !/
@@ -2225,7 +2256,7 @@ SUBROUTINE W3MPIO ( IMOD )
     IROOT  = NAPFLD - 1
     !
     !
-    IF ((FLOUT(1) .OR. FLOUT(7)) .and. (.not. LPDLIB)) THEN
+    IF ((FLOUT(1) .OR. FLOUT(7)) .and. (.not. LPDLIB) .and. (.not. use_historync)) THEN
       !
       ! NRQMAX is the maximum number of output fields that require MPI communication,
       ! aimed to gather field values stored in each processor into one processor in
@@ -4760,7 +4791,7 @@ SUBROUTINE W3MPIO ( IMOD )
         CALL EXTCDE (11)
       END IF
       !
-    END IF ! IF ((FLOUT(1) .OR. FLOUT(7)) .and. (.not. LPDLIB)) THEN
+    END IF ! IF ((FLOUT(1) .OR. FLOUT(7)) .and. (.not. LPDLIB) .and. (.not. use_historync)) THEN
     !
     ! 2.  Set-up for W3IORS ---------------------------------------------- /
     ! 2.a General preparations
@@ -4769,7 +4800,17 @@ SUBROUTINE W3MPIO ( IMOD )
     IH     = 0
     IROOT  = NAPRST - 1
     !
-    IF ((FLOUT(4) .OR. FLOUT(8)) .and. (.not. LPDLIB)) THEN
+    if (use_restartnc) then
+      if (restart_from_binary) then
+        do_rstsetup = .true.
+      else
+        do_rstsetup = .false.
+      end if
+    else
+      do_rstsetup = .true.
+    end if
+    !
+    IF ((FLOUT(4) .OR. FLOUT(8)) .and. (.not. LPDLIB) .and. do_rstsetup) THEN
       IF (OARST) THEN
         ALLOCATE ( OUTPTS(IMOD)%OUT4%IRQRS(34*NAPROC) )
       ELSE
@@ -5647,7 +5688,7 @@ SUBROUTINE W3MPIO ( IMOD )
         !
       END IF
       !
-    END IF ! IF ((FLOUT(4) .OR. FLOUT(8)) .and. (.not. LPDLIB)) THEN
+    END IF ! IF ((FLOUT(4) .OR. FLOUT(8)) .and. (.not. LPDLIB) .and. do_rstsetup) THEN
 #endif
     !
     ! 3.  Set-up for W3IOBC ( SENDs ) ------------------------------------ /