Add restart capability to job JGFS_ATMOS_POSTSND

jobs/JGFS_ATMOS_POSTSND

@@ -29,6 +29,10 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_HISTORY COM_ATMOS_BUFR \
 [[ ! -d ${COM_ATMOS_GEMPAK} ]] && mkdir -p "${COM_ATMOS_GEMPAK}"
 [[ ! -d ${COM_ATMOS_WMO} ]] && mkdir -p "${COM_ATMOS_WMO}"
 
+# Create a restart directory in ptmp to hold temporary output from the job JGFS_ATMOS_POSTSND
+
+export DATA_ATMOS_RESTART=${DATAROOT}/${RUN}.${PDY}/${cyc}/products/atmos/restart
+if [[ ! -d ${DATA_ATMOS_RESTART} ]]; then mkdir -p ${DATA_ATMOS_RESTART}; fi
 
 ########################################################
 # Execute the script.

scripts/exgfs_atmos_postsnd.sh

@@ -18,6 +18,7 @@
 #   7) 2018-07-18       Guang Ping Lou Generalize this version to other platforms
 #   8) 2019-10-18       Guang Ping Lou Transition to reading in NetCDF model data
 #   9) 2019-12-18       Guang Ping Lou generalizing to reading in NetCDF or nemsio
+#  10) 2024_05_15       Bo Cui  Add restart capability
 ################################################################
 
 source "${USHgfs}/preamble.sh"
@@ -48,9 +49,28 @@ GETDIM="${USHgfs}/getncdimlen"
 LEVS=$(${GETDIM} "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf000.${atmfm}" pfull)
 declare -x LEVS
 
+# Assume there was no run before and hence this is not a RESTART for job postsnd
+
+export RESTART_postsnd="NO"
+
+# However, if there was a run before, a directory DATA_ATMOS_RESTART must exist with data in it.
+
+nrestarts=$(find "$DATA_ATMOS_RESTART" -maxdepth 1 -type f -name '*.bufr.logf*' | wc -l)
+
+echo "${nrestarts}"
+
+if (( nrestarts == 0 )); then
+  echo "No restarts found in '${DATA_ATMOS_RESTART}', RESTART_postsnd='${RESTART_postsnd}'"
+
+else 
+  echo "Restarts found in '${DATA_ATMOS_RESTART}', RESTART_postsnd='YES'"
+  export RESTART_postsnd="YES"
+fi
+
 ### Loop for the hour and wait for the sigma and surface flux file:
 export FSTART=$STARTHOUR
 #
+
 while [ $FSTART -lt $ENDHOUR ]
 do
 export FINT=$NINT1
@@ -98,7 +118,10 @@ done
 ##############################################################
 cd "${COM_ATMOS_BUFR}" || exit 2
 tar -cf - . | /usr/bin/gzip > "${RUN}.${cycle}.bufrsnd.tar.gz"
-cd "${DATA}" || exit 2
+cd "${DATA}"
+if [ $? -ne 0 ]; then
+    exit 2
+fi
 
 ########################################
 # Send the single tar file to OSO

ush/gfs_bufr.sh

@@ -18,6 +18,7 @@
 # 2018-05-30  Guang Ping Lou: Make sure all files are available.
 # 2019-10-10  Guang Ping Lou: Read in NetCDF files
 # 2024-03-03 Bo Cui: Add options to use different bufr table for different resolution NetCDF files
+# 2024_05_15 Bo Cui: Add restart capability
 # echo "History: February 2003 - First implementation of this utility script"
 #
 source "${USHgfs}/preamble.sh"
@@ -88,19 +89,94 @@ case "${CASE}" in
         ${NLN} "${PARMgfs}/product/bufr_ij9km.txt"  fort.7
         ;;
     *)
-        echo "WARNING: No bufr table for this resolution, using the one for C768"
-        ${NLN} "${PARMgfs}/product/bufr_ij13km.txt" fort.7
+        echo "FATAL ERROR: Unrecognized bufr_ij*km.txt For CASE ${CASE}, ABORT!"
+        exit 1
         ;;
 esac
 
-${APRUN_POSTSND} "${EXECgfs}/${pgm}" < gfsparm > "out_gfs_bufr_${FEND}"
-export err=$?
 
-if [ $err -ne 0 ]; then
+if [[ ${RESTART_postsnd} == "YES" ]]; then
+
+  if [ -f "${DATA_ATMOS_RESTART}/${RUN}.${cycle}.bufr.logf${FEND}.${logfm}" ]; then
+
+    echo "Copy job postsnd files from restart directory"
+
+    cp -p "${DATA_ATMOS_RESTART}/${RUN}.${cycle}.bufr.logf${FEND}.${logfm}" .
+    while IFS= read -r fortname; do
+#     echo "Copy job postsnd files from restart directory: $fortname"
+      cp -p "${DATA_ATMOS_RESTART}/${RUN}.${cycle}.bufr_${fortname}" ${fortname}
+    done < "${RUN}.${cycle}.bufr.logf${FEND}.${logfm}"
+    err=0
+
+    if [[ ${FEND} -eq ${ENDHOUR} ]]; then
+      ${APRUN_POSTSND} "${EXECgfs}/${pgm}" < gfsparm > "out_gfs_bufr_${FEND}"
+      export err=$?
+    fi
+
+  else
+
+    echo "No more job postsnd restart file found in '${DATA_ATMOS_RESTART}'"
+    export RESTART_postsnd="NO"
+    echo "set RESTART_postsnd='${RESTART_postsnd}'"
+    ${APRUN_POSTSND} "${EXECgfs}/${pgm}" < gfsparm > "out_gfs_bufr_${FEND}"
+    export err=$?
+  fi
+
+else
+
+  ${APRUN_POSTSND} "${EXECgfs}/${pgm}" < gfsparm > "out_gfs_bufr_${FEND}"
+  export err=$?
+fi
+
+if [[ $err -ne 0 ]]; then
    echo "GFS postsnd job error, Please check files "
    echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.atmf${hh2}.${atmfm}"
    echo "${COM_ATMOS_HISTORY}/${RUN}.${cycle}.sfcf${hh2}.${atmfm}"
    err_chk
+
+else
+
+  # Count the number of restart files
+  nrestarts=$(find ./ -maxdepth 1 -type f -name 'fort.*' | wc -l)
+  find_exit_code=$?
+  if [ $find_exit_code -ne 0 ]; then
+    # handle the error, set the number of restart file is 0
+    nrestarts=0
+  else
+    echo "Number of restart fort.* files found: ${nrestarts}"
+  fi
+
+  # Check if there are restart files
+  if [[ "${nrestarts}" -gt 0 ]]; then
+    echo "Copying GFS postsnd files to restart directory..."
+
+    # Exclude specific files and save the rest to a log file
+    #ls fort.* | grep -v -e 'fort\.1' -e 'fort\.7' -e 'fort\.8' > "${RUN}.${cycle}.bufr.logf${FEND}.${logfm}"
+
+    # Initialize an empty array to store fort file names
+    files=()
+
+    # Loop through files in the directory
+    for file in fort.*; do
+      # Check if the file is not fort.1 or fort.7 or fort.8
+      if [[ ${file} != "fort.1" && $file != "fort.7" && $file != "fort.8" ]]; then
+        files+=("${file}")
+      fi
+    done
+
+    # Write the list of fort files to the log file
+    printf "%s\n" "${files[@]}" > "${RUN}.${cycle}.bufr.logf${FEND}.${logfm}"
+
+    # Copy each restart file to the restart directory
+    while IFS= read -r fortname; do
+    # echo "Copying restart file: $fortname"
+      cp -p "${fortname}" "${DATA_ATMOS_RESTART}/${RUN}.${cycle}.bufr_${fortname}"
+    done < "${RUN}.${cycle}.bufr.logf${FEND}.${logfm}"
+  fi
+
+    # Copy the log file to the restart directory
+    cp -p "${RUN}.${cycle}.bufr.logf${FEND}.${logfm}" "${DATA_ATMOS_RESTART}/"
+
 fi
 
 exit ${err}