Adjustment to wave workflow for running wave component on Hera

env/HERA.env

@@ -29,6 +29,7 @@ export NTHSTACK=1024000000
 export job=${PBS_JOBNAME:-$step}
 export jobid=${job}.${PBS_JOBID:-$$}
 
+
 if [ $step = "prep" -o $step = "prepbufr" ]; then
 
     nth_max=$(($npe_node_max / $npe_node_prep))
@@ -37,6 +38,11 @@ if [ $step = "prep" -o $step = "prepbufr" ]; then
     export BACK="NO"
     export sys_tp="HERA"
 
+elif [ $step = "waveinit" -o $step = "waveprep" -o $step = "wavepostsbs" ]; then
+
+    export mpmd="--multi-prog"
+    export CFP_MP="YES"
+
 elif [ $step = "anal" ]; then
 
     export MKL_NUM_THREADS=4

jobs/JWAVE_PREP

@@ -91,7 +91,9 @@ else
   if [ ! -L $ROTDIR/${WAVECUR_DID}.${RPDY} ]; then # Check if symlink already exists in ROTDIR
     $NLN $DMPDIR/${WAVECUR_DID}.${RPDY} $ROTDIR/${WAVECUR_DID}.${RPDY}
   fi
-  $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE}
+  if [ ! -L $ROTDIR/${CDUMP}.${PDY}/${cyc}/${WAVICEFILE} ]; then # Check if symlink already exists in ROTDIR
+    $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE}
+  fi
   export COMIN_OBS=${COMIN_OBS:-$ROTDIR/$RUN.$PDY/$cyc}
   export COMIN_WAV_ICE=${COMIN_OBS}
   export COMIN_WAV_WND=${COMIN_OBS}

modulefiles/module_base.hera

@@ -25,3 +25,6 @@ module load prod_util/1.1.0
 # python
 module use -a /contrib/modulefiles
 module load anaconda/2.3.0
+
+# waveprep
+module load cdo/1.9.5

parm/config/config.waveprep

@@ -32,8 +32,8 @@ export WAV_WND_HOUR_INC=1      # This value should match with the one used in
                          # the wind update script
 # Intake currents settings
 export WAV_CUR_DT=${WAV_CUR_DT:-3}
-export WAV_CUR_HF_DT=${WAV_CUR_HF_DT:-3}
-export WAV_CUR_HF_FH=${WAV_CUR_HF_FH:-72}
+export WAV_CUR_HF_DT=${WAV_CUR_HF_DT:-3} # Defaults to 3h for GEFSv12
+export WAV_CUR_HF_FH=${WAV_CUR_HF_FH:-0} # Constant DT for GFSv16 from getgo
 export WAV_CUR_CDO_SMOOTH="NO"
 
 # Location of CDO module

scripts/exwave_init.sh

@@ -129,7 +129,11 @@
       fi
 
       [[ ! -d $COMOUT/rundata ]] && mkdir -m 775 -p $COMOUT/rundata
-      echo "$USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile
+      if [ ${CFP_MP:-"NO"} = "YES" ]; then
+        echo "$nmoddef $USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile
+      else
+        echo "$USHwave/wave_grid_moddef.sh $grdID > $grdID.out 2>&1" >> cmdfile
+      fi
 
       nmoddef=`expr $nmoddef + 1`
 
@@ -162,7 +166,11 @@
 
     if [ "$NTASKS" -gt '1' ]
     then
-      ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile
+      if [ ${CFP_MP:-"NO"} = "YES" ]; then
+        ${wavempexec} -n ${wavenproc} ${wave_mpmd} cmdfile
+      else
+        ${wavempexec} ${wavenproc} ${wave_mpmd} cmdfile
+      fi
       exit=$?
     else
       ./cmdfile

scripts/exwave_prep.sh

@@ -638,55 +638,55 @@
       touch cmdfile
       chmod 744 cmdfile
 
-    ymdh_rtofs=${PDY}00 # RTOFS runs once daily
-    ymdh_end=`$NDATE ${FHMAX_WAV_CUR} ${ymdh_rtofs}`
-    NDATE_DT=${WAV_CUR_HF_DT}
-    FLGHF='T'
+      ymdh_rtofs=${PDY}00 # RTOFS runs once daily use ${PDY}00
+      ymdh_end=`$NDATE ${FHMAX_WAV_CUR} ${PDY}00`
+      NDATE_DT=${WAV_CUR_HF_DT}
+      FLGHF='T'
 
-    if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0 ; fi # Counter for MP CFP
-    while [ "$ymdh_rtofs" -le "$ymdh_end" ]
-    do
+      if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0 ; fi # Counter for MP CFP
+      while [ "$ymdh_rtofs" -le "$ymdh_end" ]
+      do
 # Timing has to be made relative to the single 00z RTOFS cycle for that PDY
-      fhr_rtofs=`${NHOUR} ${ymdh_rtofs} ${PDY}00`
-      fext='f'
+        fhr_rtofs=`${NHOUR} ${ymdh_rtofs} ${PDY}00`
+        fext='f'
 
-      fh3_rtofs=`printf "%03d" "${fhr_rtofs#0}"`
+        fh3_rtofs=`printf "%03d" "${fhr_rtofs#0}"`
 
-      curfile1h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_1hrly_prog.nc
-      curfile3h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_3hrly_prog.nc
+        curfile1h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_1hrly_prog.nc
+        curfile3h=${COMIN_WAV_CUR}/rtofs_glo_2ds_${fext}${fh3_rtofs}_3hrly_prog.nc
 
-      if [ -s ${curfile1h} ]  && [ "${FLGHF}" = "T" ] ; then
-        curfile=${curfile1h}
-      elif [ -s ${curfile3h} ]; then
-        curfile=${curfile3h}
-        FLGHF='F'
-      else
-        echo ' '
-        set $setoff
-        echo ' '
-        echo '************************************** '
-        echo "*** FATAL ERROR: NO CUR FILE $curfile ***  "
-        echo '************************************** '
-        echo ' '
-        set $seton
-        postmsg "$jlogfile" "FATAL ERROR - NO CURRENT FILE (RTOFS)"
-        err=11;export err;${errchk}
-        exit 0
-        echo ' '
-      fi
+        if [ -s ${curfile1h} ]  && [ "${FLGHF}" = "T" ] ; then
+          curfile=${curfile1h}
+        elif [ -s ${curfile3h} ]; then
+          curfile=${curfile3h}
+          FLGHF='F'
+        else
+          echo ' '
+          set $setoff
+          echo ' '
+          echo '************************************** '
+          echo "*** FATAL ERROR: NO CUR FILE $curfile ***  "
+          echo '************************************** '
+          echo ' '
+          set $seton
+          postmsg "$jlogfile" "FATAL ERROR - NO CURRENT FILE (RTOFS)"
+          err=11;export err;${errchk}
+          exit 0
+          echo ' '
+        fi
 
-      if [ ${CFP_MP:-"NO"} = "YES" ]; then
-        echo "$nm $USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile
-        nm=`expr $nm + 1`
-      else
-        echo "$USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile
-      fi
+        if [ ${CFP_MP:-"NO"} = "YES" ]; then
+          echo "$nm $USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile
+          nm=`expr $nm + 1`
+        else
+          echo "$USHwave/wave_prnc_cur.sh $ymdh_rtofs $curfile $fhr_rtofs > cur_$ymdh_rtofs.out 2>&1" >> cmdfile
+        fi
 
-      if [ $fhr_rtofs -ge ${WAV_CUR_HF_FH} ] ; then
-        NDATE_DT=${WAV_CUR_DT}
-      fi
-      ymdh_rtofs=`$NDATE $NDATE_DT $ymdh_rtofs`
-  done
+        if [ $fhr_rtofs -ge ${WAV_CUR_HF_FH} ] ; then
+          NDATE_DT=${WAV_CUR_DT}
+        fi
+        ymdh_rtofs=`$NDATE $NDATE_DT $ymdh_rtofs`
+      done
 
 # Set number of processes for mpmd
       wavenproc=`wc -l cmdfile | awk '{print $1}'`
@@ -747,7 +747,6 @@
       do
         echo $file
         cat $file >> cur.${WAVECUR_FID}
-        rm -f $file
       done
 
       cp -f cur.${WAVECUR_FID} ${COMOUT}/rundata/${COMPONENTwave}.${WAVECUR_FID}.$cycle.cur 

ush/wave_prnc_cur.sh

@@ -59,7 +59,7 @@ fi
 # Cleanup
 rm -f cur_temp[123].nc cur_5min_??.nc cur_glo_uv_${PDY}_${fext}${fh3}.nc weights.nc
 
-if [ ${fhr} -gt ${WAVHINDH} ] 
+if [ ${fhr} -gt 0 ] 
 then
   sed -e "s/HDRFL/F/g" ${FIXwave}/ww3_prnc.cur.${WAVECUR_FID}.inp.tmpl > ww3_prnc.inp
 else

ush/wave_tar.sh

@@ -30,7 +30,7 @@
 
   # Use LOUD variable to turn on/off trace.  Defaults to YES (on).
   export LOUD=${LOUD:-YES}; [[ $LOUD = yes ]] && export LOUD=YES
-  [[ "$LOUD" != YES ]] && set +x
+  [[ "$LOUD" != YES ]] && set -x
 
   cd $DATA
   postmsg "$jlogfile" "Making TAR FILE"
@@ -98,6 +98,7 @@
   set +x
   echo ' '
   echo '   Making tar file ...'
+  set -x
 
   count=0
   countMAX=5
@@ -106,14 +107,12 @@
   while [ "$count" -lt "$countMAX" ] && [ "$tardone" = 'no' ]
   do
 
-    [[ "$LOUD" = YES ]] && set -v
-    # JY nf=`ls $ID.*.$type | wc -l | awk '{ print $1 }'`
     nf=`ls | awk '/'$ID.*.$filext'/ {a++} END {print a}'`
-    if [ "$nf" = "$nb" ]
+    nbm2=$(( $nb - 2 ))
+    if [ $nf -ge $nbm2 ]
     then 
       tar -cf $ID.$cycle.${type}_tar ./$ID.*.$filext
       exit=$?
-      set +v; [[ "$LOUD" = YES ]] && set -x
 
       if  [ "$exit" != '0' ]
       then