Merge remote-tracking branch 'origin/develop' into feature/hpc-stack

Refs: #164

Externals.cfg

@@ -37,7 +37,7 @@ protocol = git
 required = True
 
 [EMC_verif-global]
-tag = verif_global_v1.13.5
+tag = verif_global_v2.0.2
 local_path = sorc/verif-global.fd
 repo_url = https://github.com/NOAA-EMC/EMC_verif-global.git
 protocol = git

docs/Release_Notes.gfs.v16.1.0.txt

@@ -0,0 +1,193 @@
+GFS V16.1.0 RELEASE NOTES
+
+
+PRELUDE
+
+  NOAA awarded Delivery Order 2 (DO-2) of its commercial radio occultation (RO) 
+  data purchase to GeoOptics on February 19, 2021.  This purchase covers 1300 
+  occultations a day over a six month period with the data flow starting on 
+  March 17, 2021.
+
+  The quality of the GeoOptics occultations was examined during Delivery 
+  Order 1 (DO-1) in December 2020 and January 2021.  Forecast impact assessment
+  was conducted using half resolution parallels in the GFSv16 framework.
+
+  Testing of the DO-2 data is being carried out using a real time full 
+  resolution parallel, v161rt1.  It was initially started in mid-February 
+  to test reproducibility with the GFSv16 NCO parallel, then adjusted to reduce
+  computational footprint.  Data from GeoOptics began to be assimilated as soon 
+  as it was available on March 17.  Configuration for the use of the commercial
+  data was informed by the results of DO-1 experimentation, with the quality 
+  control and observation errors being treated as other RO data with the 
+  exception of lowering the upper bound of data.
+
+  This package also addresses several GFS v16 bugzilla tickets.  GFS v16 
+  bugzilla tickets #1196 and #1205 are resolved in this package.   The error 
+  reported in bugzilla ticket #1206 has been documented to be compiler, not 
+  code, specific.  The DA aspect of GFS v16 bugzilla tickets #216, #1198, #1218,
+  #1221, and #1222 are also addressed by this package.  None of the DA 
+  bugzilla changes alter analysis results.
+
+  GFSv16 introduced a bug where the sign of the layer height (delz) increment 
+  was flipped.  The included bugfix reverts the sign as originally intended.  
+  This fix was tested in both a low resolution setting and with a full 
+  resolution parallel, v16rt2c.  Once v16rt2c was seen as viable, it also 
+  began assimilating the GeoOptics data so both changes were tested in one 
+  package.
+
+  There are several wave related fixes being included in this update:
+  * Including the RW-NH* boundary points which are used in NHC's 
+    offline NWPS system were removed from multi_1 to GFSv16, which 
+    also requires a minor script bug fix.  
+  * Fix the interpolation to the East Pacific wave grid to include 
+    all of the American Samoa islands. 
+  * Unmask the Gulf of California, Red Sea and Persian Gulf in the 
+    global 0p25 interpolated wave output grid. 
+
+
+CHANGES TO RESOURCES AND FILE SIZES
+
+  Impact of including RW-NHC points is: There should be no impact to the 
+  forecast job timing, however the *wave.out_pnt.points* files in the 
+  wave/rundata com directories will each increase in size by 6MB. The 
+  JGLOBAL_WAVE_POST_BNDPNTBLL and JGLOBAL_WAVE_POST_BNDPNT will now take 
+  approximately 20 minutes longer.  The gfswave.tZZz.ibpbull_tar will 
+  increase in size to be 78M, gfswave.tZZz.ibpcbull_tar will now be 42M 
+  and the gfswave.tZZz.ibp_tar file will now be 11 GB.  
+
+  Impact of East Pacific and global 0p25 wave grid fixes: There are no 
+  impacts in terms of resources or file size. 
+
+
+IMPLEMENTATION INSTRUCTIONS
+
+  The NOAA VLab and both the NOAA-EMC and NCAR organization spaces on GitHub.com 
+  are used to manage the GFS.v16.1 code. The SPA(s) handling the GFS.v16.1 
+  implementation need to have permissions to clone VLab gerrit repositories and 
+  the private NCAR UPP_GTG repository. All NOAA-EMC organization repositories are 
+  publicly readable and do not require access permissions.  Please follow the 
+  following steps to install the package on WCOSS-Dell:
+
+  1) cd $NWROOTp3
+
+  2) mkdir gfs.v16.1.0
+
+  3) cd gfs.v16.1.0
+
+  4) git clone -b EMC-v16.1.0  https://github.com/NOAA-EMC/global-workflow.git .
+
+  5) cd sorc
+
+  6) ./checkout.sh -o
+     * This script extracts the following GFS components:
+         MODEL     tag GFS.v16.0.16                  Jun.Wang@noaa.gov
+         GSI       tag gfsda.v16.1.0                 Russ.Treadon@noaa.gov
+         GLDAS     tag gldas_gfsv16_release.v1.12.0  Helin.Wei@noaa.gov
+         UFS_UTILS tag ops-gfsv16.0.0                George.Gayno@noaa.gov
+         POST      tag upp_gfsv16_release.v1.1.3     Wen.Meng@noaa.gov
+         WAFS      tag gfs_wafs.v6.0.21              Yali.Mao@noaa.gov
+
+  7) ./build_all.sh
+     * This script compiles all GFS components. Runtime output from the build for 
+       each package is written to log files in directory logs. To build an 
+       individual program, for instance, gsi, use build_gsi.sh.
+
+  8) ./link_fv3gfs.sh nco dell	
+
+
+SORC CHANGES
+
+* sorc/
+  * checkout.sh will checkout the following changed model tags:
+    * MODEL; tag GFS.v16.0.16 - There are no changes in this tag 
+      compared to operations.  The tag number was updated from 
+      tag GFS.v16.0.15 to incorporate the local changes made in operations.
+
+    * GSI; tag gfsda.v16.1.0 - See release notes 
+      doc/Release_Notes.gfsda.v16.1.0.txt in the GSI tag for details.
+
+    * POST; tag upp_gfsv16_release.v1.1.3 - There are no changes in this tag 
+      compared to operations.  The tag number was updated from 
+      tag upp_gfsv16_release.v1.1.1 to incorporate the local changes made 
+      in operations.
+
+    * WAFS; tag gfs_wafs.v6.0.21 - There are no changes in this tag compared to 
+      operations.  The tag number was updated from gfs_wafs.v6.0.17 to 
+      incorporate the local changes made in operations.
+
+
+JOBS CHANGES
+
+* No change from GFS v16.0
+
+
+PARM/CONFIG CHANGES
+
+* No change from GFS v16.0
+
+
+SCRIPT CHANGES
+
+* There is one bug fix for waves that is a script change in scripts/exgfs_wave_post_pnt.sh 
+  to ensure all point output is created (for example for the RW-NH* points). 
+
+
+FIX CHANGES
+
+* The contents of fix/fix_gsi have been updated.  See release notes 
+  doc/Release_Notes.gfsda.v16.1.0.txt in the NOAA-EMC/GSI tag gfsda.v16.1.0 
+  for details.
+
+* For the wave fix to include the RW-NH-* boundary points for the wave 
+  point output, the following two fix files can be copied to the 
+  fix/fix_wave_gfs folder: 
+  /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix_wave_gfs_v16.1.1/wave_gfs.buoys 
+  /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix_wave_gfs_v16.1.1/wave_gfs.buoys.full 
+
+
+* For the wave fix to the East Pacific wave grid interpolation the following 
+  fix file with fix can be copied to the fix/fix_wave_gfs folder: 
+  /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix_wave_gfs_v16.1.1/ep_10m_interp.inp.tmpl
+
+* For the wave fix to include the Gulf of California, Red Sea, and Persian Gulf 
+  the following to fix files need to be copied to the fix/fix_wave_gfs folder:
+  /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix_wave_gfs_v16.1.1/ww3_grid.inp.glo_15mxt
+  /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix_wave_gfs_v16.1.1/WHTGRIDINT.bin.glo_15mxt
+
+
+PRE-IMPLEMENTATION TESTING REQUIREMENTS
+
+* Which production jobs should be tested as part of this implementation?
+  * The entire GFS v16.1 package needs to be installed and tested. 
+
+* Does this change require a 30-day evaluation?
+  * No.
+
+
+DISSEMINATION INFORMATION
+
+* Where should this output be sent?
+  * No change from GFS v16.0
+
+* Who are the users?
+  * No change from GFS v16.0
+
+* Which output files should be transferred from PROD WCOSS to DEV WCOSS?
+  * No change from GFS v16.0
+
+* Directory changes
+  * No change from GFS v16.0
+
+* File changes
+  * No change from GFS v16.0
+
+
+HPSS ARCHIVE
+
+* No change from GFS v16.0
+
+
+JOB DEPENDENCIES AND FLOW DIAGRAM
+
+* No change from GFS v16.0
+

jobs/rocoto/arch.sh

@@ -109,20 +109,21 @@ if [ $CDUMP = "gfs" ]; then
     done
 fi
 
-# Archive atmospheric gaussian gfs forecast files for fit2obs
-VFYARC=${VFYARC:-$ROTDIR/vrfyarch}
-[[ ! -d $VFYARC ]] && mkdir -p $VFYARC
+# Archive required gaussian gfs forecast files for Fit2Obs
 if [ $CDUMP = "gfs" -a $FITSARC = "YES" ]; then
+    VFYARC=${VFYARC:-$ROTDIR/vrfyarch}
+    [[ ! -d $VFYARC ]] && mkdir -p $VFYARC
     mkdir -p $VFYARC/${CDUMP}.$PDY/$cyc
+    prefix=${CDUMP}.t${cyc}z
     fhmax=${FHMAX_FITS:-$FHMAX_GFS}
     fhr=0
     while [[ $fhr -le $fhmax ]]; do
-      fhr3=$(printf %03i $fhr)
-      sfcfile=${CDUMP}.t${cyc}z.sfcf${fhr3}${ASUFFIX}
-      sigfile=${CDUMP}.t${cyc}z.atmf${fhr3}${ASUFFIX}
-      $NCP $sfcfile $VFYARC/${CDUMP}.$PDY/$cyc/
-      $NCP $sigfile $VFYARC/${CDUMP}.$PDY/$cyc/
-      (( fhr = $fhr + 6 ))
+	fhr3=$(printf %03i $fhr)
+	sfcfile=${prefix}.sfcf${fhr3}${ASUFFIX}
+	sigfile=${prefix}.atmf${fhr3}${ASUFFIX}
+	$NCP $sfcfile $VFYARC/${CDUMP}.$PDY/$cyc/
+	$NCP $sigfile $VFYARC/${CDUMP}.$PDY/$cyc/
+	(( fhr = $fhr + 6 ))
     done
 fi
 
@@ -307,9 +308,10 @@ if [[ "${DELETE_COM_IN_ARCHIVE_JOB:-YES}" == NO ]] ; then
     exit 0
 fi
 
-# Step back every assim_freq hours
-# and remove old rotating directories for successful cycles
-# defaults from 24h to 120h
+# Step back every assim_freq hours and remove old rotating directories 
+# for successful cycles (defaults from 24h to 120h).  If GLDAS is
+# active, retain files needed by GLDAS update.  Independent of GLDAS, 
+# retain files needed by Fit2Obs
 DO_GLDAS=${DO_GLDAS:-"NO"}
 GDATEEND=$($NDATE -${RMOLDEND:-24}  $CDATE)
 GDATE=$($NDATE -${RMOLDSTD:-120} $CDATE)
@@ -330,14 +332,26 @@ while [ $GDATE -le $GDATEEND ]; do
                 if [ -d $COMINwave ]; then rm -rf $COMINwave ; fi
                 if [ -d $COMINrtofs -a $GDATE -lt $RTOFS_DATE ]; then rm -rf $COMINrtofs ; fi
                 if [ $CDUMP != "gdas" -o $DO_GLDAS = "NO" -o $GDATE -lt $GLDAS_DATE ]; then 
-                    rm -rf $COMIN 
+		    if [ $CDUMP = "gdas" ]; then
+                        for file in `ls $COMIN |grep -v prepbufr |grep -v cnvstat |grep -v atmanl.nc`; do
+                            rm -rf $COMIN/$file
+                        done
+		    else
+			rm -rf $COMIN
+		    fi
                 else
-                    for file in `ls $COMIN |grep -v sflux |grep -v RESTART`; do
-                        rm -rf $COMIN/$file
-                    done
-                    for file in `ls $COMIN/RESTART |grep -v sfcanl `; do
-                        rm -rf $COMIN/RESTART/$file
-                    done
+		    if [ $DO_GLDAS = "YES" ]; then
+			for file in `ls $COMIN |grep -v sflux |grep -v RESTART |grep -v prepbufr |grep -v cnvstat |grep -v atmanl.nc`; do
+                            rm -rf $COMIN/$file
+			done
+			for file in `ls $COMIN/RESTART |grep -v sfcanl `; do
+                            rm -rf $COMIN/RESTART/$file
+			done
+		    else
+                        for file in `ls $COMIN |grep -v prepbufr |grep -v cnvstat |grep -v atmanl.nc`; do
+                            rm -rf $COMIN/$file
+                        done
+		    fi
                 fi
             fi
 	fi
@@ -361,10 +375,13 @@ while [ $GDATE -le $GDATEEND ]; do
     GDATE=$($NDATE +$assim_freq $GDATE)
 done
 
-# Remove archived atmospheric gaussian files used for fit2obs in $VFYARC that are $FHMAX_FITS hrs behind.
-# touch existing files to prevent the files from being removed by the operation system.
+# Remove archived gaussian files used for Fit2Obs in $VFYARC that are 
+# $FHMAX_FITS plus a delta before $CDATE.  Touch existing archived 
+# gaussian files to prevent the files from being removed by automatic 
+# scrubber present on some machines.
+
 if [ $CDUMP = "gfs" ]; then
-    fhmax=$((FHMAX_FITS+36))       
+    fhmax=$((FHMAX_FITS+36))
     RDATE=$($NDATE -$fhmax $CDATE)
     rPDY=$(echo $RDATE | cut -c1-8)
     COMIN="$VFYARC/$CDUMP.$rPDY"
@@ -380,5 +397,17 @@ if [ $CDUMP = "gfs" ]; then
     done
 fi
 
+# Remove $CDUMP.$rPDY for the older of GDATE or RDATE
+GDATE=$($NDATE -${RMOLDSTD:-120} $CDATE)
+fhmax=$FHMAX_GFS
+RDATE=$($NDATE -$fhmax $CDATE)
+if [ $GDATE -lt $RDATE ]; then
+    RDATE=$GDATE
+fi
+rPDY=$(echo $RDATE | cut -c1-8)
+COMIN="$ROTDIR/$CDUMP.$rPDY"
+[[ -d $COMIN ]] && rm -rf $COMIN
+
+
 ###############################################################
 exit 0

jobs/rocoto/earc.sh

@@ -159,21 +159,29 @@ if [ $ENSGRP -eq 0 ]; then
 	gPDY=$(echo $GDATE | cut -c1-8)
 	gcyc=$(echo $GDATE | cut -c9-10)
 
-	# Handle GDAS and GFS EnKF directories separately
-	COMIN_ENS="$ROTDIR/enkfgdas.$gPDY/$gcyc/$COMPONENT"
-	[[ -d $COMIN_ENS ]] && rm -rf $COMIN_ENS
-	COMIN_ENS="$ROTDIR/enkfgfs.$gPDY/$gcyc/$COMPONENT"
-	[[ -d $COMIN_ENS ]] && rm -rf $COMIN_ENS
-
-	# Remove any empty directories
-	COMIN_ENS="$ROTDIR/enkfgdas.$gPDY/$COMPONENT"
-	if [ -d $COMIN_ENS ] ; then
-	    [[ ! "$(ls -A $COMIN_ENS)" ]] && rm -rf $COMIN_ENS
-	fi
-	COMIN_ENS="$ROTDIR/enkfgfs.$gPDY/$COMPONENT"
-	if [ -d $COMIN_ENS ] ; then
-	    [[ ! "$(ls -A $COMIN_ENS)" ]] && rm -rf $COMIN_ENS
-	fi
+	# Loop over GDAS and GFS EnKF directories separately.
+        clist="gdas gfs"
+	for ctype in $clist; do
+	    COMIN_ENS="$ROTDIR/enkf$ctype.$gPDY/$gcyc/$COMPONENT"
+            if [ -d $COMIN_ENS ]; then
+		rocotolog="$EXPDIR/logs/${GDATE}.log"
+		if [ -f $rocotolog ]; then
+		    testend=$(tail -n 1 $rocotolog | grep "This cycle is complete: Success")
+		    rc=$?
+		    if [ $rc -eq 0 ]; then
+                        # Retain f006.ens files.  Remove everything else
+			for file in `ls $COMIN_ENS | grep -v f006.ens`; do
+			    rm -rf $COMIN_ENS/$file
+			done
+		    fi
+		fi
+	    fi
+
+	    # Remove empty directories
+	    if [ -d $COMIN_ENS ] ; then
+		[[ ! "$(ls -A $COMIN_ENS)" ]] && rm -rf $COMIN_ENS
+	    fi
+	done
 
 	# Advance to next cycle
 	GDATE=$($NDATE +$assim_freq $GDATE)
@@ -182,5 +190,19 @@ if [ $ENSGRP -eq 0 ]; then
 
 fi
 
+# Remove enkf*.$rPDY for the older of GDATE or RDATE
+GDATE=$($NDATE -${RMOLDSTD_ENKF:-120} $CDATE)
+fhmax=$FHMAX_GFS
+RDATE=$($NDATE -$fhmax $CDATE)
+if [ $GDATE -lt $RDATE ]; then
+    RDATE=$GDATE
+fi
+rPDY=$(echo $RDATE | cut -c1-8)
+clist="gdas gfs"
+for ctype in $clist; do
+    COMIN="$ROTDIR/enkf$ctype.$rPDY"
+    [[ -d $COMIN ]] && rm -rf $COMIN
+done
+
 ###############################################################
 exit 0