RTMA updates (NOAA-EMC#473)

* RTMA updates
* make ifi grib files optional

scripts/exregional_process_bufr.sh

@@ -169,7 +169,7 @@ cp_vrfy $BUFR_TABLE prepobs_prep.bufrtable
 #   set observation soruce 
 #
 #-----------------------------------------------------------------------
-if [[ "${NET}" = "RTMA"* ]]; then
+if [[ "${NET}" = "RTMA"* ]] && [[ "${RTMA_OBS_FEED}" = "NCO" ]]; then
   SUBH=$(date +%M -d "${START_DATE}")
   obs_source="rtma_ru"
   obsfileprefix=${obs_source}

scripts/exregional_run_analysis.sh

@@ -432,7 +432,7 @@ sed -i "s/hh/${HH}/"     coupler.res
 # copy observation files to working directory 
 #
 #-----------------------------------------------------------------------
-if [[ "${NET}" = "RTMA"* ]]; then
+if [[ "${NET}" = "RTMA"* ]] && [[ "${RTMA_OBS_FEED}" = "NCO" ]]; then
   SUBH=$(date +%M -d "${START_DATE}")
   obs_source="rtma_ru"
   obsfileprefix=${obs_source}

scripts/exregional_run_post.sh

@@ -336,7 +336,9 @@ bgifi=${postprd_dir}/${NET}.t${cyc}z.bgifif${subh_fhr}.${tmmark}.grib2
 
 wgrib2 PRSLEV.GrbF${post_fhr} -set center 7 -grib ${bgdawp}
 wgrib2 NATLEV.GrbF${post_fhr} -set center 7 -grib ${bgrd3d}
-wgrib2 IFIFIP.GrbF${post_fhr} -set center 7 -grib ${bgifi}
+if [ -f IFIFIP.GrbF${post_fhr} ]; then
+  wgrib2 IFIFIP.GrbF${post_fhr} -set center 7 -grib ${bgifi}
+fi
 
 #
 #-----------------------------------------------------------------------

scripts/exregional_run_prdgen.sh

@@ -234,20 +234,26 @@ fi
 basetime=$( date +%y%j%H%M -d "${yyyymmdd} ${hh}" )
 cp_vrfy ${bgdawp} ${comout}/${NET}.t${cyc}z.bgdawpf${fhr}.${tmmark}.grib2
 cp_vrfy ${bgrd3d} ${comout}/${NET}.t${cyc}z.bgrd3df${fhr}.${tmmark}.grib2
-cp_vrfy ${bgifi} ${comout}/${NET}.t${cyc}z.bgifif${fhr}.${tmmark}.grib2
+if [ -f  ${bgifi} ]; then
+  cp_vrfy ${bgifi} ${comout}/${NET}.t${cyc}z.bgifif${fhr}.${tmmark}.grib2
+fi
 cp_vrfy ${bgsfc}  ${comout}/${NET}.t${cyc}z.bgsfcf${fhr}.${tmmark}.grib2
 ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgdawpf${fhr}.${tmmark}.grib2 ${comout}/BGDAWP_${basetime}${post_fhr}
 ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgrd3df${fhr}.${tmmark}.grib2 ${comout}/BGRD3D_${basetime}${post_fhr}
-ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgifif${fhr}.${tmmark}.grib2 ${comout}/BGIFI_${basetime}${post_fhr}
+if [ -f ${comout}/${NET}.t${cyc}z.bgifif${fhr}.${tmmark}.grib2 ]; then
+  ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgifif${fhr}.${tmmark}.grib2 ${comout}/BGIFI_${basetime}${post_fhr}
+fi
 ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgsfcf${fhr}.${tmmark}.grib2  ${comout}/BGSFC_${basetime}${post_fhr}
 
 net4=$(echo ${NET:0:4} | tr '[:upper:]' '[:lower:]')
 ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgdawpf${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.prslev.f${fhr}.${gridname}grib2
 wgrib2 ${comout}/${net4}.t${cyc}z.prslev.f${fhr}.${gridname}grib2 -s > ${comout}/${net4}.t${cyc}z.prslev.f${fhr}.${gridname}grib2.idx
 ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgrd3df${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.natlev.f${fhr}.${gridname}grib2
 wgrib2 ${comout}/${net4}.t${cyc}z.natlev.f${fhr}.${gridname}grib2 -s > ${comout}/${net4}.t${cyc}z.natlev.f${fhr}.${gridname}grib2.idx
-ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgifif${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.${gridname}grib2
-wgrib2 ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.${gridname}grib2 -s > ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.${gridname}grib2.idx
+if [ -f ${comout}/${NET}.t${cyc}z.bgifif${fhr}.${tmmark}.grib2 ]; then
+  ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgifif${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.${gridname}grib2
+  wgrib2 ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.${gridname}grib2 -s > ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.${gridname}grib2.idx
+fi
 ln_vrfy -sf --relative ${comout}/${NET}.t${cyc}z.bgsfcf${fhr}.${tmmark}.grib2  ${comout}/${net4}.t${cyc}z.testbed.f${fhr}.${gridname}grib2
 wgrib2 ${comout}/${net4}.t${cyc}z.testbed.f${fhr}.${gridname}grib2 -s > ${comout}/${net4}.t${cyc}z.testbed.f${fhr}.${gridname}grib2.idx
 # Remap to additional output grids if requested
@@ -457,7 +463,7 @@ if [ ${#ADDNL_OUTPUT_GRIDS[@]} -gt 0 ]; then
          wgrib2 ${comout}/${net4}.t${cyc}z.natlev.f${fhr}.${gridname}grib2 -s > ${comout}/${net4}.t${cyc}z.natlev.f${fhr}.${gridname}grib2.idx
       fi
 
-      if [ $leveltype = 'ifi' ]; then
+      if [[ $leveltype = 'ifi' ]] && [[ -f ${comout}/${grid}_grid/${NET}.t${cyc}z.bg${leveltype}f${fhr}.${tmmark}.grib2  ]]; then
          ln_vrfy -fs --relative ${comout}/${grid}_grid/${NET}.t${cyc}z.bg${leveltype}f${fhr}.${tmmark}.grib2 ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.${gridname}grib2
          wgrib2 ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.${gridname}grib2 -s > ${comout}/${net4}.t${cyc}z.ififip.f${fhr}.${gridname}grib2.idx
       fi

ush/config.sh.RTMA_CONUS_3km

@@ -1,9 +1,9 @@
 #!/bin/bash
 source fix_rrfs_locations.sh #determine current platform
 MACHINE=${PLATFORM}
+version="v0.3.1"
 ACCOUNT="nrtrr"
 RESERVATION="rrfsdet"
-RESERVATION_POST="rrfsdet"
 EXPT_BASEDIR="/home/rtrr/RTMA"
 EXPT_SUBDIR="RTMA_CONUS"
 
@@ -16,16 +16,20 @@ if [[ -n $RESERVATION ]] ; then
   PARTITION_ANALYSIS=kjet
   PARTITION_PRDGEN=kjet
   PARTITION_POST=kjet
-
   QUEUE_DEFAULT="rt"
   QUEUE_FCST="rt"
   QUEUE_ANALYSIS="rt"
   QUEUE_PRDGEN="rt"
   QUEUE_GRAPHICS="rt"
   QUEUE_HPSS="rt"
-  if [[ -n ${RESERVATION_POST} ]] ; then
-    QUEUE_POST="rt"
-  fi
+  QUEUE_POST="rt"
+  RESERVATION_POST="rrfsdet"
+  NNODES_MAKE_ICS="3"
+  PPN_MAKE_ICS="20"
+  NNODES_MAKE_LBCS="3"
+  PPN_MAKE_LBCS="20"
+  NNODES_RUN_POST="1"
+  PPN_RUN_POST="40"
 fi
 
 OBSPATH_NSSLMOSIAC=/public/data/radar/nssl/mrms/conus
@@ -34,18 +38,10 @@ if [[ $MACHINE == "hera" ]] ; then
   PARTITION_DEFAULT=""
   PARTITION_FCST=""
   QUEUE_ANALYSIS="batch"
-  QUEUE_PRDGEN="batch"
   QUEUE_GRAPHICS="batch"
 fi
 
-if [[ $MACHINE == "orion" ]] ; then
-  ACCOUNT=ztrr
-  QUEUE_ANALYSIS="batch"
-fi
-
-
 VERBOSE="TRUE"
-
 RUN_ENVIR="nco"
 
 PREDEF_GRID_NAME=RRFS_CONUS_3km
@@ -55,16 +51,20 @@ TILE_LABELS="CONUS REGIONS"
 TILE_SETS="full NE,NC,NW,SE,SC,SW"
 
 DO_DACYCLE="TRUE"
-#DO_SURFACE_CYCLE="FALSE"
-#DO_SPINUP="FALSE"
-#DO_RETRO="TRUE"
-DO_NONVAR_CLDANAL="TRUE"
-#DO_REFL2TTEN="FALSE"
+DO_SURFACE_CYCLE="FALSE"
+DO_SPINUP="FALSE"
+DO_POST_SPINUP="FALSE"
+DO_POST_PROD="TRUE" ##
+DO_RETRO="FALSE"
+DO_NONVAR_CLDANAL="TRUE" ##
+DO_ENVAR_RADAR_REF="FALSE"
+DO_SMOKE_DUST="FALSE"
+DO_REFL2TTEN="FALSE"
 RADARREFL_TIMELEVEL=(0)
 #FH_DFI_RADAR="0.0,0.25,0.5"
-#DO_SOIL_ADJUST="TRUE"
-#DO_RADDA="FALSE"
-#DO_BUFRSND="FALSE"
+DO_SOIL_ADJUST="FALSE"
+DO_RADDA="FALSE"
+DO_BUFRSND="FALSE"
 #
 #-------------------------------------------------------------------------------------
 #      GSI Namelist parameters configurable across differnt applications
@@ -125,22 +125,31 @@ EXTRN_MDL_LBCS_OFFSET_HRS="0"
 EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS="0"
 BOUNDARY_LEN_HRS="1"
 BOUNDARY_LONG_LEN_HRS="1"
+BOUNDARY_PROC_GROUP_NUM="10"
 
-DATE_FIRST_CYCL="20220801"
-DATE_LAST_CYCL="20221231"
+DATE_FIRST_CYCL="20221229"
+DATE_LAST_CYCL="20230131"
 CYCL_HRS=( "00" "12" )
 CYCL_HRS_SPINSTART=("03" "15")
 CYCL_HRS_PRODSTART=("99") # RTMA always warm-starts
 CYCLEMONTH="8-12"
 #CYCLEDAY="1-4"
 
+STARTYEAR=${DATE_FIRST_CYCL:0:4}
+STARTMONTH=${DATE_FIRST_CYCL:4:2}
+STARTDAY=${DATE_FIRST_CYCL:6:2}
+STARTHOUR="00"
+ENDYEAR=${DATE_LAST_CYCL:0:4}
+ENDMONTH=${DATE_LAST_CYCL:4:2}
+ENDDAY=${DATE_LAST_CYCL:6:2}
+ENDHOUR="23"
+
 PREEXISTING_DIR_METHOD="upgrade"
 #INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}0300 ${DATE_LAST_CYCL}2300 12:00:00"
 #BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
 PROD_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 01:00:00"
-POSTPROC_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 01:00:00"
-POSTPROC_LONG_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
-ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}0700 ${DATE_LAST_CYCL}2300 24:00:00"
+#PRODLONG_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 12:00:00"
+#ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}0700 ${DATE_LAST_CYCL}2300 24:00:00"
 if [[ $DO_SPINUP == "TRUE" ]] ; then
   SPINUP_CYCLEDEF="00 03-08,15-20 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
 fi
@@ -154,31 +163,56 @@ for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=0.0003; done
 DA_CYCLE_INTERV="1" #need to be 1 to have hourly DA
 RESTART_INTERVAL="1"
 
-USE_RRFSE_ENS="TRUE"
+USE_RRFSE_ENS="FALSE"
 CYCL_HRS_HYB_FV3LAM_ENS=("19" "20" "21" "22" "23" "00")
 
+#SST_update_hour=01
+#GVF_update_hour=04
+#SNOWICE_update_hour=00
+netcdf_diag=.true.
+binary_diag=.false.
+#SOIL_SURGERY_time=2022072404
+
+DT_ATMOS=1
 NCORES_RUN_ANAL=240
+#NCORES_RUN_OBSERVER=80
 WTIME_RUN_ANAL="01:00:00"
 HYBENSMEM_NMIN=66
 HALO_BLEND=20
 IO_LAYOUT_Y=1
 PPN_RUN_REF2TTEN=1
 PPN_RUN_NONVARCLDANL=${IO_LAYOUT_Y}
+regional_ensemble_option=1
 
-#PRINT_DIFF_PGR="TRUE"
-
-if [[ -n $RESERVATION ]] ; then
-  NNODES_MAKE_ICS="3"
-  PPN_MAKE_ICS="20"
-  NNODES_MAKE_LBCS="3"
-  PPN_MAKE_LBCS="20"
-  NNODES_RUN_POST="1"
-  PPN_RUN_POST="40"
-fi
+MAXTRIES_MAKE_ICS="3"
+MAXTRIES_MAKE_LBCS="3"
+MAXTRIES_RUN_PREPSTART="3"
+MAXTRIES_RUN_FCST="3"
+MAXTRIES_ANAL_GSI="3"
+MAXTRIES_CLDANL_NONVAR="3"
+MAXTRIES_RUN_POST="3"
+MAXTRIES_RUN_PRDGEN="3"
+#adjust to low priority queue and possible preemption, slurm socket error, etc
+MAXTRIES_PROCESS_RADARREF="3"
+MAXTRIES_PROCESS_LIGHTNING="3"
+MAXTRIES_PROCESS_BUFR="3"
 
 WTIME_RUN_FCST="00:20:00"
+WTIME_RUN_FCST_LONG="03:45:00"
 WTIME_MAKE_LBCS="02:00:00"
+WTIME_RUN_ANAL="00:50:00"
+WTIME_RUN_PREPSTART="00:20:00"
+
+START_TIME_SPINUP="02:10:00"
+START_TIME_PROD="08:20:00"
+START_TIME_LATE_ANALYSIS="08:40:00"
+START_TIME_CONVENTIONAL_SPINUP="00:40:00"
+START_TIME_CONVENTIONAL="00:40:00"
+START_TIME_NSSLMOSIAC="00:45:00"
+START_TIME_LIGHTNINGNC="00:45:00"
+START_TIME_PROCSMOKE="00:45:00"
 
+#PRINT_DIFF_PGR="TRUE"
 EXTRN_MDL_NAME_ICS="FV3GFS"
 EXTRN_MDL_NAME_LBCS="FV3GFS"
 
@@ -191,28 +225,17 @@ NET="RTMA_CONUS" ##must start with "RTMA". ${NET}.t${cyc}z.bgdawpf${fhr}.${tmmar
 TAG="RTMA_CONUS" ## used by job names
 IS_RTMA="TRUE"
 NCORES_PER_NODE=40 #kjet
-FG_ROOTDIR="/lfs4/BMC/nrtrr/NCO_dirs/nwges/para/RRFS_CONUS" #only needed by RTMA
-DT_ATMOS=1
-#adjust to low priority queue and possible preemption, slurm socket error, etc
-MAXTRIES_RUN_PREPSTART="3"
-MAXTRIES_RUN_FCST="3"
-MAXTRIES_ANAL_GSI="3"
-MAXTRIES_RUN_POST="3"
-MAXTRIES_RUN_PRDGEN="3"
-MAXTRIES_PROCESS_RADARREF="3"
-MAXTRIES_PROCESS_LIGHTNING="3"
-MAXTRIES_PROCESS_BUFR="3"
-MAXTRIES_CLDANL_NONVAR="3"
-
+FG_ROOTDIR="/home/rtrr/rrfs_b_nwges" #only needed by RTMA
 PYTHON_GRAPHICS_YML_FN="rtma.yml"
 
 USE_CUSTOM_POST_CONFIG_FILE="TRUE"
 TESTBED_FIELDS_FN="testbed_fields_rtma.txt"
+TESTBED_FIELDS_FN2="testbed_fields_bgrd3d.txt"
 CUSTOM_POST_CONFIG_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/postxconfig-NT-fv3lam_rrfs.txt"
 CUSTOM_POST_PARAMS_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/params_grib2_tbl_new"
 POST_FULL_MODEL_NAME="FV3R"
 POST_SUB_MODEL_NAME="RTMA"
-ARCHIVEDIR="/5year/BMC/wrfruc/rtma_dev1"
+ARCHIVEDIR="/5year/BMC/wrfruc/rtma_b"
 NCARG_ROOT="/apps/ncl/6.5.0-CentOS6.10_64bit_nodap_gnu447"
 NCL_HOME="/home/rtrr/RTMA/graphics"
 NCL_REGION="conus"
@@ -229,14 +252,15 @@ MODEL="RTMA_CONUS" #used by NCL
 
 RUN="RTMA_CONUS"
 COMINgfs=""
-OBSPATH="/lfs4/BMC/wrfruc/gge/rtma/wcoss_obs"
+#RTMA_OBS_FEED="NCO"
+#OBSPATH="/lfs4/BMC/wrfruc/gge/rtma/wcoss_obs"
 
 STMP="/lfs4/BMC/nrtrr/NCO_dirs/stmp"  # Path to directory STMP that mostly contains input files.
 PTMP="/lfs4/BMC/nrtrr/NCO_dirs/ptmp"  # Path to directory STMP that mostly contains input files.
 NWGES="/lfs4/BMC/nrtrr/NCO_dirs/nwges"  # Path to directory NWGES that save boundary, cold initial, restart files
 if [[ ${regional_ensemble_option} == "5" ]]; then
-   RRFSE_NWGES="/lfs4/BMC/wrfruc/RRFSE/NCO_dirs/nwges"  # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid.
-   NUM_ENS_MEMBERS=30     # FV3LAM ensemble size for GSI hybrid analysis
+  RRFSE_NWGES="/lfs4/BMC/wrfruc/RRFSE/NCO_dirs/nwges"  # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid.
+  NUM_ENS_MEMBERS=30     # FV3LAM ensemble size for GSI hybrid analysis
 fi
 
 # change settings if it is wcoss_dell_p3
@@ -250,8 +274,8 @@ if [[ "$MACHINE" == "wcoss_dell_p3" ]]; then
   PARTITION_FCST=""
   PARTITION_ANALYSIS=""
   PARTITION_GRAPHICS=""
-  PARTITION_PRDGEN=""
   DO_BUFRSND="FALSE"
+  RTMA_OBS_FEED="NCO"
 
   OBSPATH_NSSLMOSIAC="/gpfs/dell1/nco/ops/dcom/prod/ldmdata/obs/upperair/mrms/conus/MergedReflectivityQC"
   AIRCRAFT_REJECT="/gpfs/dell2/emc/modeling/noscrub/Edward.Colon/FixData/obsuselist/amdar_reject_lists"

ush/config_defaults.sh

@@ -1410,6 +1410,10 @@ SAVE_CYCLE_LOG="TRUE"
 #  but we needs to explicitly specify where to find FG for RTMA.
 #  So this parameter only matters for RTMA
 #
+# RTMA_OBS_FEED:
+#   "" or "GSL":  RTMA's observations follow the GSL naming convention
+#         "NCO":  RTMA's observations follow the NCO naming convention
+#
 # PYTHON_GRAPHICS_YML_FN
 #  The name of the yml file under ${PYTHON_GRAPHICS_DIR}/image_lists
 #  to be used by current application
@@ -1430,6 +1434,7 @@ SFC_CLIMO_DIR="/path/to/pregenerated/surface/climo/files"
 NCORES_PER_NODE=24 #Jet default value
 IS_RTMA="FALSE"
 FG_ROOTDIR=""
+RTMA_OBS_FEED=""
 PYTHON_GRAPHICS_YML_FN="rrfs_subset.yml"
 #
 #-----------------------------------------------------------------------

ush/templates/FV3LAM_wflow.xml

@@ -1741,10 +1741,12 @@ MODULES_RUN_TASK_FP script.
            <and>
              <timedep><cyclestr offset="&START_TIME_CONVENTIONAL;">@Y@m@d@H@M00</cyclestr></timedep>
              <taskdep task="&MAKE_ICS_TN;{{ uscore_ensmem_name }}"/>
+             {%- if not is_rtma %}
              <or>
                <metataskdep metatask="&MAKE_LBCS_TN;{{ uscore_ensmem_name }}"/>
                <metataskdep metatask="&MAKE_LBCS_TN;{{ uscore_ensmem_name }}" cycle_offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00"/>
              </or>
+             {%- endif %}
            </and>
 {%- endif %}
 {%- endif %}