Update scripts for running ensemble retros on Hera. (NOAA-EMC#445)

EricJames-NOAA · Nov 9, 2022 · ae8b1cb · ae8b1cb
1 parent d71623d
commit ae8b1cb
Show file tree

Hide file tree

Showing 7 changed files with 51 additions and 31 deletions.
diff --git a/scripts/exregional_run_analysis.sh b/scripts/exregional_run_analysis.sh
@@ -258,7 +258,8 @@ if  [[ ${regional_ensemble_option:-1} -eq 1 ]]; then #using GDAS
         avail_time=$(echo "${availtime}" | sed 's/\([[:digit:]]\{2\}\)$/ \1/')
         avail_time=$(date -d "${avail_time}")
 
-        stamp_avail=$(date -d "${avail_time} ${loop} hours" +%s)
+        loopfcst=$(echo ${loop}| cut -c 1-3)      # for nemsio 009s to get 009
+        stamp_avail=$(date -d "${avail_time} ${loopfcst} hours" +%s)
 
         hourDiff=$(echo "($stampcycle - $stamp_avail) / (60 * 60 )" | bc);
         if [[ ${stampcycle} -lt ${stamp_avail} ]]; then
@@ -295,7 +296,8 @@ if  [[ ${regional_ensemble_option:-1} -eq 1 ]]; then #using GDAS
         avail_time=$(echo "${availtime}" | sed 's/\([[:digit:]]\{2\}\)$/ \1/')
         avail_time=$(date -d "${avail_time}")
 
-        stamp_avail=$(date -d "${avail_time} ${loop} hours" +%s)
+        loopfcst=$(echo ${loop}| cut -c 1-3)      # for nemsio 009s to get 009
+        stamp_avail=$(date -d "${avail_time} ${loopfcst} hours" +%s)
 
         hourDiff=$(echo "($stampcycle - $stamp_avail) / (60 * 60 )" | bc);
         if [[ ${stampcycle} -lt ${stamp_avail} ]]; then

diff --git a/ush/config.sh.RRFS_CONUS_3km b/ush/config.sh.RRFS_CONUS_3km
@@ -67,7 +67,9 @@ DO_SURFACE_CYCLE="TRUE"
 DO_SPINUP="TRUE"
 DO_RETRO="TRUE"
 DO_NONVAR_CLDANAL="TRUE"
-DO_ENVAR_RADAR_REF="FALSE"
+#USE_RRFSE_ENS="TRUE"
+#CYCL_HRS_HYB_FV3LAM_ENS=("19" "20" "21" "22" "23" "00")
+#DO_ENVAR_RADAR_REF="TRUE"
 DO_SMOKE_DUST="FALSE"
 DO_REFL2TTEN="FALSE"
 RADARREFL_TIMELEVEL=(0)
@@ -140,7 +142,7 @@ HALO_BLEND=20
 IO_LAYOUT_Y=1
 PPN_RUN_REF2TTEN=1
 PPN_RUN_NONVARCLDANL=${IO_LAYOUT_Y}
-regional_ensemble_option=1
+regional_ensemble_option=1   # 5 for RRFS ensemble
 
 MAXTRIES_MAKE_ICS="3"
 MAXTRIES_MAKE_LBCS="3"
@@ -249,7 +251,7 @@ PTMP="YourOwnSpace/NCO_dirs/ptmp"  # Path to directory STMP that mostly contains
 NWGES="YourOwnSpace/NCO_dirs/nwges"  # Path to directory NWGES that save boundary, cold initial, restart files
 if [[ ${regional_ensemble_option} == "5" ]]; then
   RRFSE_NWGES="YourOwnSpace/NCO_dirs/nwges"  # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid.
-  NUM_ENS_MEMBERS=10     # FV3LAM ensemble size for GSI hybrid analysis
+  NUM_ENS_MEMBERS=30     # FV3LAM ensemble size for GSI hybrid analysis
 fi
 
 

diff --git a/ush/config.sh.RRFS_CONUS_3km_ens b/ush/config.sh.RRFS_CONUS_3km_ens
@@ -76,7 +76,9 @@ CCPP_PHYS_SUITE="FV3_HRRR"
 EXTRN_MDL_ICS_OFFSET_HRS="30"
 LBC_SPEC_INTVL_HRS="1"
 EXTRN_MDL_LBCS_OFFSET_HRS="6"
-BOUNDARY_LEN_HRS="36"
+BOUNDARY_LEN_HRS="6"
+BOUNDARY_LONG_LEN_HRS="36"
+BOUNDARY_PROC_GROUP_NUM="3"
 
 # avaialble retro period:
 # 20210511-20210531; 20210718-20210801

diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh
@@ -552,8 +552,8 @@ WFLOW_LAUNCH_LOG_FN="log.launch_FV3LAM_wflow"
 DATE_FIRST_CYCL="YYYYMMDD"
 DATE_LAST_CYCL="YYYYMMDD"
 STARTYEAR="2022"
-STARTMONTH="21"
-STARTDAY="10"
+STARTMONTH="10"
+STARTDAY="21"
 STARTHOUR="00"
 ENDYEAR="2022"
 ENDMONTH="10"

diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh
@@ -125,6 +125,9 @@ else
     "JET")
       EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS:-/mnt/lfs4/BMC/public/data/grids/gens/pgrb2b}"
       ;;
+    "HERA")
+      EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS:-/mnt/lfs4/BMC/public/data/grids/gens/pgrb2b}"
+      ;;
     esac
     ;;
 
@@ -318,6 +321,9 @@ else
     "JET")
       EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-/mnt/lfs4/BMC/public/data/grids/gens/pgrb2b}"
       ;;
+    "HERA")
+      EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-/mnt/lfs4/BMC/public/data/grids/gens/pgrb2b}"
+      ;;
     esac
   ;;
 

diff --git a/ush/set_rrfs_config.sh b/ush/set_rrfs_config.sh
@@ -74,20 +74,22 @@ if [[ $DO_RETRO == "TRUE" ]] ; then
 
     if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then
       if [[ ${EXTRN_MDL_NAME_ICS} == "GEFS" ]]; then
-        EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch1/BMC/wrfruc/chunhua/data/GEFS/pgrb2"
+#        EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch1/BMC/wrfruc/chunhua/data/GEFS/pgrb2"
+        EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch2/BMC/zrtrr/rli/data/GEFS/dsg"
       elif [[ ${EXTRN_MDL_NAME_ICS} == "HRRRDAS" ]]; then
         EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch1/BMC/wrfruc/chunhua/data/HRRRDAS"
       elif [[ ${EXTRN_MDL_NAME_ICS} == "GDASENKF" ]]; then
         EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch1/BMC/wrfruc/chunhua/data/GDASENKF"
       fi
       if [[ ${EXTRN_MDL_NAME_LBCS} == "GEFS" ]]; then
-        EXTRN_MDL_SOURCE_BASEDIR_LBCS="/scratch1/BMC/wrfruc/chunhua/data/GEFS/pgrb2"
+#        EXTRN_MDL_SOURCE_BASEDIR_LBCS="/scratch1/BMC/wrfruc/chunhua/data/GEFS/pgrb2"
+        EXTRN_MDL_SOURCE_BASEDIR_LBCS="/scratch2/BMC/zrtrr/rli/data/GEFS/dsg"
       elif [[ ${EXTRN_MDL_NAME_LBCS} == "GDASENKF" ]]; then
         EXTRN_MDL_SOURCE_BASEDIR_LBCS="/scratch1/BMC/wrfruc/chunhua/data/GDASENKF"
       elif [[ ${EXTRN_MDL_NAME_LBCS} == "FV3GFS" ]]; then
         EXTRN_MDL_SOURCE_BASEDIR_LBCS="/scratch1/BMC/wrfruc/chunhua/data/FV3GFS"
       fi
-      OBSPATH="/scratch1/BMC/wrfruc/chunhua/data/obs"
+      OBSPATH="/scratch2/BMC/zrtrr/rli/data/obs_rap"
     else
 #    EXTRN_MDL_SOURCE_BASEDIR_ICS=/scratch2/BMC/zrtrr/rli/data/hrrr/conus/wrfnat/grib2
 #    EXTRN_MDL_SOURCE_BASEDIR_LBCS=/scratch2/BMC/zrtrr/rli/data/rap/full/wrfnat/grib2

diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml
@@ -219,7 +219,7 @@ tasks; and the "FCST" type is used for the RUN_FCST_TN task.
 ]>
 
 {%- if do_retro %}
-<workflow realtime="F" scheduler="&SCHED;" cyclethrottle="5" cyclelifespan="01:00:00:00">
+<workflow realtime="F" scheduler="&SCHED;" cyclethrottle="5">
 {% else %}
 <workflow realtime="T" scheduler="&SCHED;" cyclethrottle="24" cyclelifespan="01:00:00:00">
 {%- endif %}
@@ -499,23 +499,20 @@ MODULES_RUN_TASK_FP script.
 
   </task>
 
-{%- if do_ensemble %}
-   </metatask>
-{%- endif %}
 <!--
 ************************************************************************
 ************************************************************************
 -->
-  <task name="&GET_EXTRN_LBCS_LONG_TN;" cycledefs="boundary_long" maxtries="{{ maxtries_get_extrn_lbcs }}">
+  <task name="&GET_EXTRN_LBCS_LONG_TN;{{ uscore_ensmem_name }}" cycledefs="boundary_long" maxtries="{{ maxtries_get_extrn_lbcs }}">
 
     &WALL_LIMIT_PRE;
     &RSRV_DEFAULT;
     <command>&LOAD_MODULES_RUN_TASK_FP; "&GET_EXTRN_LBCS_TN;" "&JOBSDIR;/JREGIONAL_GET_EXTRN_MDL_FILES"</command>
     <nodes>{{ nnodes_get_extrn_lbcs }}:ppn={{ ppn_get_extrn_lbcs }}</nodes>
     <walltime>{{ wtime_get_extrn_lbcs }}</walltime>
     <nodesize>&NCORES_PER_NODE;</nodesize>
-    <jobname>&TAG;_&GET_EXTRN_LBCS_LONG_TN;</jobname>
-    <join><cyclestr>&LOGDIR;/&GET_EXTRN_LBCS_LONG_TN;_@Y@m@d@H.log</cyclestr></join>
+    <jobname>&TAG;_&GET_EXTRN_LBCS_LONG_TN;{{ uscore_ensmem_name }}</jobname>
+    <join><cyclestr>&LOGDIR;/&GET_EXTRN_LBCS_LONG_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log</cyclestr></join>
 
     <envar><name>GLOBAL_VAR_DEFNS_FP</name><value>&GLOBAL_VAR_DEFNS_FP;</value></envar>
     <envar><name>PDY</name><value><cyclestr>@Y@m@d</cyclestr></value></envar>
@@ -524,6 +521,10 @@ MODULES_RUN_TASK_FP script.
     <envar><name>EXTRN_MDL_NAME</name><value>{{ extrn_mdl_name_lbcs }}</value></envar>
     <envar><name>ICS_OR_LBCS</name><value>LBCS</value></envar>
     <envar><name>BOUNDARY_LEN</name><value>{{ boundary_long_len_hrs }}</value></envar>
+    <envar><name>SLASH_ENSMEM_SUBDIR</name><value>{{ slash_ensmem_subdir }}</value></envar>
+    <envar><name>GEFS_INPUT_SUBDIR</name><value>#subdirGE#</value></envar>
+    <envar><name>GDASENKF_INPUT_SUBDIR</name><value>#subdirGDAS#</value></envar>
+    <envar><name>GDAS_MEM_NAME</name><value>#memNameGDAS#</value></envar>
 
     <dependency>
        <and>
@@ -533,26 +534,22 @@ MODULES_RUN_TASK_FP script.
          {%- elif machine in ["ORION"]  %}
          <datadep age="00:00:05:00"><cyclestr>{{ extrn_mdl_sysbasedir_ics }}/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf{{ "%03d" % h }}.nc</cyclestr></datadep>
          {%- else %}
+         {%- if extrn_mdl_name_lbcs in ["GEFS"] %}
+         <datadep age="00:00:05:00"> <cyclestr offset="-{{ extrn_mdl_lbcs_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_lbcs }}/#subdirGE#/@y@j@H000{{ "%03d" % h }}</cyclestr></datadep>
+         {%- else %}
          <datadep age="00:00:05:00"> <cyclestr offset="-{{ extrn_mdl_lbcs_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }}</cyclestr></datadep>
          {%- endif %}
+         {%- endif %}
          {%- endfor %}
        </and>
     </dependency>
 
   </task>
+
 <!--
 ************************************************************************
 ************************************************************************
 -->
- {%- if do_ensemble %}
- <metatask name="run_ensemble">
-   <var name="{{ ensmem_indx_name }}">
-    {%- for m in range(1, num_ens_members+1) -%}
-    {%- set fmtstr=" %0"~ndigits_ensmem_names~"d" -%}
-    {{- fmtstr%m -}}
-   {%- endfor %} </var>
- {%- endif %}
-
   <task name="&MAKE_ICS_TN;{{ uscore_ensmem_name }}" cycledefs="initial" maxtries="{{ maxtries_make_ics }}">
 
     &RSRV_DEFAULT;
@@ -626,7 +623,7 @@ MODULES_RUN_TASK_FP script.
       <and>
         <or>
           <taskdep task="&GET_EXTRN_LBCS_TN;{{ uscore_ensmem_name }}"/>
-          <taskdep task="&GET_EXTRN_LBCS_LONG_TN;"/>
+          <taskdep task="&GET_EXTRN_LBCS_LONG_TN;{{ uscore_ensmem_name }}"/>
         </or>
         <or>
 <!--          <taskdep task="&MAKE_GRID_TN;"/> -->
@@ -697,7 +694,7 @@ MODULES_RUN_TASK_FP script.
              <timedep><cyclestr offset="&START_TIME_SPINUP;">@Y@m@d@H@M00</cyclestr></timedep>
              <taskdep task="&MAKE_ICS_TN;"/>
              {%- for h in range(0,  boundary_proc_group_num) %}
-             <taskdep task="&MAKE_LBCS_TN;_{{ "%02d" % h }}" cycle_offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00"/>
+             <taskdep task="&MAKE_LBCS_TN;_{{ "%02d" % h }}{{ uscore_ensmem_name }}" cycle_offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00"/>
              {%- endfor %}
            </and>
 {%- endif %}
@@ -797,8 +794,16 @@ MODULES_RUN_TASK_FP script.
              <timedep><cyclestr offset="&START_TIME_CONVENTIONAL;">@Y@m@d@H@M00</cyclestr></timedep>
              <taskdep task="&MAKE_ICS_TN;{{ uscore_ensmem_name }}"/>
              <or>
-               <taskdep task="&MAKE_LBCS_TN;{{ uscore_ensmem_name }}"/>
-               <taskdep task="&MAKE_LBCS_TN;{{ uscore_ensmem_name }}" cycle_offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00"/>
+                <and>
+                  {%- for h in range(0,  boundary_proc_group_num) %}
+                   <taskdep task="&MAKE_LBCS_TN;_{{ "%02d" % h }}{{ uscore_ensmem_name }}"/>
+                  {%- endfor %}
+                </and>
+                <and>
+                  {%- for h in range(0,  boundary_proc_group_num) %}
+                   <taskdep task="&MAKE_LBCS_TN;_{{ "%02d" % h }}{{ uscore_ensmem_name }}" cycle_offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00"/>
+                  {%- endfor %}
+                </and>
              </or>
            </and>
 {%- endif %}
@@ -2084,6 +2089,7 @@ MODULES_RUN_TASK_FP script.
     <dependency>
        <and>
          <taskdep task="&PREP_CYC_ENSMEAN_TN;"/>
+         <taskdep task="&PROCESS_RADAR_REF_TN;_prod"/>
          {% if machine in ["JET", "HERA"] -%}
          <datadep age="05:00"><cyclestr>&OBSPATH;/@Y@m@d@H.rap.t@Hz.prepbufr.tm00</cyclestr></datadep>
          {% else -%}