From aca57495e2a6d7ae987cd0671df3afecb16df045 Mon Sep 17 00:00:00 2001 From: Ming Hu Date: Tue, 7 Feb 2023 17:48:46 -0700 Subject: [PATCH] code change for running RRFS_A on wcoss2 (#481) * Tune v0.3.6 on wocss2 for speeding up ensemble DA to catch the control cycles. Add grid and reource setup for NA 3km on wcoss2. * Add configuration file for NA 3km control run and enkf DA run on WCOSS2 Add fix files for NA 3km smoke/dust Add section to setup NA 3km resources fro WCOSS2. --------- Co-authored-by: Shun Liu Co-authored-by: ming hu --- .../RRFS_NA_3km/CONUS_G2_3km_weight_file.nc | 1 - .../RRFS_NA_3km/NA.nearest_weigt.nc | 1 + .../RRFS_NA_3km/RRFS_CONUS_3km_weight_file.nc | 1 - .../RRFS_NA_3km/dummy_24hr_smoke.nc | 1 - .../RRFS_NA_3km/dust12m_data.nc_old | 1 + fix/smoke_dust/RRFS_NA_3km/grid_in.nc | 1 - scripts/exregional_nonvarcldanl.sh | 1 + ush/config.sh.rrfs_a_c3 | 27 +-- ush/config.sh_rrfs_a_enkf_c3 | 49 +++-- ush/config.sh_rrfs_a_enkf_n3 | 179 ++++++++++++++++++ ush/config.sh_rrfs_a_n3 | 120 ++++++++++++ ush/config_defaults.sh | 3 + ush/generate_FV3LAM_wflow.sh | 3 + ush/set_rrfs_config_general.sh | 73 ++++++- ush/setup.sh | 4 +- ush/templates/FV3LAM_wflow.xml | 55 +++--- 16 files changed, 455 insertions(+), 65 deletions(-) delete mode 120000 fix/smoke_dust/RRFS_NA_3km/CONUS_G2_3km_weight_file.nc create mode 120000 fix/smoke_dust/RRFS_NA_3km/NA.nearest_weigt.nc delete mode 120000 fix/smoke_dust/RRFS_NA_3km/RRFS_CONUS_3km_weight_file.nc delete mode 120000 fix/smoke_dust/RRFS_NA_3km/dummy_24hr_smoke.nc create mode 120000 fix/smoke_dust/RRFS_NA_3km/dust12m_data.nc_old delete mode 120000 fix/smoke_dust/RRFS_NA_3km/grid_in.nc create mode 100644 ush/config.sh_rrfs_a_enkf_n3 create mode 100644 ush/config.sh_rrfs_a_n3 diff --git a/fix/smoke_dust/RRFS_NA_3km/CONUS_G2_3km_weight_file.nc b/fix/smoke_dust/RRFS_NA_3km/CONUS_G2_3km_weight_file.nc deleted file mode 120000 index 6ec415621..000000000 --- a/fix/smoke_dust/RRFS_NA_3km/CONUS_G2_3km_weight_file.nc +++ /dev/null @@ -1 +0,0 @@ -../../.agent/smoke_dust/RRFS_NA_3km/CONUS_G2_3km_weight_file.nc \ No newline at end of file diff --git a/fix/smoke_dust/RRFS_NA_3km/NA.nearest_weigt.nc b/fix/smoke_dust/RRFS_NA_3km/NA.nearest_weigt.nc new file mode 120000 index 000000000..dfa43bc59 --- /dev/null +++ b/fix/smoke_dust/RRFS_NA_3km/NA.nearest_weigt.nc @@ -0,0 +1 @@ +../../.agent/smoke_dust/RRFS_NA_3km/NA.nearest_weigt.nc \ No newline at end of file diff --git a/fix/smoke_dust/RRFS_NA_3km/RRFS_CONUS_3km_weight_file.nc b/fix/smoke_dust/RRFS_NA_3km/RRFS_CONUS_3km_weight_file.nc deleted file mode 120000 index bcc803a94..000000000 --- a/fix/smoke_dust/RRFS_NA_3km/RRFS_CONUS_3km_weight_file.nc +++ /dev/null @@ -1 +0,0 @@ -../../.agent/smoke_dust/RRFS_NA_3km/RRFS_CONUS_3km_weight_file.nc \ No newline at end of file diff --git a/fix/smoke_dust/RRFS_NA_3km/dummy_24hr_smoke.nc b/fix/smoke_dust/RRFS_NA_3km/dummy_24hr_smoke.nc deleted file mode 120000 index 0823c44e2..000000000 --- a/fix/smoke_dust/RRFS_NA_3km/dummy_24hr_smoke.nc +++ /dev/null @@ -1 +0,0 @@ -../../.agent/smoke_dust/RRFS_NA_3km/dummy_24hr_smoke.nc \ No newline at end of file diff --git a/fix/smoke_dust/RRFS_NA_3km/dust12m_data.nc_old b/fix/smoke_dust/RRFS_NA_3km/dust12m_data.nc_old new file mode 120000 index 000000000..e53f50ea6 --- /dev/null +++ b/fix/smoke_dust/RRFS_NA_3km/dust12m_data.nc_old @@ -0,0 +1 @@ +../../.agent/smoke_dust/RRFS_NA_3km/dust12m_data.nc_old \ No newline at end of file diff --git a/fix/smoke_dust/RRFS_NA_3km/grid_in.nc b/fix/smoke_dust/RRFS_NA_3km/grid_in.nc deleted file mode 120000 index 00664c446..000000000 --- a/fix/smoke_dust/RRFS_NA_3km/grid_in.nc +++ /dev/null @@ -1 +0,0 @@ -../../.agent/smoke_dust/RRFS_NA_3km/grid_in.nc \ No newline at end of file diff --git a/scripts/exregional_nonvarcldanl.sh b/scripts/exregional_nonvarcldanl.sh index 9c1b348cf..a225f5255 100755 --- a/scripts/exregional_nonvarcldanl.sh +++ b/scripts/exregional_nonvarcldanl.sh @@ -82,6 +82,7 @@ case $MACHINE in APRUN="mpiexec -n ${IO_LAYOUT_Y} -ppn 16" ncores=$(( NNODES_RUN_NONVARCLDANL*PPN_RUN_NONVARCLDANL )) APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_NONVARCLDANL}" + APRUN="mpiexec -n 1 -ppn 1" ;; # "HERA") diff --git a/ush/config.sh.rrfs_a_c3 b/ush/config.sh.rrfs_a_c3 index d343f7877..e30928762 100644 --- a/ush/config.sh.rrfs_a_c3 +++ b/ush/config.sh.rrfs_a_c3 @@ -9,9 +9,12 @@ PREDEF_GRID_NAME=RRFS_CONUS_3km . set_rrfs_config_general.sh +#DO_ENSEMBLE="TRUE" +#DO_ENSFCST="TRUE" DO_DACYCLE="TRUE" DO_SURFACE_CYCLE="TRUE" DO_SPINUP="TRUE" +#DO_SAVE_INPUT="TRUE" DO_POST_SPINUP="FALSE" DO_POST_PROD="TRUE" DO_RETRO="FALSE" @@ -23,7 +26,7 @@ RADARREFL_TIMELEVEL=(0) FH_DFI_RADAR="0.0,0.25,0.5" DO_SOIL_ADJUST="TRUE" DO_RADDA="FALSE" -DO_BUFRSND="FALSE" +DO_BUFRSND="TRUE" USE_FVCOM="TRUE" PREP_FVCOM="TRUE" @@ -35,13 +38,13 @@ BOUNDARY_PROC_GROUP_NUM="72" # avaialble retro period: # 20210511-20210531; 20210718-20210801 -DATE_FIRST_CYCL="20230120" -DATE_LAST_CYCL="20230131" +DATE_FIRST_CYCL="20230201" +DATE_LAST_CYCL="20230331" CYCL_HRS=( "00" "12" ) CYCL_HRS_SPINSTART=("03" "15") CYCL_HRS_PRODSTART=("09" "21") -CYCLEMONTH="01" -CYCLEDAY="20-31" +CYCLEMONTH="02,03" +CYCLEDAY="*" STARTYEAR=${DATE_FIRST_CYCL:0:4} STARTMONTH=${DATE_FIRST_CYCL:4:2} @@ -70,8 +73,8 @@ POSTPROC_LONG_LEN_HRS="60" for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=18; done for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=60; done DA_CYCLE_INTERV="1" -RESTART_INTERVAL="1 2 3" -RESTART_INTERVAL_LONG="1 2 3" +RESTART_INTERVAL="1 2" +RESTART_INTERVAL_LONG="1 2" USE_RRFSE_ENS="TRUE" CYCL_HRS_HYB_FV3LAM_ENS=("00" "01" "02" "03" "04" "05" "06" "07" "08" "09" "10" "11" "12" "13" "14" "15" "16" "17" "18" "19" "20" "21" "22" "23") @@ -94,7 +97,7 @@ TAG="c3v36" ARCHIVEDIR="/NCEPDEV/emc-meso/1year/emc.lam" NCL_REGION="conus" -MODEL="RRFS_A" +MODEL="rrfs_a" # # In NCO mode, the following don't need to be explicitly set to "FALSE" @@ -110,12 +113,12 @@ COMINgfs="" . set_rrfs_config.sh -STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}/stmp" # Path to directory STMP that mostly contains input files. +STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. PTMP="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. -NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges/${envir}/${NET}" # Path to directory NWGES that save boundary, cold initial, restart files +NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files if [[ ${regional_ensemble_option} == "5" ]]; then - RRFSE_NWGES=="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid. + RRFSE_NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid. NUM_ENS_MEMBERS=30 # FV3LAM ensemble size for GSI hybrid analysis - CYCL_HRS_PRODSTART_ENS=( "19" ) + CYCL_HRS_PRODSTART_ENS=( "07" "19" ) DO_ENVAR_RADAR_REF="TRUE" fi diff --git a/ush/config.sh_rrfs_a_enkf_c3 b/ush/config.sh_rrfs_a_enkf_c3 index 730073e1a..5eab5fc78 100644 --- a/ush/config.sh_rrfs_a_enkf_c3 +++ b/ush/config.sh_rrfs_a_enkf_c3 @@ -20,12 +20,17 @@ DO_POST_SPINUP="FALSE" DO_POST_PROD="FALSE" #DO_RETRO="TRUE" DO_NONVAR_CLDANAL="TRUE" +#DO_ENVAR_RADAR_REF="TRUE" DO_SMOKE_DUST="FALSE" #DO_REFL2TTEN="FALSE" #RADARREFL_TIMELEVEL=(0) #FH_DFI_RADAR="0.0,0.25,0.5" #DO_SOIL_ADJUST="TRUE" #DO_RADDA="FALSE" +#DO_BUFRSND="TRUE" +#USE_FVCOM="TRUE" +#PREP_FVCOM="TRUE" + if [[ ${DO_ENSFCST} == "TRUE" ]] ; then DO_SPINUP="FALSE" DO_SAVE_DA_OUTPUT="FALSE" @@ -41,16 +46,16 @@ BOUNDARY_PROC_GROUP_NUM="4" # avaialble retro period: # 20210511-20210531; 20210718-20210801 -DATE_FIRST_CYCL="20230121" -DATE_LAST_CYCL="20230131" +DATE_FIRST_CYCL="20230201" +DATE_LAST_CYCL="20230331" CYCL_HRS=( "00" "12" ) CYCL_HRS=( "18" ) -CYCL_HRS_SPINSTART=("18") -CYCL_HRS_PRODSTART=("19") +CYCL_HRS_SPINSTART=("06" "18") +CYCL_HRS_PRODSTART=("07" "19") #CYCL_HRS_STOCH=("00" "12") #CYCL_HRS_RECENTER=("19") -CYCLEMONTH="01" -CYCLEDAY="21-31" +CYCLEMONTH="02,03" +CYCLEDAY="*" STARTYEAR=${DATE_FIRST_CYCL:0:4} STARTMONTH=${DATE_FIRST_CYCL:4:2} @@ -62,9 +67,9 @@ ENDDAY=${DATE_LAST_CYCL:6:2} ENDHOUR="23" PREEXISTING_DIR_METHOD="upgrade" # "rename" -INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 24:00:00" +INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 12:00:00" BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 06:00:00" -PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 19-30 ${CYCLEMONTH} ${STARTYEAR} *" +PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *" PRODLONG_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00" #RECENTER_CYCLEDEF="00 19 * 10 2022 *" ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1500 ${DATE_LAST_CYCL}2300 24:00:00" @@ -77,7 +82,7 @@ if [[ ${DO_ENSFCST} == "TRUE" ]] ; then PRODLONG_CYCLEDEF="00 01 01 01 2100 *" fi if [[ $DO_SPINUP == "TRUE" ]] ; then - SPINUP_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 24:00:00" + SPINUP_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 12:00:00" fi if [[ $DO_SAVE_DA_OUTPUT == "TRUE" ]] ; then SAVEDA_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 12:00:00" @@ -88,15 +93,15 @@ FCST_LEN_HRS_SPINUP="1" POSTPROC_LEN_HRS="1" #FCST_LEN_HRS_CYCLES=(48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18) for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=1; done -for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=2; done +for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=1; done if [[ ${DO_ENSFCST} == "TRUE" ]] ; then - for i in {0..23..12}; do FCST_LEN_HRS_CYCLES[$i]=24; done - POSTPROC_LEN_HRS="24" + for i in {0..23..12}; do FCST_LEN_HRS_CYCLES[$i]=36; done + POSTPROC_LEN_HRS="36" BOUNDARY_PROC_GROUP_NUM="8" fi DA_CYCLE_INTERV="1" RESTART_INTERVAL="1" -RESTART_INTERVAL_LONG="1 2" +RESTART_INTERVAL_LONG="1" netcdf_diag=.true. binary_diag=.false. NFHOUT="3" @@ -118,11 +123,11 @@ FV3GFS_FILE_FMT_ICS="grib2" FV3GFS_FILE_FMT_LBCS="grib2" envir="para" +NET="rrfs_a" +TAG="cenkf36" +ARCHIVEDIR="/NCEPDEV/emc-meso/1year/emc.lam" NCL_REGION="conus" MODEL="rrfs_b" -NET="rrfs_a" -TAG="cenkf32" -ARCHIVEDIR="/1year/BMC/wrfruc/rrfs_dev1" if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then NUM_ENS_MEMBERS=30 @@ -142,16 +147,18 @@ if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then # SPPT_MAG=0.5 WTIME_RUN_FCST="04:45:00" WTIME_MAKE_LBCS="01:30:00" + TAG="cefcst36" fi RADAR_REF_THINNING="2" ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens" NNODES_RUN_RECENTER="3" - PPN_RUN_RECENTER="$(( ${NUM_ENS_MEMBERS} + 1 ))" + PPN_RUN_RECENTER="128" CLEAN_OLDFCST_HRS="48" CLEAN_OLDSTMPPOST_HRS="48" cld_bld_hgt=0.0 l_precip_clear_only=.true. write_diag_2=.true. + START_TIME_SPINUP="00:30:00" fi RUN="rrfs_a" @@ -162,13 +169,13 @@ COMINgfs="" . set_rrfs_config.sh -STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}/stmp" # Path to directory STMP that mostly contains input files. +STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. if [[ ${DO_ENSFCST} == "TRUE" ]] ; then STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}/enfcst" # Path to directory STMP that mostly contains input files. fi PTMP="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. -NWGES="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files -ENSCTRL_STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}/stmp" # Path to directory STMP that mostly contains control input files for ensemble recentering. +NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files +ENSCTRL_STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains control input files for ensemble recentering. ENSCTRL_PTMP="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains control input files for ensemble recentering. -ENSCTRL_NWGES="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}/nwges" # Path to directory STMP that mostly contains control input files for ensemble recentering. +ENSCTRL_NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory STMP that mostly contains control input files for ensemble recentering. diff --git a/ush/config.sh_rrfs_a_enkf_n3 b/ush/config.sh_rrfs_a_enkf_n3 new file mode 100644 index 000000000..ea6372555 --- /dev/null +++ b/ush/config.sh_rrfs_a_enkf_n3 @@ -0,0 +1,179 @@ +MACHINE="wcoss2" +version="v0.3.7" +ACCOUNT="RRFS_DEV" +#RESERVATION="rrfsdet" +EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/${version}" +EXPT_SUBDIR="rrfs_na_ens" + +PREDEF_GRID_NAME="RRFS_NA_3km" +. set_rrfs_config_general.sh + +DO_ENSEMBLE="TRUE" +#DO_ENSFCST="TRUE" +#DO_DACYCLE="TRUE" +#DO_SURFACE_CYCLE="TRUE" +DO_SPINUP="TRUE" +DO_SAVE_INPUT="FALSE" +DO_SAVE_DA_OUTPUT="TRUE" +DO_POST_SPINUP="FALSE" +DO_POST_PROD="FALSE" +#DO_RETRO="FALSE" +DO_NONVAR_CLDANAL="TRUE" +#DO_ENVAR_RADAR_REF="FALSE" +DO_SMOKE_DUST="FALSE" +#DO_REFL2TTEN="FALSE" +#RADARREFL_TIMELEVEL=(0) +#FH_DFI_RADAR="0.0,0.25,0.5" +#DO_SOIL_ADJUST="TRUE" +#DO_RADDA="FALSE" +#DO_BUFRSND="TRUE" +#USE_FVCOM="TRUE" +#PREP_FVCOM="TRUE" + +if [[ ${DO_ENSFCST} == "TRUE" ]] ; then + DO_SPINUP="FALSE" + DO_SAVE_DA_OUPUT="FALSE" + DO_NONVAR_CLDANAL="FALSE" + DO_POST_PROD="TRUE" +fi + +NET="rrfs_a" +TAG="nenkf37" +ARCHIVEDIR="/NCEPDEV/emc-meso/1year/emc.lam" +NCL_REGION="conus" +MODEL="RRFS_A" +RUN="rrfs_a" +envir="para" + +EXTRN_MDL_ICS_OFFSET_HRS="30" +LBC_SPEC_INTVL_HRS="1" +EXTRN_MDL_LBCS_OFFSET_HRS="6" +BOUNDARY_LEN_HRS="12" +BOUNDARY_PROC_GROUP_NUM="12" + +# avaialble retro period: +# 20210511-20210531; 20210718-20210801 +DATE_FIRST_CYCL="20230202" +DATE_LAST_CYCL="20230331" +CYCL_HRS=( "00" "12" ) +CYCL_HRS_SPINSTART=("06" "18") +CYCL_HRS_PRODSTART=("07" "19") +#CYCL_HRS_STOCH=("00" "12") +#CYCL_HRS_RECENTER=("19") +CYCLEMONTH="02,03" +CYCLEDAY="*" + +STARTYEAR=${DATE_FIRST_CYCL:0:4} +STARTMONTH=${DATE_FIRST_CYCL:4:2} +STARTDAY=${DATE_FIRST_CYCL:6:2} +STARTHOUR="00" +ENDYEAR=${DATE_LAST_CYCL:0:4} +ENDMONTH=${DATE_LAST_CYCL:4:2} +ENDDAY=${DATE_LAST_CYCL:6:2} +ENDHOUR="23" + +PREEXISTING_DIR_METHOD="upgrade" +INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 12:00:00" +BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 06:00:00" +PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *" +PRODLONG_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00" +#RECENTER_CYCLEDEF="00 19 * 10 2022 *" +ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1400 ${DATE_LAST_CYCL}2300 24:00:00" +if [[ ${DO_ENSFCST} == "TRUE" ]] ; then + BOUNDARY_LEN_HRS="36" + DO_SPINUP="FALSE" + INITIAL_CYCLEDEF="00 01 01 01 2100 *" + BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 12:00:00" + PROD_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 12:00:00" + PRODLONG_CYCLEDEF="00 01 01 01 2100 *" +fi +if [[ $DO_SPINUP == "TRUE" ]] ; then + SPINUP_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 12:00:00" +fi +if [[ $DO_SAVE_DA_OUTPUT == "TRUE" ]] ; then + SAVEDA_CYCLEDEF="${DATE_FIRST_CYCL}1200 ${DATE_LAST_CYCL}2300 12:00:00" +fi + +FCST_LEN_HRS="1" +FCST_LEN_HRS_SPINUP="1" +POSTPROC_LEN_HRS="1" +POSTPROC_LONG_LEN_HRS="1" +#FCST_LEN_HRS_CYCLES=(21 21 21 63 21 21 21 21 21 63 21 21 21 21 21 63 21 21 21 21 21 63 21 21) +for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=1; done +for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=1; done +if [[ ${DO_ENSFCST} == "TRUE" ]] ; then + for i in {0..23..12}; do FCST_LEN_HRS_CYCLES[$i]=36; done + POSTPROC_LEN_HRS="36" + BOUNDARY_PROC_GROUP_NUM="8" +fi +DA_CYCLE_INTERV="1" +RESTART_INTERVAL="1" +RESTART_INTERVAL_LONG="1" +netcdf_diag=.true. +binary_diag=.false. +NFHOUT="3" +NFHMAX_HF="12" + +WTIME_RUN_FCST="00:30:00" +WTIME_RUN_FCST_LONG="00:45:00" +NNODES_RUN_ANAL="1" + +EXTRN_MDL_NAME_ICS="GEFS" +EXTRN_MDL_NAME_LBCS="GEFS" +if [[ $MACHINE == "wcoss2" ]] ; then + EXTRN_MDL_NAME_LBCS="FV3GFS" +fi +EXTRN_MDL_SOURCE_BASEDIR_ICS="/mnt/lfs4/BMC/wrfruc/RRFSE/data/gens" +EXTRN_MDL_SOURCE_BASEDIR_LBCS="/mnt/lfs4/BMC/wrfruc/RRFSE/data/gens" + +FV3GFS_FILE_FMT_ICS="grib2" +FV3GFS_FILE_FMT_LBCS="grib2" + +if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then + NUM_ENS_MEMBERS=30 +# DO_ENSCONTROL="TRUE" + DO_GSIOBSERVER="TRUE" + DO_ENKFUPDATE="TRUE" +# DO_RECENTER="TRUE" + DO_ENS_GRAPHICS="TRUE" + DO_ENKF_RADAR_REF="TRUE" + DO_ENSPOST="FALSE" + DO_ENSINIT="TRUE" + NUM_ENS_MEMBERS_FCST=9 + LAYOUT_X="20" + if [[ ${DO_ENSFCST} == "TRUE" ]] ; then + NUM_ENS_MEMBERS=${NUM_ENS_MEMBERS_FCST} +# DO_SPPT=TRUE + DO_SPP=TRUE +# SPPT_MAG=0.5 + WTIME_RUN_FCST="04:45:00" + WTIME_MAKE_LBCS="01:30:00" + TAG="nefcst37" + LAYOUT_X="40" + fi + RADAR_REF_THINNING="2" + ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens" + NNODES_RUN_RECENTER="3" + PPN_RUN_RECENTER="128" + CLEAN_OLDFCST_HRS="24" + CLEAN_OLDSTMPPOST_HRS="24" + CLEAN_OLDRUN_HRS="12" + cld_bld_hgt=0.0 + l_precip_clear_only=.true. + write_diag_2=.true. + START_TIME_SPINUP="00:30:00" +fi + +COMINgfs="" + +. set_rrfs_config.sh + +STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. +if [[ ${DO_ENSFCST} == "TRUE" ]] ; then + STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}/enfcst" # Path to directory STMP that mostly contains input files. +fi +PTMP="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. +NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files +ENSCTRL_STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains control input files for ensemble recentering. +ENSCTRL_PTMP="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains control input files for ensemble recentering. +ENSCTRL_NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory STMP that mostly contains control input files for ensemble recentering. diff --git a/ush/config.sh_rrfs_a_n3 b/ush/config.sh_rrfs_a_n3 new file mode 100644 index 000000000..838632b1e --- /dev/null +++ b/ush/config.sh_rrfs_a_n3 @@ -0,0 +1,120 @@ +MACHINE="wcoss2" +version="v0.3.7" +ACCOUNT="RRFS_DEV" +#RESERVATION="rrfsdet" +EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/${version}" +EXPT_SUBDIR="rrfs_na" + +PREDEF_GRID_NAME="RRFS_NA_3km" +. set_rrfs_config_general.sh + +DO_DACYCLE="TRUE" +DO_SURFACE_CYCLE="TRUE" +DO_SPINUP="TRUE" +DO_POST_SPINUP="FALSE" +DO_POST_PROD="TRUE" +DO_RETRO="FALSE" +DO_NONVAR_CLDANAL="TRUE" +DO_ENVAR_RADAR_REF="FALSE" +DO_SMOKE_DUST="TRUE" +DO_REFL2TTEN="FALSE" +RADARREFL_TIMELEVEL=(0) +FH_DFI_RADAR="0.0,0.25,0.5" +DO_SOIL_ADJUST="TRUE" +DO_RADDA="FALSE" +DO_BUFRSND="TRUE" +USE_FVCOM="TRUE" +PREP_FVCOM="TRUE" +DO_PARALLEL_PRDGEN="TRUE" + +NET="rrfs_a" +TAG="n3v37" +ARCHIVEDIR="/NCEPDEV/emc-meso/1year/emc.lam" +NCL_REGION="conus" +MODEL="rrfs_a" +RUN="rrfs_a" +envir="para" + +EXTRN_MDL_ICS_OFFSET_HRS="3" +LBC_SPEC_INTVL_HRS="1" +EXTRN_MDL_LBCS_OFFSET_HRS="6" +BOUNDARY_LEN_HRS="72" +BOUNDARY_PROC_GROUP_NUM="72" + +# avaialble retro period: +# 20210511-20210531; 20210718-20210801 +DATE_FIRST_CYCL="20230202" +DATE_LAST_CYCL="20230331" +CYCL_HRS=( "00" "12" ) +CYCL_HRS_SPINSTART=("03" "15") +CYCL_HRS_PRODSTART=("09" "21") +CYCLEMONTH="02,03" +CYCLEDAY="*" + +STARTYEAR=${DATE_FIRST_CYCL:0:4} +STARTMONTH=${DATE_FIRST_CYCL:4:2} +STARTDAY=${DATE_FIRST_CYCL:6:2} +STARTHOUR="00" +ENDYEAR=${DATE_LAST_CYCL:0:4} +ENDMONTH=${DATE_LAST_CYCL:4:2} +ENDDAY=${DATE_LAST_CYCL:6:2} +ENDHOUR="23" + +PREEXISTING_DIR_METHOD="upgrade" +INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}0300 ${DATE_LAST_CYCL}2300 12:00:00" +BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00" +PROD_CYCLEDEF="00 01-05,07-11,13-17,19-23 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *" +PRODLONG_CYCLEDEF="00 00,06,12,18 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *" +ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1400 ${DATE_LAST_CYCL}2300 24:00:00" +if [[ $DO_SPINUP == "TRUE" ]] ; then + SPINUP_CYCLEDEF="00 03-08,15-20 ${CYCLEDAY} ${CYCLEMONTH} ${STARTYEAR} *" +fi +ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}0700 ${DATE_LAST_CYCL}2300 24:00:00" + +FCST_LEN_HRS="18" +FCST_LEN_HRS_SPINUP="1" +POSTPROC_LEN_HRS="18" +POSTPROC_LONG_LEN_HRS="60" +#FCST_LEN_HRS_CYCLES=(21 21 21 63 21 21 21 21 21 63 21 21 21 21 21 63 21 21 21 21 21 63 21 21) +for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=18; done +for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=60; done +DA_CYCLE_INTERV="1" +RESTART_INTERVAL="1 2 3" + +USE_RRFSE_ENS="FALSE" +CYCL_HRS_HYB_FV3LAM_ENS=("00" "01" "02" "03" "04" "05" "06" "07" "08" "09" "10" "11" "12" "13" "14" "15" "16" "17" "18" "19" "20" "21" "22" "23") + +SST_update_hour=01 +GVF_update_hour=04 +SNOWICE_update_hour=00 +netcdf_diag=.true. +binary_diag=.false. + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" + +PRINT_DIFF_PGR="TRUE" + +. set_rrfs_config.sh + +STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. +PTMP="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files. +NWGES="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files +if [[ ${regional_ensemble_option} == "5" ]]; then + RRFSE_NWGES=="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to RRFSE directory NWGES that mostly contains ensemble restart files for GSI hybrid. + NUM_ENS_MEMBERS=30 # FV3LAM ensemble size for GSI hybrid analysis + CYCL_HRS_PRODSTART_ENS=( "19" ) + DO_ENVAR_RADAR_REF="TRUE" +fi + + +if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then + NUM_ENS_MEMBERS=30 +# DO_SPPT=TRUE +# SPPT_MAG=0.5 +# DO_ENSCONTROL="TRUE" + DO_GSIOBSERVER="TRUE" + DO_ENKFUPDATE="TRUE" + ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens" +fi + diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 4f6c3308f..f619a7320 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1766,6 +1766,7 @@ TPP_MAKE_LBCS="1" TPP_RUN_ANAL="1" TPP_RUN_ENKF="1" TPP_RUN_FCST="1" +TPP_RUN_POST="1" # # Walltimes. # @@ -1820,6 +1821,8 @@ MEMO_RUN_PRDGEN="24G" MEMO_RUN_JEDIENVAR_IODA="20G" MEMO_PREP_CYC="40G" MEMO_SAVE_RESTART="40G" +MEMO_SAVE_INPUT="40G" +MEMO_PROC_SMOKE="80" MEMO_SAVE_DA_OUTPUT="40G" # # Maximum number of attempts. diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 4a27a922d..a12281123 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -303,6 +303,7 @@ settings="\ 'tpp_run_anal': ${TPP_RUN_ANAL} 'tpp_run_enkf': ${TPP_RUN_ENKF} 'tpp_run_fcst': ${TPP_RUN_FCST} + 'tpp_run_post': ${TPP_RUN_POST} # # Maximum wallclock time for each task. # @@ -356,6 +357,8 @@ settings="\ 'memo_run_jedienvar_ioda': ${MEMO_RUN_JEDIENVAR_IODA} 'memo_prep_cyc': ${MEMO_PREP_CYC} 'memo_save_restart': ${MEMO_SAVE_RESTART} + 'memo_save_input': ${MEMO_SAVE_INPUT} + 'memo_proc_smoke': ${MEMO_PROC_SMOKE} 'memo_save_da_output': ${MEMO_SAVE_DA_OUTPUT} # # Maximum number of tries for each task. diff --git a/ush/set_rrfs_config_general.sh b/ush/set_rrfs_config_general.sh index 458cae89e..8fd140f01 100644 --- a/ush/set_rrfs_config_general.sh +++ b/ush/set_rrfs_config_general.sh @@ -166,11 +166,13 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then NNODES_RUN_FCST="16" PPN_RUN_POST="64" NNODES_RUN_POST="2" - PPN_PROC_RADAR="64" + PPN_PROC_RADAR="128" NNODES_PROC_RADAR="1" PPN_RUN_ENKF="32" TPP_RUN_ENKF="4" NNODES_RUN_ENKF="24" + MEMO_RUN_PRDGEN="80G" + MEMO_RUN_PROCESSBUFR="80G" MAXTRIES_GET_EXTRN_ICS="2" MAXTRIES_GET_EXTRN_LBCS="2" @@ -189,9 +191,9 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then MAXTRIES_RUN_PRDGEN="2" MAXTRIES_ANAL_GSI="2" - WTIME_RUN_FCST="00:45:00" + WTIME_RUN_FCST="00:55:00" WTIME_RUN_FCST_LONG="02:15:00" - WTIME_RUN_FCST_SPINUP="00:15:00" + WTIME_RUN_FCST_SPINUP="00:25:00" WTIME_MAKE_ICS="00:20:00" WTIME_MAKE_LBCS="02:00:00" WTIME_RUN_ANAL="00:50:00" @@ -347,6 +349,71 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_NA_3km" ]] ; then NNODES_RUN_POST="10" WTIME_RUN_POST="00:35:00" fi + + if [[ $MACHINE == "wcoss2" ]] ; then + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group="64" + LAYOUT_X="50" + LAYOUT_Y="64" + PPN_MAKE_ICS="32" + TPP_MAKE_ICS="4" + PPN_MAKE_LBCS="64" + TPP_MAKE_LBCS="2" + NNODES_MAKE_ICS="2" + NNODES_MAKE_LBCS="1" + PPN_RUN_ANAL="16" + NNODES_RUN_ANAL="22" + TPP_RUN_ANAL="8" + PPN_RUN_FCST="64" + NNODES_RUN_FCST="31" + TPP_RUN_FCST="2" + PPN_RUN_POST="32" + NNODES_RUN_POST="4" + TPP_RUN_POST="4" + PPN_PROC_RADAR="64" + NNODES_PROC_RADAR="1" + PPN_RUN_ENKF="4" + NNODES_RUN_ENKF="300" + TPP_RUN_ENKF="32" + PPN_RUN_NONVARCLDANL="128" + + MAXTRIES_MAKE_ICS="2" + MAXTRIES_MAKE_LBCS="2" + MAXTRIES_RUN_PREPSTART="2" + MAXTRIES_RUN_FCST="2" + MAXTRIES_ANAL_GSI="2" + MAXTRIES_ANAL_ENKF="2" + MAXTRIES_RUN_POST="2" + MAXTRIES_RUN_PRDGEN="2" + MAXTRIES_ANAL_GSI="2" + + MEMO_RUN_NONVARCLDANL="256G" + MEMO_PROC_SMOKE="256G" + MEMO_RUN_PREPSTART="120G" + MEMO_PREP_CYC="120G" + MEMO_SAVE_RESTART="120G" + MEMO_SAVE_INPUT="120G" + + WTIME_RUN_FCST="01:45:00" + + START_TIME_SPINUP="01:10:00" + START_TIME_PROD="02:20:00" + START_TIME_LATE_ANALYSIS="01:40:00" + START_TIME_CONVENTIONAL="00:60:00" + START_TIME_NSSLMOSIAC="00:65:00" + + FV3GFS_FILE_FMT_ICS="netcdf" + FV3GFS_FILE_FMT_LBCS="netcdf" + + WTIME_RUN_FCST="02:30:00" + WTIME_MAKE_LBCS="02:30:00" + + CLEAN_OLDRUN_HRS="24" + CLEAN_OLDFCST_HRS="12" + CLEAN_OLDSTMPPOST_HRS="12" + CLEAN_NWGES_HRS="48" + fi + fi if [[ ${PREDEF_GRID_NAME} == "RRFS_NA_13km" ]] ; then diff --git a/ush/setup.sh b/ush/setup.sh index 5f1ada679..8c04b98be 100644 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1397,8 +1397,8 @@ if [ "${RUN_ENVIR}" = "nco" ]; then ENSCTRL_CYCLE_BASEDIR="${ENSCTRL_STMP}" COMROOT="$PTMP" ENSCTRL_COMROOT="${ENSCTRL_PTMP}" - COMOUT_BASEDIR="$COMROOT/${envir}" - ENSCTRL_COMOUT_BASEDIR="${ENSCTRL_COMROOT}/${envir}" + COMOUT_BASEDIR="$COMROOT/prod" + ENSCTRL_COMOUT_BASEDIR="${ENSCTRL_COMROOT}/prod" ENSCTRL_COMOUT_DIR="${ENSCTRL_COMOUT_BASEDIR}/${RUN_ensctrl}.@Y@m@d/@H" NWGES_BASEDIR="$NWGES" ENSCTRL_NWGES_BASEDIR="${ENSCTRL_NWGES}" diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index b552feb63..d857880a6 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -170,7 +170,7 @@ define resources used for each tasks {{ nnodes_proc_smoke }}:ppn={{ ppn_proc_smoke }}"> - + {{ nnodes_proc_radar }}:ppn={{ ppn_proc_radar }}"> @@ -193,6 +193,7 @@ define resources used for each tasks {{ nnodes_run_prepstart }}:ppn={{ ppn_run_prepstart }}"> +{{ memo_prep_cyc}}"> {{ nnodes_save_restart }}:ppn={{ ppn_save_restart }}"> @@ -203,16 +204,10 @@ define resources used for each tasks {{ nnodes_run_postanal }}:ppn={{ ppn_run_postanal }}"> -{{ nnodes_run_post }}:ppn={{ ppn_run_post }}"> - - {{ nnodes_run_prdgen }}:ppn={{ ppn_run_prdgen }}"> -{{ nnodes_run_enkf }}:ppn={{ ppn_run_enkf }}:tpp={{ tpp_run_enkf }}"> - - {{ nnodes_run_nonvarcldanl }}:ppn={{ ppn_run_nonvarcldanl }}"> @@ -223,6 +218,20 @@ define resources used for each tasks {%- endif %} + +{%- if machine in ["WCOSS2"] %} +{{ nnodes_run_post }}:ppn={{ ppn_run_post }}:tpp={{ tpp_run_post }}"> +{%- else %} +{{ nnodes_run_post }}:ppn={{ ppn_run_post }}"> +{%- endif %} + + +{%- if machine in ["WCOSS2"] %} +{{ nnodes_run_enkf }}:ppn={{ ppn_run_enkf }}:tpp={{ tpp_run_enkf }}"> +{%- else %} +{{ nnodes_run_enkf }}:ppn={{ ppn_run_enkf }}"> +{%- endif %} + {%- if machine in ["WCOSS2"] %} {{ nnodes_make_ics }}:ppn={{ ppn_make_ics }}:tpp={{ tpp_make_ics }}"> @@ -958,7 +967,7 @@ MODULES_RUN_TASK_FP script. &NATIVE_ALL; &WALLTIME_PREP_CYC; &NODESIZE_ALL; - {{ memo_prep_cyc }} + &MEMO_PREP_CYC; &TAG;_&PREP_CYC_SPINUP_TN;{{ uscore_ensmem_name }} &LOGDIR;/&PREP_CYC_SPINUP_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1070,7 +1079,7 @@ MODULES_RUN_TASK_FP script. &NATIVE_ALL; &WALLTIME_SAVE_RESTART; &NODESIZE_ALL; - {{ memo_save_restart }} + &MEMO_PREP_CYC; &TAG;_&SAVE_RESTART_TN;_ensinit{{ uscore_ensmem_name }} &LOGDIR;/&SAVE_RESTART_TN;_ensinit{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1088,7 +1097,7 @@ MODULES_RUN_TASK_FP script. @Y@m@d@H@M00 - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_ensinit/RESTART/coupler.res + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_ensinit/RESTART/coupler.res @@ -1107,7 +1116,7 @@ MODULES_RUN_TASK_FP script. &NATIVE_ALL; &WALLTIME_PREP_CYC; &NODESIZE_ALL; - {{ memo_prep_cyc }} + &MEMO_PREP_CYC; &TAG;_&PREP_CYC_SPINUP_TN;_ensinit{{ uscore_ensmem_name }} &LOGDIR;/&PREP_CYC_SPINUP_TN;_ensinit{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1630,7 +1639,7 @@ MODULES_RUN_TASK_FP script. &NATIVE_ALL; &WALLTIME_SAVE_RESTART; &NODESIZE_ALL; - {{ memo_save_restart }} + &MEMO_PREP_CYC; &TAG;_&SAVE_RESTART_TN;{{ uscore_ensmem_name }}_f#fhr# &LOGDIR;/&SAVE_RESTART_TN;_spinup{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H.log @@ -1648,8 +1657,8 @@ MODULES_RUN_TASK_FP script. @Y@m@d@H@M00 - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/coupler.res - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/@Y@m@d.@H0000.coupler.res + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/coupler.res + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_spinup/RESTART/@Y@m@d.@H0000.coupler.res @@ -1760,7 +1769,7 @@ MODULES_RUN_TASK_FP script. &NATIVE_ALL; &WALLTIME_PREP_CYC; &NODESIZE_ALL; - {{ memo_prep_cyc }} + &MEMO_PREP_CYC; &TAG;_&PREP_CYC_PROD_TN;{{ uscore_ensmem_name }} &LOGDIR;/&PREP_CYC_PROD_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1938,9 +1947,9 @@ MODULES_RUN_TASK_FP script. {%- endif %} {% if machine in ["JET", "HERA"] -%} - &OBSPATH;/@Y@m@d@H.rap.t@Hz.prepbufr.tm00 + &OBSPATH;/@Y@m@d@H.rap.t@Hz.prepbufr.tm00 {% else -%} - &OBSPATH;/rap.@Y@m@d/rap.t@Hz.prepbufr.tm00 + &OBSPATH;/rap.@Y@m@d/rap.t@Hz.prepbufr.tm00 {% endif -%} @@ -2683,7 +2692,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_save_restart }}:ppn={{ ppn_save_restart }} {{ wtime_save_restart }} &NODESIZE_ALL; - {{ memo_save_restart }} + &MEMO_PREP_CYC; &TAG;_&SAVE_RESTART_TN;{{ uscore_ensmem_name }}_f#fhr# &LOGDIR;/&SAVE_RESTART_TN;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H.log @@ -2701,8 +2710,8 @@ MODULES_RUN_TASK_FP script. @Y@m@d@H@M00 - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/coupler.res - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/coupler.res + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res @@ -2722,7 +2731,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_save_restart }}:ppn={{ ppn_save_restart }} {{ wtime_save_restart }} &NODESIZE_ALL; - {{ memo_save_restart }} + &MEMO_PREP_CYC; &TAG;_&SAVE_RESTART_TN;{{ uscore_ensmem_name }}_f#fhr# &LOGDIR;/&SAVE_RESTART_TN;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H.log @@ -2740,8 +2749,8 @@ MODULES_RUN_TASK_FP script. @Y@m@d@H@M00 - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/coupler.res - &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/coupler.res + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam/RESTART/@Y@m@d.@H0000.coupler.res