diff --git a/jobs/JREGIONAL_RUN_ENKF_POST b/jobs/JREGIONAL_RUN_ENKF_POST new file mode 100755 index 000000000..16747f032 --- /dev/null +++ b/jobs/JREGIONAL_RUN_ENKF_POST @@ -0,0 +1,110 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This J-JOB script does the post-processing of the EnKF updated files +# for the FV3-LAM model +# +#----------------------------------------------------------------------- + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that runs a analysis with FV3 for +the specified cycle. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Create the working directory under the cycle directory. +# +#----------------------------------------------------------------------- +# CYCLE_TYPE is to indicate which type of cycle this job for: +# spinup: this job is for spin-up cycles +# others(prod): this job is for product cycle +# +CYCLE_TYPE=${CYCLE_TYPE:-prod} +if [ ${CYCLE_TYPE} == "spinup" ]; then + enkfworkdir="${CYCLE_DIR}/enkfupdt_spinup" +else + enkfworkdir="${CYCLE_DIR}/enkfupdt" +fi +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_run_enkf_post.sh \ + cycle_dir="${CYCLE_DIR}" \ + cycle_type="${CYCLE_TYPE}" \ + enkfworkdir="${enkfworkdir}" \ + NWGES_DIR="${NWGES_DIR}" \ + slash_ensmem_subdir="${SLASH_ENSMEM_SUBDIR}" \ + memname="${MEM_NAME}" || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/scripts/exregional_run_enkf.sh b/scripts/exregional_run_enkf.sh index dbd20d137..064d3e7e9 100755 --- a/scripts/exregional_run_enkf.sh +++ b/scripts/exregional_run_enkf.sh @@ -415,53 +415,6 @@ fi ${APRUN} $enkfworkdir/enkf.x < enkf.nml 1>stdout 2>stderr || print_err_msg_exit "\ Call to executable to run EnKF returned with nonzero exit code." -# -#----------------------------------------------------------------------- -# -# Loop through the members, restore the EnKF analysis back to -# separate tracer and dynvar files, copy them -# to nwges staging locations -# -#----------------------------------------------------------------------- -# - -for imem in $(seq 1 $nens); do - - memchar="mem"$(printf %04i $imem) - memcharv0="mem"$(printf %03i $imem) - memstr=$memchar - slash_ensmem_subdir=$memchar - - if [ ${cycle_type} == "spinup" ]; then - enkfanal_nwges_dir=${NWGES_DIR}/${slash_ensmem_subdir}/anal_enkf_spinup - bkpath=${cycle_dir}/${slash_ensmem_subdir}/fcst_fv3lam_spinup/INPUT - else - enkfanal_nwges_dir=${NWGES_DIR}/${slash_ensmem_subdir}/anal_enkf - bkpath=${cycle_dir}/${slash_ensmem_subdir}/fcst_fv3lam/INPUT - fi - mkdir_vrfy -p ${enkfanal_nwges_dir} - - FileUpdated=fv3sar_tile1_${memcharv0}_dynvartracer - - cp_vrfy $bkpath/fv_tracer.res.tile1.nc ./${memcharv0}_fv3_tracer - cp_vrfy $bkpath/fv_core.res.tile1.nc ./${memcharv0}_fv3_dynvars - if [ $imem == 1 ];then - ncvarlst_noaxis_time_new ${memcharv0}_fv3_tracer > nck_tracer_list.txt - ncvarlst_noaxis_time_new ${memcharv0}_fv3_dynvars > nck_dynvar_list.txt - fi - user_nck_dynvar_list=`cat nck_dynvar_list.txt|paste -sd "," - | tr -d '[:space:]'` - user_nck_tracer_list=`cat nck_tracer_list.txt |paste -sd "," - | tr -d '[:space:]'` - - ncks -A -v $user_nck_dynvar_list $FileUpdated ${memcharv0}_fv3_dynvars - cp_vrfy ${memcharv0}_fv3_dynvars ${enkfanal_nwges_dir}/fv_core.res.tile1.nc - cp_vrfy ${memcharv0}_fv3_dynvars ${bkpath}/fv_core.res.tile1.nc - ncks -A -v $user_nck_tracer_list $FileUpdated ${memcharv0}_fv3_tracer - ncks --no_abc -O -x -v yaxis_2 ${memcharv0}_fv3_tracer tmp_${memcharv0}_tracer - cp_vrfy tmp_${memcharv0}_tracer ${enkfanal_nwges_dir}/fv_tracer.res.tile1.nc - cp_vrfy tmp_${memcharv0}_tracer ${bkpath}/fv_tracer.res.tile1.nc - -done - print_info_msg " ======================================================================== EnKF PROCESS completed successfully!!! diff --git a/scripts/exregional_run_enkf_post.sh b/scripts/exregional_run_enkf_post.sh new file mode 100755 index 000000000..89c6881af --- /dev/null +++ b/scripts/exregional_run_enkf_post.sh @@ -0,0 +1,220 @@ +#!/bin/bash +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh + +function ncvarlst_noaxis_time { ncks --trd -m ${1} | grep -E ': type' | cut -f 1 -d ' ' | sed 's/://' | sort |grep -v -i -E "axis|time" ; } +function ncvarlst_noaxis_time_new { ncks -m ${1} | grep -E 'name.*=' | cut -f 2 -d '=' | grep -o '"*.*"' | sed 's/"//g' | sort |grep -v -i -E "axis|time" ; } +export HDF5_USE_FILE_LOCKING=FALSE #clt to avoild recenter's error "NetCDF: HDF error" +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that runs EnKF analysis with FV3 for the +specified cycle. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( "cycle_dir" "cycle_type" "enkfworkdir" "NWGES_DIR" "slash_ensmem_subdir" "memname") +process_args valid_args "$@" + +cycle_type=${cycle_type:-prod} + +case $MACHINE in +# +"WCOSS_C" | "WCOSS") +# + module load NCO/4.7.0 + module list + ulimit -s unlimited + ulimit -a + APRUN="mpirun -l -np ${PE_MEMBER01}" + ;; +# +"WCOSS_DELL_P3") +# + module load NCO/4.7.0 + module list + ulimit -s unlimited + ulimit -a + APRUN="mpirun -l -np ${PE_MEMBER01}" + ;; +# +"THEIA") +# + ulimit -s unlimited + ulimit -a + np=${SLURM_NTASKS} + APRUN="mpirun -np ${np}" + ;; +# +"HERA") + module load nco/4.9.3 + ulimit -s unlimited + ulimit -v unlimited + ulimit -a + export OMP_NUM_THREADS=1 +# export OMP_STACKSIZE=300M + APRUN="srun" + ;; +# +"ORION") + ulimit -s unlimited + ulimit -a + export OMP_NUM_THREADS=1 + export OMP_STACKSIZE=1024M + APRUN="srun" + ;; +# +"JET") + ulimit -s unlimited + ulimit -a + APRUN="srun" + ;; +# +"ODIN") +# + module list + + ulimit -s unlimited + ulimit -a + APRUN="srun -n ${PE_MEMBER01}" + ;; +# +esac +# +#----------------------------------------------------------------------- +# +# Extract from CDATE the starting year, month, day, and hour of the +# forecast. These are needed below for various operations. +# +#----------------------------------------------------------------------- +# +START_DATE=$(echo "${CDATE}" | sed 's/\([[:digit:]]\{2\}\)$/ \1/') + +YYYYMMDDHH=$(date +%Y%m%d%H -d "${START_DATE}") + +vlddate=$CDATE + +# +#----------------------------------------------------------------------- +# +# Go to working directory. +# +#----------------------------------------------------------------------- +# + +cd_vrfy $enkfworkdir + +# +#----------------------------------------------------------------------- +# +# For each member, restore the EnKF analysis back to +# separate tracer and dynvar files, copy them +# to nwges staging locations +# +#----------------------------------------------------------------------- +# + +if [ ${cycle_type} == "spinup" ]; then + enkfanal_nwges_dir=${NWGES_DIR}/${slash_ensmem_subdir}/anal_enkf_spinup + bkpath=${cycle_dir}/${slash_ensmem_subdir}/fcst_fv3lam_spinup/INPUT +else + enkfanal_nwges_dir=${NWGES_DIR}/${slash_ensmem_subdir}/anal_enkf + bkpath=${cycle_dir}/${slash_ensmem_subdir}/fcst_fv3lam/INPUT +fi +mkdir_vrfy -p ${enkfanal_nwges_dir} + +FileUpdated=fv3sar_tile1_${memname}_dynvartracer + +cp_vrfy $bkpath/fv_tracer.res.tile1.nc ./${memname}_fv3_tracer +cp_vrfy $bkpath/fv_core.res.tile1.nc ./${memname}_fv3_dynvars +ncvarlst_noaxis_time_new ${memname}_fv3_tracer > nck_tracer_list.txt +ncvarlst_noaxis_time_new ${memname}_fv3_dynvars > nck_dynvar_list.txt +user_nck_dynvar_list=`cat nck_dynvar_list.txt|paste -sd "," - | tr -d '[:space:]'` +user_nck_tracer_list=`cat nck_tracer_list.txt |paste -sd "," - | tr -d '[:space:]'` + +# +#----------------------------------------------------------------------- +# +# Extract dynvars variables from the EnKF analysi, update the +# dynvar files, copy to nwges staging locations +# +#----------------------------------------------------------------------- +# +ncks -A -v $user_nck_dynvar_list $FileUpdated ${memname}_fv3_dynvars +cp_vrfy ${memname}_fv3_dynvars ${enkfanal_nwges_dir}/fv_core.res.tile1.nc +cp_vrfy ${memname}_fv3_dynvars ${bkpath}/fv_core.res.tile1.nc +# +#----------------------------------------------------------------------- +# +# Extract tracer variables from the EnKF analysi, update the +# tracer files, copy to nwges staging locations +# +#----------------------------------------------------------------------- +# +ncks -A -v $user_nck_tracer_list $FileUpdated ${memname}_fv3_tracer +ncks --no_abc -O -x -v yaxis_2 ${memname}_fv3_tracer tmp_${memname}_tracer +cp_vrfy tmp_${memname}_tracer ${enkfanal_nwges_dir}/fv_tracer.res.tile1.nc +cp_vrfy tmp_${memname}_tracer ${bkpath}/fv_tracer.res.tile1.nc + + +print_info_msg " +======================================================================== +EnKF POST-PROCESS completed successfully for mem"${memname}"!!! + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/ush/config.sh.RRFS_CONUS_13km b/ush/config.sh.RRFS_CONUS_13km index 206112090..1d729d646 100644 --- a/ush/config.sh.RRFS_CONUS_13km +++ b/ush/config.sh.RRFS_CONUS_13km @@ -42,37 +42,6 @@ DO_RETRO="TRUE" QUILTING="TRUE" CCPP_PHYS_SUITE="FV3_HRRR" -if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then - NUM_ENS_MEMBERS=9 - DO_SPPT=TRUE - SPPT_MAG=0.5 - DO_DACYCLE="FALSE" - DO_ENKFUPDATE="TRUE" - DO_SPINUP="FALSE" - DO_SURFACE_CYCLE="FALSE" - DO_NONVAR_CLDANAL="FALSE" - DO_REFL2TTEN="FALSE" - DO_ENSCONTROL="TRUE" - DO_GSIOBSERVER="TRUE" - EXTRN_MDL_NAME_ICS="GEFS" - EXTRN_MDL_NAME_ICS="HRRRDAS" - EXTRN_MDL_NAME_LBCS="GEFS" - EXTRN_MDL_ICS_OFFSET_HRS="0" - EXTRN_MDL_LBCS_OFFSET_HRS="0" - LBC_SPEC_INTVL_HRS="3" - DA_CYCLE_INTERV="3" - ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens" -else - EXTRN_MDL_NAME_ICS="FV3GFS" - EXTRN_MDL_NAME_LBCS="FV3GFS" - EXTRN_MDL_ICS_OFFSET_HRS="3" - EXTRN_MDL_LBCS_OFFSET_HRS="6" - LBC_SPEC_INTVL_HRS="1" - DA_CYCLE_INTERV="1" - ARCHIVEDIR="/1year/BMC/wrfruc/rrfs_dev1" -fi - - BOUNDARY_LEN_HRS="24" EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS="0" @@ -145,7 +114,7 @@ USE_CUSTOM_POST_CONFIG_FILE="TRUE" TESTBED_FIELDS_FN="testbed_fields_bgdawp.txt" CUSTOM_POST_CONFIG_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/postxconfig-NT-fv3lam_rrfs.txt" CUSTOM_POST_PARAMS_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/params_grib2_tbl_new" - +ARCHIVEDIR="/1year/BMC/wrfruc/rrfs_dev1 NCARG_ROOT="/apps/ncl/6.5.0-CentOS6.10_64bit_nodap_gnu447" NCL_HOME="/home/rtrr/RRFS/graphics" NCL_REGION="conus" @@ -172,3 +141,13 @@ STMP="YourOwnSpace" # Path to directory STMP that mostly contains input files. PTMP="YourOwnSpace" # Path to directory STMP that mostly contains input files. NWGES="YourOwnSpace" # Path to directory NWGES that save boundary, cold initial, restart files. +if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then + NUM_ENS_MEMBERS=9 +# DO_SPPT=TRUE +# SPPT_MAG=0.5 +# DO_ENSCONTROL="TRUE" + DO_GSIOBSERVER="TRUE" + DO_ENKFUPDATE="TRUE" + ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens" +fi + diff --git a/ush/config.sh.RRFS_CONUS_3km b/ush/config.sh.RRFS_CONUS_3km index 178bec107..338bbe018 100644 --- a/ush/config.sh.RRFS_CONUS_3km +++ b/ush/config.sh.RRFS_CONUS_3km @@ -46,37 +46,6 @@ DO_RETRO="TRUE" QUILTING="TRUE" CCPP_PHYS_SUITE="FV3_HRRR" -if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then - NUM_ENS_MEMBERS=9 -# DO_SPPT=TRUE -# SPPT_MAG=0.5 - DO_DACYCLE="FALSE" - DO_SPINUP="FALSE" - DO_SURFACE_CYCLE="FALSE" - DO_NONVAR_CLDANAL="FALSE" - DO_REFL2TTEN="FALSE" -# DO_ENSCONTROL="TRUE" - DO_GSIOBSERVER="TRUE" - DO_ENKFUPDATE="TRUE" -# EXTRN_MDL_NAME_ICS="HRRRDAS" - EXTRN_MDL_NAME_ICS="GDASENKF" -# EXTRN_MDL_NAME_LBCS="GEFS" - EXTRN_MDL_NAME_LBCS="FV3GFS" - EXTRN_MDL_ICS_OFFSET_HRS="6" - EXTRN_MDL_LBCS_OFFSET_HRS="0" - LBC_SPEC_INTVL_HRS="1" - DA_CYCLE_INTERV="1" - ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens" -else - EXTRN_MDL_NAME_ICS="FV3GFS" - EXTRN_MDL_NAME_LBCS="FV3GFS" - EXTRN_MDL_ICS_OFFSET_HRS="3" - EXTRN_MDL_LBCS_OFFSET_HRS="0" - LBC_SPEC_INTVL_HRS="1" - DA_CYCLE_INTERV="1" - ARCHIVEDIR="/1year/BMC/wrfruc/rrfs_dev1" -fi - BOUNDARY_LEN_HRS="21" EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS="0" @@ -114,7 +83,6 @@ SST_update_hour=01 GVF_update_hour=04 NCORES_RUN_ANAL=240 -NCORES_RUN_ENKF=100 HYBENSMEM_NMIN=66 HALO_BLEND=20 @@ -145,6 +113,7 @@ USE_CUSTOM_POST_CONFIG_FILE="TRUE" TESTBED_FIELDS_FN="testbed_fields_bgdawp.txt" CUSTOM_POST_CONFIG_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/postxconfig-NT-fv3lam_rrfs.txt" CUSTOM_POST_PARAMS_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/params_grib2_tbl_new" +ARCHIVEDIR="/1year/BMC/wrfruc/rrfs_dev1" NCARG_ROOT="/apps/ncl/6.5.0-CentOS6.10_64bit_nodap_gnu447" NCL_HOME="/home/rtrr/RRFS/graphics" NCL_REGION="conus" @@ -168,3 +137,13 @@ STMP="YourOwnSpace/NCO_dirs/stmp" # Path to directory STMP that mostly contains PTMP="YourOwnSpace/NCO_dirs/ptmp" # Path to directory STMP that mostly contains input files. NWGES="YourOwnSpace/NCO_dirs/nwges" # Path to directory NWGES that save boundary, cold initial, restart files +if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then + NUM_ENS_MEMBERS=9 +# DO_SPPT=TRUE +# SPPT_MAG=0.5 +# DO_ENSCONTROL="TRUE" + DO_GSIOBSERVER="TRUE" + DO_ENKFUPDATE="TRUE" + ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens" +fi + diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 3c72f3f49..9e21f78b6 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1555,7 +1555,7 @@ PPN_RUN_FCST="24" # This may have to be changed depending on the number of thre PPN_RUN_POST="24" PPN_RUN_WGRIB2="1" PPN_RUN_ANAL="24" -PPN_RUN_ENKF="24" +PPN_RUN_ENKF="4" PPN_PROC_RADAR="24" PPN_PROC_LIGHTNING="1" PPN_PROC_BUFR="1" @@ -1573,12 +1573,12 @@ WTIME_GET_EXTRN_LBCS="00:45:00" WTIME_MAKE_ICS="00:30:00" WTIME_MAKE_LBCS="01:30:00" WTIME_RUN_PREPSTART="00:10:00" -WTIME_RUN_PREPSTART_ENSMEAN="00:30:00" +WTIME_RUN_PREPSTART_ENSMEAN="02:30:00" WTIME_RUN_FCST="04:30:00" WTIME_RUN_POST="00:15:00" WTIME_RUN_WGRIB2="00:40:00" WTIME_RUN_ANAL="00:30:00" -WTIME_RUN_ENKF="00:45:00" +WTIME_RUN_ENKF="01:00:00" WTIME_PROC_RADAR="00:25:00" WTIME_PROC_LIGHTNING="00:25:00" WTIME_PROC_BUFR="00:25:00" diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index ac25f2487..4a124190c 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -1040,7 +1040,7 @@ MODULES_RUN_TASK_FP script. - + {%- else %} @@ -1439,7 +1439,7 @@ MODULES_RUN_TASK_FP script. &WALL_LIMIT_ANAL; &LOAD_MODULES_RUN_TASK_FP; "&RUN_ENKFUPDT_TN;" &JOBSDIR;/JREGIONAL_RUN_ENKF - {% if machine in ["JET", "HERA"] -%} + {% if machine in ["JET"] -%} {{ ncores_run_enkf }} {{ native_run_enkf }} &RRFS_RESERVE; {% else -%} @@ -1462,6 +1462,50 @@ MODULES_RUN_TASK_FP script. + + + + + {%- for m in range(1, num_ens_members+1) -%} + {%- set fmtstr=" %0"~ndigits_ensmem_names~"d" -%} + {{- fmtstr%m -}} + {%- endfor %} + {% for m in range(1, num_ens_members+1) %}{{ " mem%03d"%m }}{% endfor %} + + + + &RSRV_DEFAULT; + &WALL_LIMIT_PRE; + + &LOAD_MODULES_RUN_TASK_FP; "&RUN_PREPSTART_TN;" "&JOBSDIR;/JREGIONAL_RUN_ENKF_POST" + {{ nnodes_run_prepstart }}:ppn={{ ppn_run_prepstart }} + &NCORES_PER_NODE; + {{ wtime_run_prepstart }} + &TAG;run_enkfpost{{ uscore_ensmem_name }} + &LOGDIR;/run_enkfpost{{ uscore_ensmem_name }}_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + PDY@Y@m@d + CDATE@Y@m@d@H + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CYCLE_ROOT&CYCLE_BASEDIR; + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + MEM_NAME#memName# + + + + + + + + + + + {%- endif %}