From 1005721a70dfc3eb4aba581982e011db44913de9 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 18 Oct 2024 17:35:53 +0000 Subject: [PATCH 01/25] Fix Look-in dir for gather3 --- .../grid2obs/StatAnalysis_fcstCAM_GatherByDay.conf | 2 +- .../precip/StatAnalysis_fcstCAM_GatherByDay.conf | 2 +- .../snowfall/StatAnalysis_fcstCAM_GatherByDay.conf | 2 +- scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh | 2 +- scripts/stats/cam/exevs_hireswarw_precip_stats.sh | 2 +- scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh | 2 +- .../stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh | 2 +- .../stats/cam/exevs_hireswarwmem2_precip_stats.sh | 2 +- .../stats/cam/exevs_hireswarwmem2_snowfall_stats.sh | 2 +- scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh | 2 +- scripts/stats/cam/exevs_hireswfv3_precip_stats.sh | 2 +- scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh | 2 +- scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh | 2 +- scripts/stats/cam/exevs_hrrr_precip_stats.sh | 2 +- scripts/stats/cam/exevs_hrrr_snowfall_stats.sh | 2 +- scripts/stats/cam/exevs_namnest_grid2obs_stats.sh | 2 +- scripts/stats/cam/exevs_namnest_precip_stats.sh | 2 +- scripts/stats/cam/exevs_namnest_snowfall_stats.sh | 2 +- ush/cam/cam_create_output_dirs.py | 13 +++++++++++++ ush/cam/cam_stats_grid2obs_create_job_script.py | 2 +- 20 files changed, 32 insertions(+), 19 deletions(-) diff --git a/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_GatherByDay.conf b/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_GatherByDay.conf index 2f0d70082c..7d10259bc7 100644 --- a/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_GatherByDay.conf +++ b/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_GatherByDay.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[COMOUTsmall]} +INPUT_BASE = {ENV[COMOUTsmall]}/gather_small OUTPUT_BASE = {ENV[COMOUTfinal]} LOG_DIR = {ENV[MET_PLUS_OUT]}/out/logs TMP_DIR = {ENV[MET_PLUS_OUT]}/stat_analysis/tmp diff --git a/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_GatherByDay.conf b/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_GatherByDay.conf index 5463b9be86..33a658afb1 100755 --- a/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_GatherByDay.conf +++ b/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_GatherByDay.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[COMOUTsmall]} +INPUT_BASE = {ENV[COMOUTsmall]}/gather_small OUTPUT_BASE = {ENV[COMOUTfinal]} LOG_DIR = {ENV[MET_PLUS_OUT]}/out/logs TMP_DIR = {ENV[MET_PLUS_OUT]}/stat_analysis/tmp diff --git a/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_GatherByDay.conf b/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_GatherByDay.conf index 5463b9be86..33a658afb1 100755 --- a/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_GatherByDay.conf +++ b/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_GatherByDay.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[COMOUTsmall]} +INPUT_BASE = {ENV[COMOUTsmall]}/gather_small OUTPUT_BASE = {ENV[COMOUTfinal]} LOG_DIR = {ENV[MET_PLUS_OUT]}/out/logs TMP_DIR = {ENV[MET_PLUS_OUT]}/stat_analysis/tmp diff --git a/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh b/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh index 74e35a03dc..9b5ab2a1ce 100755 --- a/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh @@ -298,7 +298,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_hireswarw_precip_stats.sh b/scripts/stats/cam/exevs_hireswarw_precip_stats.sh index fe62de4409..5333505fcd 100755 --- a/scripts/stats/cam/exevs_hireswarw_precip_stats.sh +++ b/scripts/stats/cam/exevs_hireswarw_precip_stats.sh @@ -307,7 +307,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh b/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh index cc64575394..934e58a113 100755 --- a/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh @@ -298,7 +298,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh b/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh index b4de92c0b8..691de37333 100755 --- a/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh @@ -299,7 +299,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh b/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh index 393c1889fe..7cff86cb24 100755 --- a/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh +++ b/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh @@ -306,7 +306,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh b/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh index 78eb229bdf..c83caea4d2 100755 --- a/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh @@ -298,7 +298,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh b/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh index ada9e7e974..7180b2c07c 100755 --- a/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh @@ -299,7 +299,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh b/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh index 7bbbbd58a2..aa0590214f 100755 --- a/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh +++ b/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh @@ -306,7 +306,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh b/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh index 1189ce0f43..f151a8dbb6 100755 --- a/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh @@ -298,7 +298,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh b/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh index 32d5a3f5c0..dbd7b723c7 100755 --- a/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh @@ -299,7 +299,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_hrrr_precip_stats.sh b/scripts/stats/cam/exevs_hrrr_precip_stats.sh index 7872d009c2..113d2f5133 100755 --- a/scripts/stats/cam/exevs_hrrr_precip_stats.sh +++ b/scripts/stats/cam/exevs_hrrr_precip_stats.sh @@ -307,7 +307,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh b/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh index 221e546f3a..a6dda5d7ca 100755 --- a/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh @@ -298,7 +298,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh b/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh index 6e42a19fa2..046df92a1f 100755 --- a/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh @@ -299,7 +299,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_namnest_precip_stats.sh b/scripts/stats/cam/exevs_namnest_precip_stats.sh index a8536a5241..4a4693056f 100755 --- a/scripts/stats/cam/exevs_namnest_precip_stats.sh +++ b/scripts/stats/cam/exevs_namnest_precip_stats.sh @@ -308,7 +308,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/scripts/stats/cam/exevs_namnest_snowfall_stats.sh b/scripts/stats/cam/exevs_namnest_snowfall_stats.sh index 83780dc91c..1f9dc1b4a3 100755 --- a/scripts/stats/cam/exevs_namnest_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_namnest_snowfall_stats.sh @@ -298,7 +298,7 @@ if [ $SENDCOM = YES ]; then for MODEL_DIR_PATH in $MET_PLUS_OUT/stat_analysis/$MODELNAME*; do for FILE in $MODEL_DIR_PATH/*; do if [ -s "$FILE" ]; then - cp -v $FILE $COMOUTsmall/. + cp -v $FILE $COMOUTsmall/gather_small/. fi done done diff --git a/ush/cam/cam_create_output_dirs.py b/ush/cam/cam_create_output_dirs.py index 9c6de98edf..1ecc901293 100644 --- a/ush/cam/cam_create_output_dirs.py +++ b/ush/cam/cam_create_output_dirs.py @@ -33,6 +33,7 @@ evs_ver = os.environ['evs_ver'] COMIN = os.environ['COMIN'] COMOUT = os.environ['COMOUT'] +COMOUTsmall = os.environ['COMOUTsmall'] DATA = os.environ['DATA'] NET = os.environ['NET'] RUN = os.environ['RUN'] @@ -270,6 +271,10 @@ COMOUT_restart_base_dir, 'stat_analysis', MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') )) + COMOUT_dir_list.append(os.path.join( + COMOUTsmall, + 'gather_small' + )) COMOUT_dir_list.append(os.path.join( COMOUT, MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') @@ -407,6 +412,10 @@ COMOUT_restart_base_dir, 'stat_analysis', MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') )) + COMOUT_dir_list.append(os.path.join( + COMOUTsmall, + 'gather_small' + )) COMOUT_dir_list.append(os.path.join( COMOUT, MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') @@ -498,6 +507,10 @@ COMOUT_restart_base_dir, 'stat_analysis', MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') )) + COMOUT_dir_list.append(os.path.join( + COMOUTsmall, + 'gather_small' + )) COMOUT_dir_list.append(os.path.join( COMOUT, MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') diff --git a/ush/cam/cam_stats_grid2obs_create_job_script.py b/ush/cam/cam_stats_grid2obs_create_job_script.py index 4060d5a9f3..dd0980d5b4 100644 --- a/ush/cam/cam_stats_grid2obs_create_job_script.py +++ b/ush/cam/cam_stats_grid2obs_create_job_script.py @@ -2043,7 +2043,7 @@ + f"\"job{njob}\", job_type=\"{job_type}\")'" ) elif job_type == 'gather3': - if glob.glob(os.path.join(COMOUTsmall,'*stat')): + if glob.glob(os.path.join(COMOUTsmall,'gather_small','*stat')): job_cmd_list.append( f'{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' From d58ef60c0d3555d799e464206d7a3203514af988 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Thu, 24 Oct 2024 20:19:09 +0000 Subject: [PATCH 02/25] Bugzilla 1547: Run child processes in separate working directories --- .../config.evs.prod.plots.cam.atmos.grid2obs | 2 +- .../config.evs.prod.plots.cam.atmos.headline | 2 +- .../config.evs.prod.plots.cam.atmos.precip | 2 +- .../config.evs.prod.plots.cam.atmos.snowfall | 2 +- .../grid2obs/PointStat_fcstCAM_obsMETAR.conf | 2 +- .../PointStat_fcstCAM_obsMETAR_FIREWX.conf | 2 +- ...intStat_fcstCAM_obsMETAR_FIREWX_PTYPE.conf | 2 +- ...ointStat_fcstCAM_obsMETAR_FIREWX_VAR2.conf | 2 +- .../PointStat_fcstCAM_obsMETAR_PTYPE.conf | 2 +- .../PointStat_fcstCAM_obsMETAR_VAR2.conf | 2 +- .../grid2obs/PointStat_fcstCAM_obsRAOB.conf | 2 +- .../PointStat_fcstCAM_obsRAOB_FIREWX.conf | 2 +- ...PointStat_fcstCAM_obsRAOB_FIREWX_VAR2.conf | 2 +- .../PointStat_fcstCAM_obsRAOB_VAR2.conf | 2 +- .../StatAnalysis_fcstCAM_GatherByCycle.conf | 2 +- ...Analysis_fcstCAM_obsMETAR_GatherByDay.conf | 2 +- ...Analysis_fcstCAM_obsMPING_GatherByDay.conf | 2 +- ...tAnalysis_fcstCAM_obsRAOB_GatherByDay.conf | 2 +- .../cam/precip/GridStat_fcstCAM_obsCCPA.conf | 2 +- .../cam/precip/GridStat_fcstCAM_obsMRMS.conf | 2 +- .../StatAnalysis_fcstCAM_GatherByCycle.conf | 2 +- ...tAnalysis_fcstCAM_obsCCPA_GatherByDay.conf | 2 +- ...tAnalysis_fcstCAM_obsMRMS_GatherByDay.conf | 2 +- .../radar/GridStat_fcstCAM_obsMRMS_REFC.conf | 6 +- .../radar/GridStat_fcstCAM_obsMRMS_RETOP.conf | 6 +- .../GridStat_fcstHREFPPF_obsMRMS_REFC.conf | 6 +- .../GridStat_fcstHREFPPF_obsMRMS_RETOP.conf | 6 +- .../GridStat_fcstHREFPROB_obsMRMS_REFC.conf | 6 +- .../GridStat_fcstHREFPROB_obsMRMS_RETOP.conf | 6 +- .../snowfall/GridStat_fcstCAM_obsNOHRSC.conf | 2 +- .../StatAnalysis_fcstCAM_GatherByCycle.conf | 2 +- ...nalysis_fcstCAM_obsNOHRSC_GatherByDay.conf | 2 +- scripts/plots/cam/exevs_cam_grid2obs_plots.sh | 12 ++- scripts/plots/cam/exevs_cam_headline_plots.sh | 12 ++- scripts/plots/cam/exevs_cam_precip_plots.sh | 12 ++- scripts/plots/cam/exevs_cam_snowfall_plots.sh | 12 ++- scripts/stats/cam/exevs_cam_radar_stats.sh | 10 +++ .../cam/exevs_hireswarw_grid2obs_stats.sh | 50 ++++++++++++ .../stats/cam/exevs_hireswarw_precip_stats.sh | 50 ++++++++++++ .../cam/exevs_hireswarw_snowfall_stats.sh | 50 ++++++++++++ .../cam/exevs_hireswarwmem2_grid2obs_stats.sh | 50 ++++++++++++ .../cam/exevs_hireswarwmem2_precip_stats.sh | 50 ++++++++++++ .../cam/exevs_hireswarwmem2_snowfall_stats.sh | 50 ++++++++++++ .../cam/exevs_hireswfv3_grid2obs_stats.sh | 50 ++++++++++++ .../stats/cam/exevs_hireswfv3_precip_stats.sh | 50 ++++++++++++ .../cam/exevs_hireswfv3_snowfall_stats.sh | 50 ++++++++++++ .../stats/cam/exevs_hrrr_grid2obs_stats.sh | 50 ++++++++++++ scripts/stats/cam/exevs_hrrr_precip_stats.sh | 50 ++++++++++++ .../stats/cam/exevs_hrrr_snowfall_stats.sh | 50 ++++++++++++ .../stats/cam/exevs_namnest_grid2obs_stats.sh | 50 ++++++++++++ .../stats/cam/exevs_namnest_precip_stats.sh | 50 ++++++++++++ .../stats/cam/exevs_namnest_snowfall_stats.sh | 50 ++++++++++++ ush/cam/cam_create_child_workdirs.py | 78 +++++++++++++++++++ ush/cam/cam_create_output_dirs.py | 17 +++- .../cam_plots_grid2obs_create_job_script.py | 2 +- .../cam_plots_headline_create_job_script.py | 2 +- ush/cam/cam_plots_precip_create_job_script.py | 2 +- .../cam_plots_snowfall_create_job_script.py | 2 +- ush/cam/cam_production_restart.py | 4 +- .../cam_stats_grid2obs_create_job_script.py | 24 +++++- .../cam_stats_grid2obs_create_merged_ptype.py | 5 +- ush/cam/cam_stats_precip_create_job_script.py | 24 +++++- .../cam_stats_snowfall_create_job_script.py | 24 +++++- 63 files changed, 1015 insertions(+), 65 deletions(-) create mode 100644 ush/cam/cam_create_child_workdirs.py diff --git a/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.grid2obs b/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.grid2obs index bbb12a6afc..d0438f46ec 100644 --- a/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.grid2obs +++ b/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.grid2obs @@ -17,7 +17,7 @@ printf -v MET_VERSION '%s.' "${MET_VER[@]:0:2}" export MET_VERSION="${MET_VERSION%.}" # Logging -export SAVE_DIR="${DATA}/${VERIF_CASE}/out" +export SAVE_DIR="${DATA}/${VERIF_CASE}/out/workdirs/job{njob}" export RESTART_DIR="${COMOUTplots}/${VERIF_CASE}/restart" export COMPLETED_JOBS_FILE="completed_jobs_${EVAL_PERIOD}.txt" export LOG_TEMPLATE="${SAVE_DIR}/logs/EVS_verif_plotting_job{njob}_$($NDATE)_$$.out" diff --git a/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.headline b/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.headline index e403d3c895..976e7d3ef8 100644 --- a/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.headline +++ b/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.headline @@ -17,7 +17,7 @@ printf -v MET_VERSION '%s.' "${MET_VER[@]:0:2}" export MET_VERSION="${MET_VERSION%.}" # Logging -export SAVE_DIR="${DATA}/${VERIF_CASE}/out" +export SAVE_DIR="${DATA}/${VERIF_CASE}/out/workdirs/job{njob}" export RESTART_DIR="${COMOUTplots}/${VERIF_CASE}/restart" export COMPLETED_JOBS_FILE="completed_jobs.txt" export LOG_TEMPLATE="${SAVE_DIR}/logs/EVS_verif_plotting_job{njob}_$($NDATE)_$$.out" diff --git a/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.precip b/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.precip index ef561fa91f..4d7c2e89a1 100644 --- a/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.precip +++ b/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.precip @@ -17,7 +17,7 @@ printf -v MET_VERSION '%s.' "${MET_VER[@]:0:2}" export MET_VERSION="${MET_VERSION%.}" # Logging -export SAVE_DIR="${DATA}/${VERIF_CASE}/out" +export SAVE_DIR="${DATA}/${VERIF_CASE}/out/workdirs/job{njob}" export RESTART_DIR="${COMOUTplots}/${VERIF_CASE}/restart" export COMPLETED_JOBS_FILE="completed_jobs_${EVAL_PERIOD}.txt" export LOG_TEMPLATE="${SAVE_DIR}/logs/EVS_verif_plotting_job{njob}_$($NDATE)_$$.out" diff --git a/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.snowfall b/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.snowfall index 63195fc597..66e2ce5515 100644 --- a/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.snowfall +++ b/parm/evs_config/cam/config.evs.prod.plots.cam.atmos.snowfall @@ -17,7 +17,7 @@ printf -v MET_VERSION '%s.' "${MET_VER[@]:0:2}" export MET_VERSION="${MET_VERSION%.}" # Logging -export SAVE_DIR="${DATA}/${VERIF_CASE}/out" +export SAVE_DIR="${DATA}/${VERIF_CASE}/out/workdirs/job{njob}" export RESTART_DIR="${COMOUTplots}/${VERIF_CASE}/restart" export COMPLETED_JOBS_FILE="completed_jobs.txt" export LOG_TEMPLATE="${SAVE_DIR}/logs/EVS_verif_plotting_job{njob}_$($NDATE)_$$.out" diff --git a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR.conf b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR.conf index dedacc207c..58125ee433 100755 --- a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR.conf +++ b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX.conf b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX.conf index 8e8a2423ac..a695c78b1c 100755 --- a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX.conf +++ b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX_PTYPE.conf b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX_PTYPE.conf index 3e87e721c5..4d3789bc4f 100755 --- a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX_PTYPE.conf +++ b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX_PTYPE.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX_VAR2.conf b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX_VAR2.conf index 0dcbd9c42a..82895c9de5 100755 --- a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX_VAR2.conf +++ b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_FIREWX_VAR2.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_PTYPE.conf b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_PTYPE.conf index 5eeda483ec..7f0cfa825c 100755 --- a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_PTYPE.conf +++ b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_PTYPE.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_VAR2.conf b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_VAR2.conf index cef2663d18..337aa954c4 100755 --- a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_VAR2.conf +++ b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsMETAR_VAR2.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB.conf b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB.conf index a0968327fd..c1c2db6173 100755 --- a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB.conf +++ b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_FIREWX.conf b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_FIREWX.conf index ec3703220c..b52f1a9739 100755 --- a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_FIREWX.conf +++ b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_FIREWX.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_FIREWX_VAR2.conf b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_FIREWX_VAR2.conf index b5269d88c9..a95cec9220 100755 --- a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_FIREWX_VAR2.conf +++ b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_FIREWX_VAR2.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_VAR2.conf b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_VAR2.conf index 919f06547b..83073656d7 100755 --- a/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_VAR2.conf +++ b/parm/metplus_config/stats/cam/grid2obs/PointStat_fcstCAM_obsRAOB_VAR2.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/{ENV[NEST]}/pb2nc OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_GatherByCycle.conf b/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_GatherByCycle.conf index d8cebd475a..c59ff1611d 100644 --- a/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_GatherByCycle.conf +++ b/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_GatherByCycle.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/gather_small/stat_analysis +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/gather_small/stat_analysis OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/stat_analysis LOG_DIR = {ENV[MET_PLUS_OUT]}/gather_small/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsMETAR_GatherByDay.conf b/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsMETAR_GatherByDay.conf index 735cc1f237..3fe5db233b 100644 --- a/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsMETAR_GatherByDay.conf +++ b/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsMETAR_GatherByDay.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/point_stat OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/gather_small/stat_analysis LOG_DIR = {ENV[MET_PLUS_OUT]}/gather_small/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsMPING_GatherByDay.conf b/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsMPING_GatherByDay.conf index a7ec0e337d..ab0f5e4981 100644 --- a/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsMPING_GatherByDay.conf +++ b/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsMPING_GatherByDay.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/point_stat OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/gather_small/stat_analysis LOG_DIR = {ENV[MET_PLUS_OUT]}/gather_small/out/logs diff --git a/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsRAOB_GatherByDay.conf b/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsRAOB_GatherByDay.conf index ab4fbb32b5..84911cba1b 100644 --- a/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsRAOB_GatherByDay.conf +++ b/parm/metplus_config/stats/cam/grid2obs/StatAnalysis_fcstCAM_obsRAOB_GatherByDay.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/point_stat +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/point_stat OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/gather_small/stat_analysis LOG_DIR = {ENV[MET_PLUS_OUT]}/gather_small/out/logs diff --git a/parm/metplus_config/stats/cam/precip/GridStat_fcstCAM_obsCCPA.conf b/parm/metplus_config/stats/cam/precip/GridStat_fcstCAM_obsCCPA.conf index 1ccfdeb9dc..4a5f053f3d 100755 --- a/parm/metplus_config/stats/cam/precip/GridStat_fcstCAM_obsCCPA.conf +++ b/parm/metplus_config/stats/cam/precip/GridStat_fcstCAM_obsCCPA.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/pcp_combine +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/pcp_combine OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/grid_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/precip/GridStat_fcstCAM_obsMRMS.conf b/parm/metplus_config/stats/cam/precip/GridStat_fcstCAM_obsMRMS.conf index 92d8ee8309..aff60e1a75 100755 --- a/parm/metplus_config/stats/cam/precip/GridStat_fcstCAM_obsMRMS.conf +++ b/parm/metplus_config/stats/cam/precip/GridStat_fcstCAM_obsMRMS.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/pcp_combine +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/pcp_combine OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/grid_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_GatherByCycle.conf b/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_GatherByCycle.conf index d2e9c3f8b2..24309e5374 100755 --- a/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_GatherByCycle.conf +++ b/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_GatherByCycle.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/gather_small/stat_analysis +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/gather_small/stat_analysis OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/stat_analysis LOG_DIR = {ENV[MET_PLUS_OUT]}/out/logs diff --git a/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_obsCCPA_GatherByDay.conf b/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_obsCCPA_GatherByDay.conf index 1ecc52abdd..be772c9404 100755 --- a/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_obsCCPA_GatherByDay.conf +++ b/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_obsCCPA_GatherByDay.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/grid_stat +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/grid_stat OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/gather_small/stat_analysis LOG_DIR = {ENV[MET_PLUS_OUT]}/gather_small/out/logs diff --git a/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_obsMRMS_GatherByDay.conf b/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_obsMRMS_GatherByDay.conf index ee353e77dc..99ac356f0c 100755 --- a/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_obsMRMS_GatherByDay.conf +++ b/parm/metplus_config/stats/cam/precip/StatAnalysis_fcstCAM_obsMRMS_GatherByDay.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/grid_stat +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/grid_stat OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/gather_small/stat_analysis LOG_DIR = {ENV[MET_PLUS_OUT]}/gather_small/out/logs diff --git a/parm/metplus_config/stats/cam/radar/GridStat_fcstCAM_obsMRMS_REFC.conf b/parm/metplus_config/stats/cam/radar/GridStat_fcstCAM_obsMRMS_REFC.conf index e1c7be4427..2c2316296e 100644 --- a/parm/metplus_config/stats/cam/radar/GridStat_fcstCAM_obsMRMS_REFC.conf +++ b/parm/metplus_config/stats/cam/radar/GridStat_fcstCAM_obsMRMS_REFC.conf @@ -2,8 +2,8 @@ [config] # Final conf file used by all processes -LOG_DIR = {OUTPUT_BASE}/logs -METPLUS_CONF = {TMP_DIR}/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf +LOG_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/logs +METPLUS_CONF = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/tmp/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf LOG_METPLUS = {LOG_DIR}/run_metplus.job{ENV[JOBNUM]}.gridstat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.log.{LOG_TIMESTAMP_TEMPLATE} SCRUB_STAGING_DIR = False @@ -98,7 +98,7 @@ OBS_GRID_STAT_INPUT_DIR = {ENV[EVSINmrms]} OBS_GRID_STAT_INPUT_TEMPLATE = {ENV[OBS_INPUT_TEMPLATE]} -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/grid_stat +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/grid_stat GRID_STAT_OUTPUT_PREFIX = {MODEL}_{FCST_VAR1_NAME}_{ENV[DOMAIN]} diff --git a/parm/metplus_config/stats/cam/radar/GridStat_fcstCAM_obsMRMS_RETOP.conf b/parm/metplus_config/stats/cam/radar/GridStat_fcstCAM_obsMRMS_RETOP.conf index 99c6ba9e96..7633fb3120 100644 --- a/parm/metplus_config/stats/cam/radar/GridStat_fcstCAM_obsMRMS_RETOP.conf +++ b/parm/metplus_config/stats/cam/radar/GridStat_fcstCAM_obsMRMS_RETOP.conf @@ -2,8 +2,8 @@ [config] # Final conf file used by all processes -LOG_DIR = {OUTPUT_BASE}/logs -METPLUS_CONF = {TMP_DIR}/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf +LOG_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/logs +METPLUS_CONF = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/tmp/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf LOG_METPLUS = {LOG_DIR}/run_metplus.job{ENV[JOBNUM]}.gridstat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.log.{LOG_TIMESTAMP_TEMPLATE} SCRUB_STAGING_DIR = False @@ -99,7 +99,7 @@ OBS_GRID_STAT_INPUT_DIR = {ENV[EVSINmrms]} OBS_GRID_STAT_INPUT_TEMPLATE = {ENV[OBS_INPUT_TEMPLATE]} -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/grid_stat +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/grid_stat GRID_STAT_OUTPUT_PREFIX = {MODEL}_{FCST_VAR1_NAME}_{ENV[DOMAIN]} diff --git a/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPPF_obsMRMS_REFC.conf b/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPPF_obsMRMS_REFC.conf index 851ad0091f..b14f870f07 100644 --- a/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPPF_obsMRMS_REFC.conf +++ b/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPPF_obsMRMS_REFC.conf @@ -2,8 +2,8 @@ [config] # Final conf file used by all processes -LOG_DIR = {OUTPUT_BASE}/logs -METPLUS_CONF = {TMP_DIR}/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf +LOG_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/logs +METPLUS_CONF = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/tmp/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf LOG_METPLUS = {LOG_DIR}/run_metplus.job{ENV[JOBNUM]}.gridstat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.log.{LOG_TIMESTAMP_TEMPLATE} SCRUB_STAGING_DIR = False @@ -132,7 +132,7 @@ OBS_GRID_STAT_INPUT_DIR = {ENV[EVSINmrms]} OBS_GRID_STAT_INPUT_TEMPLATE = {ENV[OBS_INPUT_TEMPLATE]} -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/grid_stat +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/grid_stat GRID_STAT_OUTPUT_PREFIX = {MODEL}_{ENV[PROD]}_{FCST_VAR1_NAME}_{ENV[DOMAIN]} diff --git a/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPPF_obsMRMS_RETOP.conf b/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPPF_obsMRMS_RETOP.conf index 39127df7c9..7c5f37546a 100644 --- a/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPPF_obsMRMS_RETOP.conf +++ b/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPPF_obsMRMS_RETOP.conf @@ -2,8 +2,8 @@ [config] # Final conf file used by all processes -LOG_DIR = {OUTPUT_BASE}/logs -METPLUS_CONF = {TMP_DIR}/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf +LOG_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/logs +METPLUS_CONF = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/tmp/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf LOG_METPLUS = {LOG_DIR}/run_metplus.job{ENV[JOBNUM]}.gridstat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.log.{LOG_TIMESTAMP_TEMPLATE} SCRUB_STAGING_DIR = False @@ -122,7 +122,7 @@ OBS_GRID_STAT_INPUT_DIR = {ENV[EVSINmrms]} OBS_GRID_STAT_INPUT_TEMPLATE = {ENV[OBS_INPUT_TEMPLATE]} -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/grid_stat +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/grid_stat GRID_STAT_OUTPUT_PREFIX = {MODEL}_{ENV[PROD]}_{FCST_VAR1_NAME}_{ENV[DOMAIN]} diff --git a/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPROB_obsMRMS_REFC.conf b/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPROB_obsMRMS_REFC.conf index 877b647db8..8f2d3a5230 100644 --- a/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPROB_obsMRMS_REFC.conf +++ b/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPROB_obsMRMS_REFC.conf @@ -2,8 +2,8 @@ [config] # Final conf file used by all processes -LOG_DIR = {OUTPUT_BASE}/logs -METPLUS_CONF = {TMP_DIR}/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf +LOG_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/logs +METPLUS_CONF = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/tmp/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf LOG_METPLUS = {LOG_DIR}/run_metplus.job{ENV[JOBNUM]}.gridstat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.log.{LOG_TIMESTAMP_TEMPLATE} SCRUB_STAGING_DIR = False @@ -127,7 +127,7 @@ OBS_GRID_STAT_INPUT_DIR = {ENV[EVSINmrms]} OBS_GRID_STAT_INPUT_TEMPLATE = {ENV[OBS_INPUT_TEMPLATE]} -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/grid_stat +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/grid_stat GRID_STAT_OUTPUT_PREFIX = {MODEL}_{FCST_VAR1_NAME}_{ENV[DOMAIN]} diff --git a/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPROB_obsMRMS_RETOP.conf b/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPROB_obsMRMS_RETOP.conf index 653951f82a..fa21717df9 100644 --- a/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPROB_obsMRMS_RETOP.conf +++ b/parm/metplus_config/stats/cam/radar/GridStat_fcstHREFPROB_obsMRMS_RETOP.conf @@ -2,8 +2,8 @@ [config] # Final conf file used by all processes -LOG_DIR = {OUTPUT_BASE}/logs -METPLUS_CONF = {TMP_DIR}/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf +LOG_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/logs +METPLUS_CONF = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/tmp/metplus_final.job{ENV[JOBNUM]}.GridStat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.conf LOG_METPLUS = {LOG_DIR}/run_metplus.job{ENV[JOBNUM]}.gridstat_{FCST_VAR1_NAME}_{ENV[DOMAIN]}.log.{LOG_TIMESTAMP_TEMPLATE} SCRUB_STAGING_DIR = False @@ -118,7 +118,7 @@ OBS_GRID_STAT_INPUT_DIR = {ENV[EVSINmrms]} OBS_GRID_STAT_INPUT_TEMPLATE = {ENV[OBS_INPUT_TEMPLATE]} -GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/grid_stat +GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/workdirs/job{ENV[JOBNUM]}/grid_stat GRID_STAT_OUTPUT_PREFIX = {MODEL}_{FCST_VAR1_NAME}_{ENV[DOMAIN]} diff --git a/parm/metplus_config/stats/cam/snowfall/GridStat_fcstCAM_obsNOHRSC.conf b/parm/metplus_config/stats/cam/snowfall/GridStat_fcstCAM_obsNOHRSC.conf index 1970b85e3d..ba4ac349a8 100755 --- a/parm/metplus_config/stats/cam/snowfall/GridStat_fcstCAM_obsNOHRSC.conf +++ b/parm/metplus_config/stats/cam/snowfall/GridStat_fcstCAM_obsNOHRSC.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/pcp_combine +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/pcp_combine OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/grid_stat CONFIG_DIR = {ENV[METPLUS_PATH]}/parm/met_config LOG_DIR = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/out/logs diff --git a/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_GatherByCycle.conf b/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_GatherByCycle.conf index d2e9c3f8b2..24309e5374 100755 --- a/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_GatherByCycle.conf +++ b/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_GatherByCycle.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/gather_small/stat_analysis +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/gather_small/stat_analysis OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/stat_analysis LOG_DIR = {ENV[MET_PLUS_OUT]}/out/logs diff --git a/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_obsNOHRSC_GatherByDay.conf b/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_obsNOHRSC_GatherByDay.conf index d645492897..3515c65504 100755 --- a/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_obsNOHRSC_GatherByDay.conf +++ b/parm/metplus_config/stats/cam/snowfall/StatAnalysis_fcstCAM_obsNOHRSC_GatherByDay.conf @@ -1,7 +1,7 @@ [dir] METPLUS_PARM_BASE = {METPLUS_BASE}/parm -INPUT_BASE = {ENV[MET_PLUS_OUT]}/{ENV[VERIF_TYPE]}/grid_stat +INPUT_BASE = {ENV[DATA]}/{ENV[VERIF_CASE]}/METplus_output/{ENV[VERIF_TYPE]}/grid_stat OUTPUT_BASE = {ENV[MET_PLUS_OUT]}/gather_small/stat_analysis LOG_DIR = {ENV[MET_PLUS_OUT]}/gather_small/out/logs diff --git a/scripts/plots/cam/exevs_cam_grid2obs_plots.sh b/scripts/plots/cam/exevs_cam_grid2obs_plots.sh index 4672268d3e..0e770f992e 100755 --- a/scripts/plots/cam/exevs_cam_grid2obs_plots.sh +++ b/scripts/plots/cam/exevs_cam_grid2obs_plots.sh @@ -40,6 +40,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All CAM grid2obs/plots Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/plotting_job_scripts/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/plotting_job_scripts/job* |wc -l) @@ -71,6 +75,12 @@ else set -x fi +# Copy Plots Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/out/workdirs/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. + export err=$?; err_chk +done + # Cat the plotting log files log_dir="$DATA/$VERIF_CASE/out/logs" if [ -d $log_dir ]; then @@ -88,7 +98,7 @@ if [ -d $log_dir ]; then fi # Tar and Copy output files to EVS COMOUT directory -find ${DATA}/${VERIF_CASE}/* -name "*.png" -type f -print | tar -cvf ${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.${EVAL_PERIOD}.v${VDATE}.tar --transform='s#.*/##' -T - +find ${DATA}/${VERIF_CASE}/* -name "*.png" -type f -not -path "*workdirs*" -print | tar -cvf ${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.${EVAL_PERIOD}.v${VDATE}.tar --transform='s#.*/##' -T - if [ $SENDCOM = YES ]; then FILE=${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.${EVAL_PERIOD}.v${VDATE}.tar diff --git a/scripts/plots/cam/exevs_cam_headline_plots.sh b/scripts/plots/cam/exevs_cam_headline_plots.sh index 563dd5903b..64f4d9012e 100755 --- a/scripts/plots/cam/exevs_cam_headline_plots.sh +++ b/scripts/plots/cam/exevs_cam_headline_plots.sh @@ -40,6 +40,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All CAM headline/plots Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/plotting_job_scripts/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/plotting_job_scripts/job* |wc -l) @@ -70,6 +74,12 @@ else done fi +# Copy Plots Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/out/workdirs/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. + export err=$?; err_chk +done + # Cat the plotting log files log_dir="$DATA/$VERIF_CASE/out/logs" if [ -d $log_dir ]; then @@ -87,7 +97,7 @@ if [ -d $log_dir ]; then fi # Tar and Copy output files to EVS COMOUT directory - find ${DATA}/${VERIF_CASE}/* -name "*.png" -type f -print | tar -cvf ${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.v${VDATE}.tar --transform='s#.*/##' -T - +find ${DATA}/${VERIF_CASE}/* -name "*.png" -type f -not -path "*workdirs*" -print | tar -cvf ${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.v${VDATE}.tar --transform='s#.*/##' -T - if [ $SENDCOM = YES ]; then FILE=${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.v${VDATE}.tar diff --git a/scripts/plots/cam/exevs_cam_precip_plots.sh b/scripts/plots/cam/exevs_cam_precip_plots.sh index 686d96cad0..7c56ccb2e4 100755 --- a/scripts/plots/cam/exevs_cam_precip_plots.sh +++ b/scripts/plots/cam/exevs_cam_precip_plots.sh @@ -40,6 +40,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All CAM precip/plots Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/plotting_job_scripts/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/plotting_job_scripts/job* |wc -l) @@ -71,6 +75,12 @@ else set -x fi +# Copy Plots Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/out/workdirs/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. + export err=$?; err_chk +done + # Cat the plotting log files log_dir="$DATA/$VERIF_CASE/out/logs" if [ -d $log_dir ]; then @@ -88,7 +98,7 @@ if [ -d $log_dir ]; then fi # Tar and Copy output files to EVS COMOUT directory -find ${DATA}/${VERIF_CASE}/* -type f \( -name "*.png" -o -name "*.gif" \) -print | tar -cvf ${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.${EVAL_PERIOD}.v${VDATE}.tar --transform='s#.*/##' -T - +find ${DATA}/${VERIF_CASE}/* -type f \( -name "*.png" -o -name "*.gif" \) -not -path "*workdirs*" -print | tar -cvf ${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.${EVAL_PERIOD}.v${VDATE}.tar --transform='s#.*/##' -T - if [ $SENDCOM = YES ]; then FILE=${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.${EVAL_PERIOD}.v${VDATE}.tar diff --git a/scripts/plots/cam/exevs_cam_snowfall_plots.sh b/scripts/plots/cam/exevs_cam_snowfall_plots.sh index bcd1e7b215..08836404d2 100755 --- a/scripts/plots/cam/exevs_cam_snowfall_plots.sh +++ b/scripts/plots/cam/exevs_cam_snowfall_plots.sh @@ -40,6 +40,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All CAM snowfall/plots Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/plotting_job_scripts/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/plotting_job_scripts/job* |wc -l) @@ -70,6 +74,12 @@ else done fi +# Copy Plots Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/out/workdirs/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. + export err=$?; err_chk +done + # Cat the plotting log files log_dir="$DATA/$VERIF_CASE/out/logs" if [ -d $log_dir ]; then @@ -87,7 +97,7 @@ if [ -d $log_dir ]; then fi # Tar and Copy output files to EVS COMOUT directory - find ${DATA}/${VERIF_CASE}/* -name "*.png" -type f -print | tar -cvf ${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.v${VDATE}.tar --transform='s#.*/##' -T - +find ${DATA}/${VERIF_CASE}/* -name "*.png" -type f -not -path "*workdirs*" -print | tar -cvf ${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.v${VDATE}.tar --transform='s#.*/##' -T - if [ $SENDCOM = YES ]; then FILE=${DATA}/${NET}.${STEP}.${COMPONENT}.${RUN}.${VERIF_CASE}.v${VDATE}.tar diff --git a/scripts/stats/cam/exevs_cam_radar_stats.sh b/scripts/stats/cam/exevs_cam_radar_stats.sh index e6f2ae7c93..8b44d33efe 100755 --- a/scripts/stats/cam/exevs_cam_radar_stats.sh +++ b/scripts/stats/cam/exevs_cam_radar_stats.sh @@ -72,6 +72,7 @@ for PROD in ${PRODS}; do for RADAR_FIELD in ${RADAR_FIELDS}; do echo "${USHevs}/${COMPONENT}/evs_cam_stats_radar.sh $DOMAIN $RADAR_FIELD $PROD $njob" >> $DATA/poescript + mkdir -p ${DATA}/workdirs/job${njob} njob=$((njob+1)) done @@ -106,6 +107,15 @@ else fi +################################################################### +# Copy child output to main output directory +################################################################### +for CHILD_DIR in ${DATA}/workdirs/*; do + cp -ru ${CHILD_DIR}/* ${DATA}/. + export err=$?; err_chk +done + + ################################################################### # Copy hourly output to $COMOUT ################################################################### diff --git a/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh b/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh index 9b5ab2a1ce..0946a61d8a 100755 --- a/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh @@ -77,6 +77,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW grid2obs/stats Reformat Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -110,6 +114,12 @@ else set -x fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Generate MET Data export job_type="generate" export njob=1 @@ -146,6 +156,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW grid2obs/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -179,6 +193,12 @@ else set -x fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for VERIF_TYPE in $VERIF_TYPES; do @@ -204,6 +224,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW grid2obs/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -237,6 +261,12 @@ else set -x fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 source $config @@ -259,6 +289,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW grid2obs/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -292,6 +326,12 @@ else set -x fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -326,6 +366,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All NAM Nest grid2obs/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -358,5 +402,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then done set -x fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_hireswarw_precip_stats.sh b/scripts/stats/cam/exevs_hireswarw_precip_stats.sh index 5333505fcd..678efe3643 100755 --- a/scripts/stats/cam/exevs_hireswarw_precip_stats.sh +++ b/scripts/stats/cam/exevs_hireswarw_precip_stats.sh @@ -82,6 +82,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run all HiRes Window ARW precip/stats Reformat jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -115,6 +119,12 @@ else set -x fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + NEST_LIST="conus ak" # Generate MET Data export job_type="generate" @@ -162,6 +172,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW precip/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -195,6 +209,12 @@ else set -x fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for NEST in $NEST_LIST; do @@ -216,6 +236,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW precip/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -249,6 +273,12 @@ else set -x fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 source $config @@ -268,6 +298,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW precip/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -301,6 +335,12 @@ else set -x fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -345,6 +385,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All NAM Nest precip/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -377,5 +421,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then done set -x fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh b/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh index 934e58a113..7a104987b1 100755 --- a/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh @@ -83,6 +83,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW snowfall/stats Reformat Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -114,6 +118,12 @@ else done fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Generate MET Data export job_type="generate" export njob=1 @@ -160,6 +170,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW snowfall/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -191,6 +205,12 @@ else done fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for NEST in $NEST_LIST; do @@ -212,6 +232,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW snowfall/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -243,6 +267,12 @@ else done fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 source $config @@ -261,6 +291,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW snowfall/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -292,6 +326,12 @@ else done fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -325,6 +365,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All HiRes Window ARW snowfall/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -355,5 +399,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then nc=$((nc+1)) done fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh b/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh index 691de37333..4d1840331d 100755 --- a/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh @@ -77,6 +77,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW2 grid2obs/stats Reformat Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -110,6 +114,12 @@ else set -x fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Generate MET Data export job_type="generate" export njob=1 @@ -146,6 +156,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW2 grid2obs/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -179,6 +193,12 @@ else set -x fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for VERIF_TYPE in $VERIF_TYPES; do @@ -204,6 +224,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW2 grid2obs/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -237,6 +261,12 @@ else set -x fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 export VERIF_TYPE=$VERIF_TYPE @@ -260,6 +290,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW2 grid2obs/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -293,6 +327,12 @@ else set -x fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -327,6 +367,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Reformat Gather 3 Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All NAM Nest grid2obs/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -359,5 +403,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then done set -x fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh b/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh index 7cff86cb24..15611e667e 100755 --- a/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh +++ b/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh @@ -82,6 +82,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run all HiRes Window ARW Member 2 precip/stats Reformat jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -115,6 +119,12 @@ else set -x fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + NEST_LIST="conus ak" # Generate MET Data export job_type="generate" @@ -162,6 +172,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW Member 2 precip/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -195,6 +209,12 @@ else set -x fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for NEST in $NEST_LIST; do @@ -216,6 +236,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW Member 2 precip/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -249,6 +273,12 @@ else set -x fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 source $config @@ -267,6 +297,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW Member 2 precip/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -300,6 +334,12 @@ else set -x fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -344,6 +384,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All NAM Nest precip/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -376,5 +420,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then done set -x fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh b/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh index c83caea4d2..271eb42ea0 100755 --- a/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh @@ -83,6 +83,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW2 snowfall/stats Reformat Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -114,6 +118,12 @@ else done fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Generate MET Data export job_type="generate" export njob=1 @@ -160,6 +170,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW 2 snowfall/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -191,6 +205,12 @@ else done fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for NEST in $NEST_LIST; do @@ -212,6 +232,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW 2 snowfall/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -243,6 +267,12 @@ else done fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 source $config @@ -261,6 +291,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window ARW 2 snowfall/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -292,6 +326,12 @@ else done fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -325,6 +365,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All HiRes Window ARW 2 snowfall/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -355,5 +399,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then nc=$((nc+1)) done fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh b/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh index 7180b2c07c..d82ffb6c66 100755 --- a/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh @@ -77,6 +77,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 grid2obs/stats Reformat Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -110,6 +114,12 @@ else set -x fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Generate MET Data export job_type="generate" export njob=1 @@ -146,6 +156,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 grid2obs/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -179,6 +193,12 @@ else set -x fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for VERIF_TYPE in $VERIF_TYPES; do @@ -204,6 +224,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 grid2obs/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -237,6 +261,12 @@ else set -x fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 export VERIF_TYPE=$VERIF_TYPE @@ -260,6 +290,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 grid2obs/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -293,6 +327,12 @@ else set -x fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -327,6 +367,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All NAM Nest grid2obs/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -359,5 +403,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then done set -x fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh b/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh index aa0590214f..8cac5b9916 100755 --- a/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh +++ b/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh @@ -82,6 +82,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run all HiRes Window FV3 Member precip/stats Reformat jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -115,6 +119,12 @@ else set -x fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + NEST_LIST="conus ak" # Generate MET Data export job_type="generate" @@ -162,6 +172,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 Member precip/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -195,6 +209,12 @@ else set -x fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for NEST in $NEST_LIST; do @@ -216,6 +236,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 Member precip/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -249,6 +273,12 @@ else set -x fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 source $config @@ -267,6 +297,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 Member precip/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -300,6 +334,12 @@ else set -x fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -344,6 +384,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All NAM Nest precip/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -376,5 +420,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then done set -x fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh b/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh index f151a8dbb6..055511fbfe 100755 --- a/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh @@ -83,6 +83,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 snowfall/stats Reformat Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -114,6 +118,12 @@ else done fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Generate MET Data export job_type="generate" export njob=1 @@ -160,6 +170,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 snowfall/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -191,6 +205,12 @@ else done fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for NEST in $NEST_LIST; do @@ -212,6 +232,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 snowfall/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -243,6 +267,12 @@ else done fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 source $config @@ -261,6 +291,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 snowfall/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -292,6 +326,12 @@ else done fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -325,6 +365,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All HiRes Window FV3 snowfall/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -355,5 +399,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then nc=$((nc+1)) done fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh b/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh index dbd7b723c7..6f625e729c 100755 --- a/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh @@ -77,6 +77,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HRRR grid2obs/stats Reformat Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -110,6 +114,12 @@ else set -x fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Generate MET Data export job_type="generate" export njob=1 @@ -146,6 +156,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HRRR grid2obs/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -179,6 +193,12 @@ else set -x fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for VERIF_TYPE in $VERIF_TYPES; do @@ -204,6 +224,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HRRR grid2obs/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -237,6 +261,12 @@ else set -x fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 export VERIF_TYPE=$VERIF_TYPE @@ -260,6 +290,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HRRR grid2obs/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -293,6 +327,12 @@ else set -x fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -327,6 +367,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All NAM Nest grid2obs/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -359,5 +403,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then done set -x fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_hrrr_precip_stats.sh b/scripts/stats/cam/exevs_hrrr_precip_stats.sh index 113d2f5133..0482a9b7d4 100755 --- a/scripts/stats/cam/exevs_hrrr_precip_stats.sh +++ b/scripts/stats/cam/exevs_hrrr_precip_stats.sh @@ -82,6 +82,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run all HRRR precip/stats Reformat jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -115,6 +119,12 @@ else set -x fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Generate MET Data export job_type="generate" export njob=1 @@ -161,6 +171,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HRRR precip/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -194,6 +208,12 @@ else set -x fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Create Gather Job Script export job_type="gather" export njob=1 @@ -216,6 +236,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HRRR precip/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -249,6 +273,12 @@ else set -x fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Create Gather 2 Job Script export job_type="gather2" export njob=1 @@ -268,6 +298,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HRRR precip/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -301,6 +335,12 @@ else set -x fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -345,6 +385,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All NAM Nest precip/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -377,5 +421,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then done set -x fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh b/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh index a6dda5d7ca..4b3316c999 100755 --- a/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh @@ -83,6 +83,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HRRR snowfall/stats Reformat Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -114,6 +118,12 @@ else done fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Generate MET Data export job_type="generate" export njob=1 @@ -160,6 +170,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HRRR snowfall/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -191,6 +205,12 @@ else done fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for NEST in $NEST_LIST; do @@ -212,6 +232,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HRRR snowfall/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -243,6 +267,12 @@ else done fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 source $config @@ -261,6 +291,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All HiRes Window FV3 snowfall/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -292,6 +326,12 @@ else done fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -325,6 +365,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All HiRes Window FV3 snowfall/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -355,5 +399,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then nc=$((nc+1)) done fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh b/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh index 046df92a1f..f7a56c1da9 100755 --- a/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh @@ -77,6 +77,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest grid2obs/stats Reformat Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -110,6 +114,12 @@ else set -x fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Generate MET Data export job_type="generate" export njob=1 @@ -146,6 +156,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest grid2obs/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -179,6 +193,12 @@ else set -x fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for VERIF_TYPE in $VERIF_TYPES; do @@ -204,6 +224,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest grid2obs/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -237,6 +261,12 @@ else set -x fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 export VERIF_TYPE=$VERIF_TYPE @@ -260,6 +290,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest grid2obs/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -293,6 +327,12 @@ else set -x fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -327,6 +367,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All NAM Nest grid2obs/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -359,5 +403,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then done set -x fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_namnest_precip_stats.sh b/scripts/stats/cam/exevs_namnest_precip_stats.sh index 4a4693056f..2e44aaaae0 100755 --- a/scripts/stats/cam/exevs_namnest_precip_stats.sh +++ b/scripts/stats/cam/exevs_namnest_precip_stats.sh @@ -83,6 +83,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest precip/stats Reformat Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -116,6 +120,12 @@ else set -x fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + NEST_LIST="conus ak" # Generate MET Data export job_type="generate" @@ -164,6 +174,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest precip/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -197,6 +211,12 @@ else set -x fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for NEST in $NEST_LIST; do @@ -218,6 +238,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest precip/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -251,6 +275,12 @@ else set -x fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 source $config @@ -269,6 +299,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest precip/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -302,6 +336,12 @@ else set -x fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -346,6 +386,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All NAM Nest precip/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -378,5 +422,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then done set -x fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/scripts/stats/cam/exevs_namnest_snowfall_stats.sh b/scripts/stats/cam/exevs_namnest_snowfall_stats.sh index 1f9dc1b4a3..35852b01ea 100755 --- a/scripts/stats/cam/exevs_namnest_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_namnest_snowfall_stats.sh @@ -83,6 +83,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Reformat Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest snowfall/stats Reformat Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -114,6 +118,12 @@ else done fi +# Copy Reformat Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Generate MET Data export job_type="generate" export njob=1 @@ -160,6 +170,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Generate Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest snowfall/stats Generate Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -191,6 +205,12 @@ else done fi +# Copy Generate Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather" export njob=1 for NEST in $NEST_LIST; do @@ -212,6 +232,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest snowfall/stats Gather Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -243,6 +267,12 @@ else done fi +# Copy Gather Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + export job_type="gather2" export njob=1 source $config @@ -261,6 +291,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Gather 2 Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run All NAM Nest snowfall/stats Gather 2 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -292,6 +326,12 @@ else done fi +# Copy Gather 2 Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk +done + # Copy files to desired location #all commands to copy output files into the correct EVS COMOUT directory if [ $SENDCOM = YES ]; then @@ -325,6 +365,10 @@ if [ "$vhr" -ge "$last_cyc" ]; then export err=$?; err_chk fi + # Create Gather 3 Working Directories + python $USHevs/cam/cam_create_child_workdirs.py + export err=$?; err_chk + # Run All NAM Nest snowfall/stats Gather 3 Jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/METplus_job_scripts/${job_type}/job* |wc -l) @@ -355,5 +399,11 @@ if [ "$vhr" -ge "$last_cyc" ]; then nc=$((nc+1)) done fi + + # Copy Gather 3 Output to Main Directory + for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do + cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + export err=$?; err_chk + done fi fi diff --git a/ush/cam/cam_create_child_workdirs.py b/ush/cam/cam_create_child_workdirs.py new file mode 100644 index 0000000000..1426de6093 --- /dev/null +++ b/ush/cam/cam_create_child_workdirs.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# ============================================================================= +# +# NAME: cam_create_child_workdirs.py +# CONTRIBUTOR(S): Marcel Caron, marcel.caron@noaa.gov, NOAA/NWS/NCEP/EMC-VPPPGB +# PURPOSE: Write output directories used by child processors (MPMD operations) +# +# ============================================================================= + +import os +import cam_util as cutil + +DATA = os.environ['DATA'] +VERIF_CASE = os.environ['VERIF_CASE'] +STEP = os.environ['STEP'] +if STEP == 'stats': + job_type = os.environ['job_type'] + +if STEP == 'prep': + print( + "WARNING: Child working directories are not being made for prep." + ) +elif STEP == 'stats': + jobdir = os.path.join( + DATA, VERIF_CASE, STEP, 'METplus_job_scripts', job_type + ) + outdir = os.path.join( + DATA, VERIF_CASE, 'METplus_output' + ) + workdirs = os.path.join( + outdir, 'workdirs', job_type + ) +elif STEP == 'plots': + jobdir = os.path.join( + DATA, VERIF_CASE, STEP, 'plotting_job_scripts' + ) + outdir = os.path.join( + DATA, VERIF_CASE, 'out' + ) + workdirs = os.path.join( + outdir, 'workdirs' + ) +else: + raise ValueError(f"Unrecognized STEP name: {STEP}") +if not os.path.exists(outdir): + raise OSError(f"Output directory does not exist: {outdir}.") +else: + if not os.path.exists(workdirs): + raise OSError( + f"Head working directory does not exist: {workdirs}." + ) + else: + wd = os.getcwd() + os.chdir(outdir) + job_scripts = [ + job_name for job_name in os.listdir(jobdir) + if job_name[:3] == 'job' + ] + for job_name in job_scripts: + workdir = os.path.join(workdirs, job_name) + if not os.path.exists(workdir): + os.makedirs(workdir) + cutil.run_shell_command([ + 'find', '.', '-type', 'd', '-not', '-path', + '\"*workdirs*\"', '-not', '-path', '\"*job*\"', '-exec', + 'mkdir', '-p', os.path.join(workdir,'{}'), '\\;' + ]) + if STEP == "stats": + print( + "Done making working directories for child processes " + + f"({job_type} jobs)." + ) + elif STEP == "plots": + print( + "Done making working directories for child processes." + ) + os.chdir(wd) + diff --git a/ush/cam/cam_create_output_dirs.py b/ush/cam/cam_create_output_dirs.py index 1ecc901293..d6ac03aefc 100644 --- a/ush/cam/cam_create_output_dirs.py +++ b/ush/cam/cam_create_output_dirs.py @@ -33,7 +33,6 @@ evs_ver = os.environ['evs_ver'] COMIN = os.environ['COMIN'] COMOUT = os.environ['COMOUT'] -COMOUTsmall = os.environ['COMOUTsmall'] DATA = os.environ['DATA'] NET = os.environ['NET'] RUN = os.environ['RUN'] @@ -104,6 +103,7 @@ all_eval_periods = cutil.get_all_eval_periods(graphics_hdl) COMOUTplots = os.environ['COMOUTplots'] if STEP == 'stats': + COMOUTsmall = os.environ['COMOUTsmall'] RESTART_DIR = os.environ['RESTART_DIR'] job_type = os.environ['job_type'] if STEP == 'plots': @@ -515,6 +515,9 @@ COMOUT, MODELNAME+'.'+vdate_dt.strftime('%Y%m%d') )) + working_dir_list.append(os.path.join( + DATA, VERIF_CASE, 'METplus_output', 'workdirs', job_type + )) elif STEP == 'plots': if VERIF_CASE == 'grid2obs': @@ -528,6 +531,9 @@ working_dir_list.append(os.path.join( working_output_base_dir, 'out' )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'workdirs' + )) working_dir_list.append(os.path.join( working_output_base_dir, 'out', 'logs' )) @@ -563,6 +569,9 @@ working_dir_list.append(os.path.join( working_output_base_dir, 'out' )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'workdirs' + )) working_dir_list.append(os.path.join( working_output_base_dir, 'out', 'logs' )) @@ -596,6 +605,9 @@ working_dir_list.append(os.path.join( working_output_base_dir, 'out' )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'workdirs' + )) working_dir_list.append(os.path.join( working_output_base_dir, 'out', 'logs' )) @@ -629,6 +641,9 @@ working_dir_list.append(os.path.join( working_output_base_dir, 'out' )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'out', 'workdirs' + )) working_dir_list.append(os.path.join( working_output_base_dir, 'out', 'logs' )) diff --git a/ush/cam/cam_plots_grid2obs_create_job_script.py b/ush/cam/cam_plots_grid2obs_create_job_script.py index 17eef7101d..533e23b9e1 100644 --- a/ush/cam/cam_plots_grid2obs_create_job_script.py +++ b/ush/cam/cam_plots_grid2obs_create_job_script.py @@ -75,7 +75,7 @@ 'USH_DIR': USH_DIR, 'FIXevs': FIXevs, 'PRUNE_DIR': PRUNE_DIR, - 'SAVE_DIR': SAVE_DIR, + 'SAVE_DIR': SAVE_DIR.format(njob=njob), 'RESTART_DIR': RESTART_DIR, 'STAT_OUTPUT_BASE_DIR': STAT_OUTPUT_BASE_DIR, 'STAT_OUTPUT_BASE_TEMPLATE': STAT_OUTPUT_BASE_TEMPLATE, diff --git a/ush/cam/cam_plots_headline_create_job_script.py b/ush/cam/cam_plots_headline_create_job_script.py index 1b2bbe623e..de4fb14a9a 100644 --- a/ush/cam/cam_plots_headline_create_job_script.py +++ b/ush/cam/cam_plots_headline_create_job_script.py @@ -75,7 +75,7 @@ 'USH_DIR': USH_DIR, 'FIXevs': FIXevs, 'PRUNE_DIR': PRUNE_DIR, - 'SAVE_DIR': SAVE_DIR, + 'SAVE_DIR': SAVE_DIR.format(njob=njob), 'RESTART_DIR': RESTART_DIR, 'STAT_OUTPUT_BASE_DIR': STAT_OUTPUT_BASE_DIR, 'STAT_OUTPUT_BASE_TEMPLATE': STAT_OUTPUT_BASE_TEMPLATE, diff --git a/ush/cam/cam_plots_precip_create_job_script.py b/ush/cam/cam_plots_precip_create_job_script.py index 258bc7f6ef..f50ebb9f6f 100644 --- a/ush/cam/cam_plots_precip_create_job_script.py +++ b/ush/cam/cam_plots_precip_create_job_script.py @@ -75,7 +75,7 @@ 'USH_DIR': USH_DIR, 'FIXevs': FIXevs, 'PRUNE_DIR': PRUNE_DIR, - 'SAVE_DIR': SAVE_DIR, + 'SAVE_DIR': SAVE_DIR.format(njob=njob), 'RESTART_DIR': RESTART_DIR, 'STAT_OUTPUT_BASE_DIR': STAT_OUTPUT_BASE_DIR, 'STAT_OUTPUT_BASE_TEMPLATE': STAT_OUTPUT_BASE_TEMPLATE, diff --git a/ush/cam/cam_plots_snowfall_create_job_script.py b/ush/cam/cam_plots_snowfall_create_job_script.py index 04c132473f..6ed6d6a3a1 100644 --- a/ush/cam/cam_plots_snowfall_create_job_script.py +++ b/ush/cam/cam_plots_snowfall_create_job_script.py @@ -75,7 +75,7 @@ 'USH_DIR': USH_DIR, 'FIXevs': FIXevs, 'PRUNE_DIR': PRUNE_DIR, - 'SAVE_DIR': SAVE_DIR, + 'SAVE_DIR': SAVE_DIR.format(njob=njob), 'RESTART_DIR': RESTART_DIR, 'STAT_OUTPUT_BASE_DIR': STAT_OUTPUT_BASE_DIR, 'STAT_OUTPUT_BASE_TEMPLATE': STAT_OUTPUT_BASE_TEMPLATE, diff --git a/ush/cam/cam_production_restart.py b/ush/cam/cam_production_restart.py index d183b3af2d..db1871b5ba 100644 --- a/ush/cam/cam_production_restart.py +++ b/ush/cam/cam_production_restart.py @@ -43,7 +43,7 @@ elif STEP == 'plots': COMOUTplots = os.environ['COMOUTplots'] RESTART_DIR = os.environ['RESTART_DIR'] - SAVE_DIR = os.environ['SAVE_DIR'] + working_dir = os.path.join(DATA, VERIF_CASE, 'out') if VERIF_CASE == "grid2obs": completed_jobs_file = os.path.join( RESTART_DIR, @@ -59,7 +59,7 @@ if os.path.exists(completed_jobs_file): if os.stat(completed_jobs_file).st_size != 0: cutil.run_shell_command( - ['cp', '-rpv', os.path.join(RESTART_DIR,'*'), SAVE_DIR] + ['cp', '-rpv', os.path.join(RESTART_DIR,'*'), working_dir] ) diff --git a/ush/cam/cam_stats_grid2obs_create_job_script.py b/ush/cam/cam_stats_grid2obs_create_job_script.py index dd0980d5b4..683deb6ada 100644 --- a/ush/cam/cam_stats_grid2obs_create_job_script.py +++ b/ush/cam/cam_stats_grid2obs_create_job_script.py @@ -35,7 +35,6 @@ RESTART_DIR = os.environ['RESTART_DIR'] VDATE = os.environ['VDATE'] MET_PLUS_CONF = os.environ['MET_PLUS_CONF'] -MET_PLUS_OUT = os.environ['MET_PLUS_OUT'] MET_CONFIG_OVERRIDES = os.environ['MET_CONFIG_OVERRIDES'] metplus_launcher = 'run_metplus.py' machine_conf = os.path.join( @@ -54,6 +53,9 @@ MIN_IHOUR = os.environ['MIN_IHOUR'] COMINobs = os.environ['COMINobs'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) USHevs = os.environ['USHevs'] SKIP_IF_OUTPUT_EXISTS = os.environ['SKIP_IF_OUTPUT_EXISTS'] if NEST == 'spc_otlk': @@ -83,6 +85,9 @@ if NEST not in ['firewx', 'spc_otlk']: MASK_POLY_LIST = os.environ['MASK_POLY_LIST'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) GRID = os.environ['GRID'] USHevs = os.environ['USHevs'] if NEST == 'spc_otlk': @@ -96,10 +101,19 @@ elif job_type == 'gather': VERIF_TYPE = os.environ['VERIF_TYPE'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) elif job_type == 'gather2': njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) elif job_type == 'gather3': njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) COMOUTsmall = os.environ['COMOUTsmall'] # Get expanded details from variable name @@ -1895,7 +1909,9 @@ + ')\"' ) else: - if glob.glob(os.path.join(MET_PLUS_OUT,VERIF_TYPE,'point_stat',f'{MODELNAME}.{VDATE}','*stat')): + if glob.glob(os.path.join( + DATA,VERIF_CASE,'METplus_output',VERIF_TYPE,'point_stat', + f'{MODELNAME}.{VDATE}','*stat')): job_cmd_list.append( f'{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' @@ -1982,7 +1998,9 @@ + ')\"' ) else: - if glob.glob(os.path.join(MET_PLUS_OUT,'gather_small','stat_analysis',f'{MODELNAME}.{VDATE}','*stat')): + if glob.glob(os.path.join( + DATA,VERIF_CASE,'METplus_output','gather_small', + 'stat_analysis',f'{MODELNAME}.{VDATE}','*stat')): job_cmd_list.append( f'{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' diff --git a/ush/cam/cam_stats_grid2obs_create_merged_ptype.py b/ush/cam/cam_stats_grid2obs_create_merged_ptype.py index 76eef2731f..0e8fd82a3e 100644 --- a/ush/cam/cam_stats_grid2obs_create_merged_ptype.py +++ b/ush/cam/cam_stats_grid2obs_create_merged_ptype.py @@ -33,6 +33,8 @@ VHOUR= os.environ['VHOUR'] fhr = os.environ['FHR'] fhr = int(fhr) +job_type = os.environ['job_type'] +njob = os.environ['njob'] # Create merged ptype data skip_if_output_exists = True @@ -54,7 +56,8 @@ + f"process.") else: regrid_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output', VERIF_TYPE, 'regrid_data_plane', f'{MODELNAME}.{VDATE}' + DATA, VERIF_CASE, 'METplus_output', 'workdirs', job_type, f'job{njob}', + VERIF_TYPE, 'regrid_data_plane', f'{MODELNAME}.{VDATE}' ) regrid_fname = (f'regrid_data_plane_{MODELNAME}_t{VHOUR}z_{VERIF_TYPE}_{NEST}_' + f'{job_name}_fhr{str(fhr).zfill(2)}.nc') diff --git a/ush/cam/cam_stats_precip_create_job_script.py b/ush/cam/cam_stats_precip_create_job_script.py index 85d7869868..b00b135fc9 100644 --- a/ush/cam/cam_stats_precip_create_job_script.py +++ b/ush/cam/cam_stats_precip_create_job_script.py @@ -34,7 +34,6 @@ RESTART_DIR = os.environ['RESTART_DIR'] VDATE = os.environ['VDATE'] MET_PLUS_CONF = os.environ['MET_PLUS_CONF'] -MET_PLUS_OUT = os.environ['MET_PLUS_OUT'] MET_CONFIG_OVERRIDES = os.environ['MET_CONFIG_OVERRIDES'] machine_conf = os.path.join( os.environ['PARMevs'], 'metplus_config', 'machine.conf' @@ -54,6 +53,9 @@ OBS_LEV = os.environ['OBS_LEV'] MODEL_INPUT_TEMPLATE = os.environ['MODEL_INPUT_TEMPLATE'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) BUCKET_INTERVAL = os.environ['BUCKET_INTERVAL'] elif job_type == 'generate': VERIF_TYPE = os.environ['VERIF_TYPE'] @@ -73,14 +75,26 @@ MODEL_INPUT_TEMPLATE = os.environ['MODEL_INPUT_TEMPLATE'] MASK_POLY_LIST = os.environ['MASK_POLY_LIST'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) elif job_type == 'gather': VERIF_TYPE = os.environ['VERIF_TYPE'] NEST = os.environ['NEST'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) elif job_type == 'gather2': njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) elif job_type == 'gather3': njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) COMOUTsmall = os.environ['COMOUTsmall'] if VERIF_CASE == 'precip': if job_type == 'reformat': @@ -464,7 +478,9 @@ + ')\"' ) else: - if glob.glob(os.path.join(MET_PLUS_OUT,VERIF_TYPE,'grid_stat',f'{MODELNAME}.{VDATE}','*stat')): + if glob.glob(os.path.join( + DATA,VERIF_CASE,'METplus_output',VERIF_TYPE, + 'grid_stat',f'{MODELNAME}.{VDATE}','*stat')): job_cmd_list.append( f'{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' @@ -551,7 +567,9 @@ + ')\"' ) else: - if glob.glob(os.path.join(MET_PLUS_OUT,'gather_small','stat_analysis',f'{MODELNAME}.{VDATE}','*stat')): + if glob.glob(os.path.join( + DATA,VERIF_CASE,'METplus_output','gather_small', + 'stat_analysis',f'{MODELNAME}.{VDATE}','*stat')): job_cmd_list.append( f'{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' diff --git a/ush/cam/cam_stats_snowfall_create_job_script.py b/ush/cam/cam_stats_snowfall_create_job_script.py index a2e21da4e3..30182bc6b0 100644 --- a/ush/cam/cam_stats_snowfall_create_job_script.py +++ b/ush/cam/cam_stats_snowfall_create_job_script.py @@ -35,7 +35,6 @@ RESTART_DIR = os.environ['RESTART_DIR'] VDATE = os.environ['VDATE'] MET_PLUS_CONF = os.environ['MET_PLUS_CONF'] -MET_PLUS_OUT = os.environ['MET_PLUS_OUT'] MET_CONFIG_OVERRIDES = os.environ['MET_CONFIG_OVERRIDES'] machine_conf = os.path.join( os.environ['PARMevs'], 'metplus_config', 'machine.conf' @@ -54,6 +53,9 @@ COMINfcst = os.environ['COMINfcst'] MODEL_INPUT_TEMPLATE = os.environ['MODEL_INPUT_TEMPLATE'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) BUCKET_INTERVAL = os.environ['BUCKET_INTERVAL'] elif job_type == 'generate': VERIF_TYPE = os.environ['VERIF_TYPE'] @@ -69,15 +71,27 @@ MASK_POLY_LIST = os.environ['MASK_POLY_LIST'] OBSNAME = os.environ['OBSNAME'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) elif job_type == 'gather': VERIF_TYPE = os.environ['VERIF_TYPE'] NEST = os.environ['NEST'] OBSNAME = os.environ['OBSNAME'] njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) elif job_type == 'gather2': njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) elif job_type == 'gather3': njob = os.environ['njob'] + MET_PLUS_OUT = os.path.join( + os.environ['MET_PLUS_OUT'], 'workdirs', job_type, f'job{njob}' + ) COMOUTsmall = os.environ['COMOUTsmall'] if VERIF_CASE == 'snowfall': if job_type == 'reformat': @@ -519,7 +533,9 @@ + ')\"' ) else: - if glob.glob(os.path.join(MET_PLUS_OUT,VERIF_TYPE,'grid_stat',f'{MODELNAME}.{VDATE}','*stat')): + if glob.glob(os.path.join( + DATA,VERIF_CASE,'METplus_output',VERIF_TYPE, + 'grid_stat',f'{MODELNAME}.{VDATE}','*stat')): job_cmd_list.append( f'{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' @@ -606,7 +622,9 @@ + ')\"' ) else: - if glob.glob(os.path.join(MET_PLUS_OUT,'gather_small','stat_analysis',f'{MODELNAME}.{VDATE}','*stat')): + if glob.glob(os.path.join( + DATA,VERIF_CASE,'METplus_output','gather_small', + 'stat_analysis',f'{MODELNAME}.{VDATE}','*stat')): job_cmd_list.append( f'{metplus_launcher} -c {machine_conf} ' + f'-c {MET_PLUS_CONF}/' From 9f84f4e2486561f0b9e5c359c02c1197d0b6efdd Mon Sep 17 00:00:00 2001 From: marcel caron Date: Thu, 24 Oct 2024 20:22:28 +0000 Subject: [PATCH 03/25] Bugzilla 1547: Run child processes in separate working directories (severe and radar plots jobs) --- scripts/plots/cam/exevs_cam_radar_plots.sh | 33 +++++++++------- scripts/plots/cam/exevs_cam_severe_plots.sh | 43 ++++++++++++--------- ush/cam/evs_cam_plots_radar.sh | 2 + ush/cam/evs_cam_plots_severe.sh | 2 + 4 files changed, 49 insertions(+), 31 deletions(-) diff --git a/scripts/plots/cam/exevs_cam_radar_plots.sh b/scripts/plots/cam/exevs_cam_radar_plots.sh index aba0b023e0..b42d8ea5fa 100755 --- a/scripts/plots/cam/exevs_cam_radar_plots.sh +++ b/scripts/plots/cam/exevs_cam_radar_plots.sh @@ -50,12 +50,9 @@ export PRUNE_DIR=${DATA}/data export STAT_OUTPUT_BASE_DIR=${DATA}/stat_archive export STAT_OUTPUT_BASE_TEMPLATE="{MODEL}.{valid?fmt=%Y%m%d}/${NET}.stats.{MODEL}.${RUN}.${VERIF_CASE}.v{valid?fmt=%Y%m%d}.stat" -export SAVE_DIR=${DATA}/out -export LOG_DIR=${SAVE_DIR}/logs -export OUTPUT_DIR=${SAVE_DIR}/${VERIF_CASE}/${eval_period} +export OUTPUT_DIR=${DATA}/out/${VERIF_CASE}/${eval_period} export IMG_HEADER=${NET}.${COMPONENT} -export LOG_TEMPLATE="${LOG_DIR}/EVS_verif_plotting_job{njob}_$($NDATE)_$$.out" export LOG_LEVEL="DEBUG" export PYTHONDONTWRITEBYTECODE=1 @@ -75,8 +72,8 @@ export MET_VERSION="${MET_VERSION%.}" # Create working directories mkdir -p ${PRUNE_DIR} mkdir -p ${STAT_OUTPUT_BASE_DIR} -mkdir -p ${LOG_DIR} mkdir -p ${OUTPUT_DIR} +mkdir -p ${DATA}/out/logs # main log output dir model_list="hrrr namnest hireswarw hireswarwmem2 hireswfv3 href_pmmn" @@ -92,7 +89,7 @@ for model in ${model_list}; do stat_file=evs.stats.${model}.${RUN}.${VERIF_CASE}.v${day}.stat dest=${STAT_OUTPUT_BASE_DIR}/${model}.${day}/${stat_file} - if [ ${model:0:4} = href ]; then + if [ "${model:0:4}" = "href" ]; then origin=${COMIN}/stats/${COMPONENT}/${model:0:4}.${day}/${stat_file} else origin=${COMIN}/stats/${COMPONENT}/${model}.${day}/${stat_file} @@ -109,9 +106,9 @@ done -if [ $LINE_TYPE = nbrcnt ]; then +if [ "$LINE_TYPE" = "nbrcnt" ]; then PLOT_TYPES="lead_average threshold_average" -elif [ $LINE_TYPE = nbrctc ]; then +elif [ "$LINE_TYPE" = "nbrctc" ]; then PLOT_TYPES="lead_average performance_diagram threshold_average" fi @@ -125,9 +122,9 @@ for PLOT_TYPE in ${PLOT_TYPES}; do for DOMAIN in ${DOMAINS}; do # Set list of fields based on domain - if [ $DOMAIN = conus ]; then + if [ "$DOMAIN" = "conus" ]; then RADAR_FIELDS="REFC RETOP" - elif [ $DOMAIN = alaska ]; then + elif [ "$DOMAIN" = "alaska" ]; then RADAR_FIELDS="REFC" fi @@ -138,6 +135,7 @@ for PLOT_TYPE in ${PLOT_TYPES}; do for FCST_INIT_HOUR in ${FCST_INIT_HOURS}; do echo "${USHevs}/${COMPONENT}/evs_cam_plots_radar.sh $PLOT_TYPE $DOMAIN $RADAR_FIELD $LINE_TYPE $FCST_INIT_HOUR $njob" >> $DATA/poescript + mkdir -p ${DATA}/out/workdirs/job${njob}/logs njob=$((njob+1)) done @@ -161,7 +159,7 @@ export MP_CMDFILE=${DATA}/poescript export USE_CFP=YES -if [ $USE_CFP = YES ]; then +if [ "$USE_CFP" = "YES" ]; then echo "running cfp" mpiexec -np $nproc --cpu-bind verbose,core cfp ${MP_CMDFILE} @@ -177,6 +175,15 @@ else fi +################################################################### +# Copy Plots Output to Main Directory +################################################################### +for CHILD_DIR in ${DATA}/out/workdirs/*; do + cp -ru $CHILD_DIR/* ${DATA}/out/. + export err=$?; err_chk +done + + ################################################################### # Copy output to $COMOUT ################################################################### @@ -189,13 +196,13 @@ if [ "$(ls -A $OUTPUT_DIR)" ]; then tar -cvf ${tarfile} ./*.png fi -if [ $SENDCOM = YES ]; then +if [ "$SENDCOM" = "YES" ]; then mkdir -p $COMOUT/${RUN}.${VDATE} if [ -s $tarfile ]; then cp -v $tarfile $COMOUT/${RUN}.${VDATE}/ - if [ $SENDDBN = YES ]; then + if [ "$SENDDBN" = "YES" ]; then $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUT/${RUN}.${VDATE}/${tarfile} fi else diff --git a/scripts/plots/cam/exevs_cam_severe_plots.sh b/scripts/plots/cam/exevs_cam_severe_plots.sh index 8be9110f9e..d2b7665deb 100755 --- a/scripts/plots/cam/exevs_cam_severe_plots.sh +++ b/scripts/plots/cam/exevs_cam_severe_plots.sh @@ -45,12 +45,9 @@ export PRUNE_DIR=${DATA}/data export STAT_OUTPUT_BASE_DIR=${DATA}/stat_archive export STAT_OUTPUT_BASE_TEMPLATE="{MODEL}.{valid?fmt=%Y%m%d}/${NET}.stats.{MODEL}.${RUN}.${VERIF_CASE}.v{valid?fmt=%Y%m%d}.stat" -export SAVE_DIR=${DATA}/out -export LOG_DIR=${SAVE_DIR}/logs -export OUTPUT_DIR=${SAVE_DIR}/${VERIF_CASE}/${eval_period} +export OUTPUT_DIR=${DATA}/out/${VERIF_CASE}/${eval_period} export IMG_HEADER=${NET}.${COMPONENT} -export LOG_TEMPLATE="${LOG_DIR}/EVS_verif_plotting_job{njob}_$($NDATE)_$$.out" export LOG_LEVEL="DEBUG" export PYTHONDONTWRITEBYTECODE=1 @@ -71,8 +68,8 @@ export MET_VERSION="${MET_VERSION%.}" # Create working directories mkdir -p ${PRUNE_DIR} mkdir -p ${STAT_OUTPUT_BASE_DIR} -mkdir -p ${LOG_DIR} mkdir -p ${OUTPUT_DIR} +mkdir -p ${DATA}/out/logs # main log output dir model_list="hrrr namnest hireswarw hireswarwmem2 hireswfv3 href" @@ -88,7 +85,7 @@ for model in ${model_list}; do stat_file=evs.stats.${model}.${RUN}.${VERIF_CASE}.v${day}.stat dest=${STAT_OUTPUT_BASE_DIR}/${model}.${day}/${stat_file} - if [ ${model:0:4} = href ]; then + if [ "${model:0:4}" = "href" ]; then origin=${COMIN}/stats/${COMPONENT}/${model:0:4}.${day}/${stat_file} else origin=${COMIN}/stats/${COMPONENT}/${model}.${day}/${stat_file} @@ -109,11 +106,11 @@ done DOMAINS="conus" -if [ $LINE_TYPE = nbrcnt ]; then +if [ "$LINE_TYPE" = "nbrcnt" ]; then PLOT_TYPES="lead_average threshold_average" -elif [ $LINE_TYPE = nbrctc ]; then +elif [ "$LINE_TYPE" = "nbrctc" ]; then PLOT_TYPES="lead_average performance_diagram threshold_average" -elif [ $LINE_TYPE = pstd ]; then +elif [ "$LINE_TYPE" = "pstd" ]; then PLOT_TYPES="lead_average threshold_average" fi @@ -123,7 +120,7 @@ njob=0 #Loop over plot types for PLOT_TYPE in ${PLOT_TYPES}; do - if [ $PLOT_TYPE = lead_average ]; then + if [ "$PLOT_TYPE" = "lead_average" ]; then FCST_INIT_HOURS="0,6,12,18" else FCST_INIT_HOURS="0 6 12 18" @@ -135,16 +132,16 @@ for PLOT_TYPE in ${PLOT_TYPES}; do # Loop over forecast initializations for FCST_INIT_HOUR in ${FCST_INIT_HOURS}; do - if [ $FCST_INIT_HOUR = "0,6,12,18" ]; then + if [ "$FCST_INIT_HOUR" = "0,6,12,18" ]; then # export FCST_LEADs="24,30,36,42,48,54,60" export FCST_LEADs="24,36,48,60" - elif [ $FCST_INIT_HOUR = 0 ]; then + elif [ "$FCST_INIT_HOUR" = "0" ]; then export FCST_LEADs="36 60" - elif [ $FCST_INIT_HOUR = 6 ]; then + elif [ "$FCST_INIT_HOUR" = "6" ]; then export FCST_LEADs="30 54" - elif [ $FCST_INIT_HOUR = 12 ]; then + elif [ "$FCST_INIT_HOUR" = "12" ]; then export FCST_LEADs="24 48" - elif [ $FCST_INIT_HOUR = 18 ]; then + elif [ "$FCST_INIT_HOUR" = "18" ]; then export FCST_LEADs="42" fi @@ -152,6 +149,7 @@ for PLOT_TYPE in ${PLOT_TYPES}; do for FCST_LEAD in ${FCST_LEADs}; do echo "${USHevs}/${COMPONENT}/evs_cam_plots_severe.sh $PLOT_TYPE $DOMAIN $LINE_TYPE $FCST_INIT_HOUR $FCST_LEAD $njob" >> $DATA/poescript + mkdir -p ${DATA}/out/workdirs/job${njob}/logs njob=$((njob+1)) @@ -175,7 +173,7 @@ export MP_CMDFILE=${DATA}/poescript export USE_CFP=NO -if [ $USE_CFP = YES ]; then +if [ "$USE_CFP" = "YES" ]; then echo "running cfp" mpiexec -np $nproc --cpu-bind verbose,core cfp ${MP_CMDFILE} @@ -191,6 +189,15 @@ else fi +################################################################### +# Copy Plots Output to Main Directory +################################################################### +for CHILD_DIR in ${DATA}/out/workdirs/*; do + cp -ru $CHILD_DIR/* ${DATA}/out/. + export err=$?; err_chk +done + + ################################################################### # Copy output to $COMOUT ################################################################### @@ -203,13 +210,13 @@ if [ "$(ls -A $OUTPUT_DIR)" ]; then tar -cvf ${tarfile} ./*.png fi -if [ $SENDCOM = YES ]; then +if [ "$SENDCOM" = "YES" ]; then mkdir -p $COMOUT/${RUN}.${VDATE} if [ -s $tarfile ]; then cp -v $tarfile $COMOUT/${RUN}.${VDATE}/ - if [ $SENDDBN = YES ]; then + if [ "$SENDDBN" = "YES" ]; then $DBNROOT/bin/dbn_alert MODEL EVS_RZDM $job $COMOUT/${RUN}.${VDATE}/${tarfile} fi else diff --git a/ush/cam/evs_cam_plots_radar.sh b/ush/cam/evs_cam_plots_radar.sh index ed5a6abacf..a311b2f556 100755 --- a/ush/cam/evs_cam_plots_radar.sh +++ b/ush/cam/evs_cam_plots_radar.sh @@ -18,6 +18,8 @@ export FCST_INIT_HOUR=$5 export JOBNUM=$6 export job_name="job${JOBNUM}" +export SAVE_DIR=${DATA}/out/workdirs/job${JOBNUM} +export LOG_DIR=${SAVE_DIR}/logs export LOG_TEMPLATE="${LOG_DIR}/EVS_verif_plotting_job${JOBNUM}_$($NDATE)_$$.out" diff --git a/ush/cam/evs_cam_plots_severe.sh b/ush/cam/evs_cam_plots_severe.sh index 6b4214ac74..1f9788dc6e 100755 --- a/ush/cam/evs_cam_plots_severe.sh +++ b/ush/cam/evs_cam_plots_severe.sh @@ -18,6 +18,8 @@ export FCST_LEAD=$5 export JOBNUM=$6 export job_name="job${JOBNUM}" +export SAVE_DIR=${DATA}/out/workdirs/job${JOBNUM} +export LOG_DIR=${SAVE_DIR}/logs export LOG_TEMPLATE="${LOG_DIR}/EVS_verif_plotting_job${JOBNUM}_$($NDATE)_$$.out" From 018e3486f9f1adbaf1ab38df602e91b559bb0762 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 25 Oct 2024 15:43:37 +0000 Subject: [PATCH 04/25] Fix ASNOW verification + adjust snowfall graphics --- ...fig.evs.prod.stats.cam.atmos.snowfall.hrrr | 4 +- ush/cam/cam_plots_snowfall_graphx_defs.py | 275 +++++++++++++++++- ush/cam/cam_stats_snowfall_var_defs.py | 70 ----- ush/cam/lead_average.py | 7 +- ush/cam/performance_diagram.py | 7 +- ush/cam/roc_curve.py | 3 +- ush/cam/settings.py | 68 ++++- ush/cam/stat_by_level.py | 7 +- ush/cam/threshold_average.py | 7 +- ush/cam/time_series.py | 7 +- ush/cam/valid_hour_average.py | 7 +- 11 files changed, 356 insertions(+), 106 deletions(-) diff --git a/parm/evs_config/cam/config.evs.prod.stats.cam.atmos.snowfall.hrrr b/parm/evs_config/cam/config.evs.prod.stats.cam.atmos.snowfall.hrrr index efb315e672..bb9c93bcad 100644 --- a/parm/evs_config/cam/config.evs.prod.stats.cam.atmos.snowfall.hrrr +++ b/parm/evs_config/cam/config.evs.prod.stats.cam.atmos.snowfall.hrrr @@ -71,7 +71,7 @@ if [ "$BOOL_NBRHD" = True ]; then export COMINobs=${DCOMINsnow} export OBSNAME=${VERIF_TYPE} if [ "$OBSNAME" = "nohrsc" ]; then - export VAR_NAME_LIST="WEASD ASNOW" + export VAR_NAME_LIST="WEASD SNOD ASNOW" else err_exit "The provided OBSNAME, $OBSNAME, is not supported for $MODELNAME" fi @@ -92,7 +92,7 @@ else export COMINobs=${DCOMINsnow} export OBSNAME=${VERIF_TYPE} if [ "$OBSNAME" = "nohrsc" ]; then - export VAR_NAME_LIST="WEASD ASNOW" + export VAR_NAME_LIST="WEASD SNOD ASNOW" else err_exit "The provided OBSNAME, $OBSNAME, is not supported for $MODELNAME" fi diff --git a/ush/cam/cam_plots_snowfall_graphx_defs.py b/ush/cam/cam_plots_snowfall_graphx_defs.py index 6b49f21e59..966f4c342f 100755 --- a/ush/cam/cam_plots_snowfall_graphx_defs.py +++ b/ush/cam/cam_plots_snowfall_graphx_defs.py @@ -28,7 +28,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['30,36,42,48'], 'FCST_LEVEL':'Z0', 'OBS_LEVEL':'A06', 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], @@ -56,7 +56,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['30,36,42,48'], 'FCST_LEVEL':'Z0', 'OBS_LEVEL':'A06', 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], @@ -95,7 +95,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['30,36,42,48'], 'FCST_LEVEL':'Z0', 'OBS_LEVEL':'A06', 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], @@ -132,6 +132,19 @@ 'EVAL_PERIODS':['last31days','last90days'], 'VARIABLES':{ 'ctc':{ + 'WEASD_06':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['42,48,54,60'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A06', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'WEASD_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -147,6 +160,19 @@ }, }, 'nbrcnt':{ + 'WEASD_06':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['42,48,54,60'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A06', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'WEASD_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -173,6 +199,19 @@ 'EVAL_PERIODS':['last31days','last90days'], 'VARIABLES':{ 'ctc':{ + 'WEASD_06':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['42,48,54,60'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A06', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'WEASD_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -190,7 +229,7 @@ } }, }, - 'init_group1, fcst_group1, namnest, hrrr':{ + 'init_group1, fcst_group1, var_group1, namnest, hrrr':{ 'threshold_average':{ 'DATE_TYPE':'INIT', 'VALID_BEG':'', @@ -205,7 +244,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['30,36,42,48'], 'FCST_LEVEL':'Z0', 'OBS_LEVEL':'A06', 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], @@ -233,7 +272,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['30,36,42,48'], 'FCST_LEVEL':'Z0', 'OBS_LEVEL':'A06', 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], @@ -272,7 +311,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['30,36,42,48'], 'FCST_LEVEL':'Z0', 'OBS_LEVEL':'A06', 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], @@ -298,7 +337,7 @@ } }, }, - 'init_group1, fcst_group2, namnest':{ + 'init_group1, fcst_group1, var_group2, hrrr':{ 'threshold_average':{ 'DATE_TYPE':'INIT', 'VALID_BEG':'', @@ -309,6 +348,127 @@ 'EVAL_PERIODS':['last31days','last90days'], 'VARIABLES':{ 'ctc':{ + 'ASNOW_06':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['30,36,42,48'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A06', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, + 'ASNOW_24':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['24','36','48'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A24', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, + }, + 'nbrcnt':{ + 'ASNOW_06':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['30,36,42,48'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A06', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, + 'ASNOW_24':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['24','36','48'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A24', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, + }, + } + }, + 'performance_diagram':{ + 'DATE_TYPE':'INIT', + 'VALID_BEG':'', + 'VALID_END':'', + 'INIT_BEG':'', + 'INIT_END':'', + 'VX_MASK_LIST':'CONUS,CONUS_East,CONUS_West,CONUS_Central,CONUS_South', + 'EVAL_PERIODS':['last31days','last90days'], + 'VARIABLES':{ + 'ctc':{ + 'ASNOW_06':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['30,36,42,48'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A06', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, + 'ASNOW_24':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['24','36','48'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A24', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, + } + } + }, + }, + 'init_group1, fcst_group2, var_group1, namnest':{ + 'threshold_average':{ + 'DATE_TYPE':'INIT', + 'VALID_BEG':'', + 'VALID_END':'', + 'INIT_BEG':'', + 'INIT_END':'', + 'VX_MASK_LIST':'CONUS,CONUS_East,CONUS_West,CONUS_Central,CONUS_South', + 'EVAL_PERIODS':['last31days','last90days'], + 'VARIABLES':{ + 'ctc':{ + 'SNOD_06':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['42,48,54,60'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A06', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'SNOD_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -324,6 +484,19 @@ }, }, 'nbrcnt':{ + 'SNOD_06':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['42,48,54,60'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A06', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'SNOD_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -350,6 +523,19 @@ 'EVAL_PERIODS':['last31days','last90days'], 'VARIABLES':{ 'ctc':{ + 'SNOD_06':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['42,48,54,60'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A06', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'SNOD_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -367,7 +553,7 @@ } }, }, - 'init_group2, fcst_group1, namnest, hrrr':{ + 'init_group2, fcst_group1, var_group1, namnest, hrrr':{ 'threshold_average':{ 'DATE_TYPE':'INIT', 'VALID_BEG':'', @@ -475,7 +661,76 @@ } }, }, - 'init_group2, fcst_group2, namnest':{ + 'init_group2, fcst_group1, var_group2, hrrr':{ + 'threshold_average':{ + 'DATE_TYPE':'INIT', + 'VALID_BEG':'', + 'VALID_END':'', + 'INIT_BEG':'', + 'INIT_END':'', + 'VX_MASK_LIST':'CONUS,CONUS_East,CONUS_West,CONUS_Central,CONUS_South', + 'EVAL_PERIODS':['last31days','last90days'], + 'VARIABLES':{ + 'ctc':{ + 'ASNOW_24':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['6','18'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['30','42'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A24', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, + }, + 'nbrcnt':{ + 'ASNOW_24':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['6','18'], + 'STATSs':['fss'], + 'FCST_LEADS':['30','42'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A24', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, + }, + } + }, + 'performance_diagram':{ + 'DATE_TYPE':'INIT', + 'VALID_BEG':'', + 'VALID_END':'', + 'INIT_BEG':'', + 'INIT_END':'', + 'VX_MASK_LIST':'CONUS,CONUS_East,CONUS_West,CONUS_Central,CONUS_South', + 'EVAL_PERIODS':['last31days','last90days'], + 'VARIABLES':{ + 'ctc':{ + 'ASNOW_24':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['6','18'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['30','42'], + 'FCST_LEVEL':'Z0', + 'OBS_LEVEL':'A24', + 'FCST_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'OBS_THRESHs':['>=0.0254,>=0.0508,>=0.1016,>=0.2032,>=0.3048'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, + } + } + }, + }, + 'init_group2, fcst_group2, var_group1, namnest':{ 'threshold_average':{ 'DATE_TYPE':'INIT', 'VALID_BEG':'', diff --git a/ush/cam/cam_stats_snowfall_var_defs.py b/ush/cam/cam_stats_snowfall_var_defs.py index 93da553beb..1fb02da207 100644 --- a/ush/cam/cam_stats_snowfall_var_defs.py +++ b/ush/cam/cam_stats_snowfall_var_defs.py @@ -72,36 +72,6 @@ }, 'SNOD': { 'nohrsc': { - 'hireswarw': { - 'var1_fcst_name': 'SNOD', - 'var1_fcst_levels': 'Z0', - 'var1_fcst_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_fcst_options': 'censor_thresh = lt0; censor_val = 0; set_attr_units = \\"m\\";', - 'var1_obs_name': 'ASNOW', - 'var1_obs_levels': '', - 'var1_obs_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_obs_options': '', - }, - 'hireswarwmem2': { - 'var1_fcst_name': 'SNOD', - 'var1_fcst_levels': 'Z0', - 'var1_fcst_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_fcst_options': 'censor_thresh = lt0; censor_val = 0; set_attr_units = \\"m\\";', - 'var1_obs_name': 'ASNOW', - 'var1_obs_levels': '', - 'var1_obs_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_obs_options': '', - }, - 'hireswfv3': { - 'var1_fcst_name': 'SNOD', - 'var1_fcst_levels': 'Z0', - 'var1_fcst_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_fcst_options': 'censor_thresh = lt0; censor_val = 0; set_attr_units = \\"m\\";', - 'var1_obs_name': 'ASNOW', - 'var1_obs_levels': '', - 'var1_obs_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_obs_options': '', - }, 'hrrr': { 'var1_fcst_name': 'SNOD', 'var1_fcst_levels': 'Z0', @@ -134,36 +104,6 @@ }, 'ASNOW': { 'nohrsc': { - 'hireswarw': { - 'var1_fcst_name': 'ASNOW', - 'var1_fcst_levels': 'Z0', - 'var1_fcst_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_fcst_options': 'censor_thresh = lt0; censor_val = 0; set_attr_level = \\"Z0\\"; set_attr_units = \\"m\\";', - 'var1_obs_name': 'ASNOW', - 'var1_obs_levels': '', - 'var1_obs_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_obs_options': '', - }, - 'hireswarwmem2': { - 'var1_fcst_name': 'ASNOW', - 'var1_fcst_levels': 'Z0', - 'var1_fcst_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_fcst_options': 'censor_thresh = lt0; censor_val = 0; set_attr_level = \\"Z0\\"; set_attr_units = \\"m\\";', - 'var1_obs_name': 'ASNOW', - 'var1_obs_levels': '', - 'var1_obs_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_obs_options': '', - }, - 'hireswfv3': { - 'var1_fcst_name': 'ASNOW', - 'var1_fcst_levels': 'Z0', - 'var1_fcst_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_fcst_options': 'censor_thresh = lt0; censor_val = 0; set_attr_level = \\"Z0\\"; set_attr_units = \\"m\\";', - 'var1_obs_name': 'ASNOW', - 'var1_obs_levels': '', - 'var1_obs_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_obs_options': '', - }, 'hrrr': { 'var1_fcst_name': 'ASNOW', 'var1_fcst_levels': 'Z0', @@ -174,16 +114,6 @@ 'var1_obs_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', 'var1_obs_options': '', }, - 'namnest': { - 'var1_fcst_name': 'ASNOW', - 'var1_fcst_levels': 'Z0', - 'var1_fcst_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_fcst_options': 'censor_thresh = lt0; censor_val = 0; set_attr_level = \\"Z0\\"; set_attr_units = \\"m\\";', - 'var1_obs_name': 'ASNOW', - 'var1_obs_levels': '', - 'var1_obs_thresholds': 'ge0.0254,ge0.0508,ge0.1016,ge0.2032,ge0.3048', - 'var1_obs_options': '', - }, 'output_types': { 'CTC': 'STAT', 'SL1L2': 'NONE', diff --git a/ush/cam/lead_average.py b/ush/cam/lead_average.py index 0e8475f93b..241de0c398 100755 --- a/ush/cam/lead_average.py +++ b/ush/cam/lead_average.py @@ -4,7 +4,6 @@ # Name: lead_average.py # Contact(s): Marcel Caron # Developed: Nov. 18, 2021 by Marcel Caron -# Last Modified: Jul. 05, 2023 by Marcel Caron # Title: Line plot of verification metric as a function of # lead time # Abstract: Plots METplus output (e.g., BCRMSE) as a line plot, @@ -103,7 +102,7 @@ def plot_lead_average(df: pd.DataFrame, logger: logging.Logger, return None # filter by forecast lead times if isinstance(flead, list): - if len(flead) <= 8: + if len(flead) <= 3: if len(flead) > 1: frange_phrase = 's '+', '.join([str(f) for f in flead]) else: @@ -1109,8 +1108,10 @@ def plot_lead_average(df: pd.DataFrame, logger: logging.Logger, var_savename = 'APCP' elif 'PROB_MXUPHL25_A24_GEHWT' in var_savename.upper(): var_savename = 'MXUPHL25' - elif any(field in var_savename.upper() for field in ['ASNOW','SNOD']): + elif any(field in var_savename.upper() for field in ['ASNOW']): var_savename = 'ASNOW' + elif any(field in var_savename.upper() for field in ['SNOD']): + var_savename = 'SNOD' elif str(df['OBS_VAR'].tolist()[0]).upper() in ['HPBL']: var_savename = 'HPBL' elif str(df['OBS_VAR'].tolist()[0]).upper() in ['MSLET','MSLMA','PRMSL']: diff --git a/ush/cam/performance_diagram.py b/ush/cam/performance_diagram.py index f09c24f03a..b70bd56e28 100755 --- a/ush/cam/performance_diagram.py +++ b/ush/cam/performance_diagram.py @@ -4,7 +4,6 @@ # Name: performance_diagram.py # Contact(s): Marcel Caron # Developed: Nov. 22, 2021 by Marcel Caron -# Last Modified: Mar. 15, 2023 by Marcel Caron # Title: Performance Diagram: plot displaying multiple skill scores # Abstract: Plots METplus output as a line plot, expressed as prob. # of detection as a function of success ratio but displaying @@ -124,7 +123,7 @@ def plot_performance_diagram(df: pd.DataFrame, logger: logging.Logger, # filter by forecast lead times if isinstance(flead, list): - if len(flead) <= 8: + if len(flead) <= 3: if len(flead) > 1: frange_phrase = 's '+', '.join([str(f) for f in flead]) else: @@ -932,8 +931,10 @@ def plot_performance_diagram(df: pd.DataFrame, logger: logging.Logger, var_savename = df['FCST_VAR'].tolist()[0] if any(field in var_savename.upper() for field in ['APCP']): var_savename = re.sub('[^a-zA-Z \n\.]', '', var_savename) - elif any(field in var_savename.upper() for field in ['ASNOW','SNOD']): + elif any(field in var_savename.upper() for field in ['ASNOW']): var_savename = 'ASNOW' + elif any(field in var_savename.upper() for field in ['SNOD']): + var_savename = 'SNOD' elif 'PROB_MXUPHL25_A24_GEHWT' in var_savename.upper(): var_savename = 'MXUPHL25' elif str(df['OBS_VAR'].tolist()[0]).upper() in ['HPBL']: diff --git a/ush/cam/roc_curve.py b/ush/cam/roc_curve.py index 0ea2694de6..fbbfd8836f 100755 --- a/ush/cam/roc_curve.py +++ b/ush/cam/roc_curve.py @@ -4,7 +4,6 @@ # Name: roc_curve.py # Contact(s): Marcel Caron # Developed: Sep. 26, 2022 by Marcel Caron -# Last Modified: Sep. 26, 2022 by Marcel Caron # Title: Receiver Operator Characteristic (ROC) curve # Abstract: Plots METplus CTC output as a line plot, probability of # detection, which represents the y-axis, varying by false alarm @@ -113,7 +112,7 @@ def plot_performance_diagram(df: pd.DataFrame, logger: logging.Logger, # filter by forecast lead times if isinstance(flead, list): - if len(flead) <= 8: + if len(flead) <= 3: if len(flead) > 1: frange_phrase = 's '+', '.join([str(f) for f in flead]) else: diff --git a/ush/cam/settings.py b/ush/cam/settings.py index f29197fbf3..0d46bfc4ca 100755 --- a/ush/cam/settings.py +++ b/ush/cam/settings.py @@ -3814,7 +3814,7 @@ def __init__(self): + ' >=0.3048,'), 'obs_var_options': '', 'plot_group':'precip'}, - 'SNOD_06': {'fcst_var_names': ['SNOD', 'ASNOW', 'SNOD_06', 'ASNOW_06'], + 'SNOD_06': {'fcst_var_names': ['SNOD', 'SNOD_06'], 'fcst_var_levels': ['Z0','A06','A6'], 'fcst_var_thresholds': ('>=0.0254, >=0.0508,' + ' >=0.1016,' @@ -3829,7 +3829,37 @@ def __init__(self): + ' >=0.3048,'), 'obs_var_options': '', 'plot_group':'precip'}, - 'SNOD_24': {'fcst_var_names': ['SNOD', 'ASNOW', 'SNOD_24', 'ASNOW_24'], + 'SNOD_24': {'fcst_var_names': ['SNOD', 'SNOD_24'], + 'fcst_var_levels': ['Z0','A24'], + 'fcst_var_thresholds': ('>=0.0254, >=0.0508,' + + ' >=0.1016,' + + ' >=0.2032,' + + ' >=0.3048,'), + 'fcst_var_options': '', + 'obs_var_names': ['ASNOW', 'ASNOW_24'], + 'obs_var_levels': ['A24'], + 'obs_var_thresholds': ('>=0.0254, >=0.0508,' + + ' >=0.1016,' + + ' >=0.2032,' + + ' >=0.3048,'), + 'obs_var_options': '', + 'plot_group':'precip'}, + 'ASNOW_06': {'fcst_var_names': ['ASNOW', 'ASNOW_06'], + 'fcst_var_levels': ['Z0','A06','A6'], + 'fcst_var_thresholds': ('>=0.0254, >=0.0508,' + + ' >=0.1016,' + + ' >=0.2032,' + + ' >=0.3048,'), + 'fcst_var_options': '', + 'obs_var_names': ['ASNOW', 'ASNOW_06'], + 'obs_var_levels': ['A06','A6'], + 'obs_var_thresholds': ('>=0.0254, >=0.0508,' + + ' >=0.1016,' + + ' >=0.2032,' + + ' >=0.3048,'), + 'obs_var_options': '', + 'plot_group':'precip'}, + 'ASNOW_24': {'fcst_var_names': ['ASNOW', 'ASNOW_24'], 'fcst_var_levels': ['Z0','A24'], 'fcst_var_thresholds': ('>=0.0254, >=0.0508,' + ' >=0.1016,' @@ -3887,7 +3917,37 @@ def __init__(self): + ' >=0.3048,'), 'obs_var_options': '', 'plot_group':'precip'}, - 'SNOD_06': {'fcst_var_names': ['SNOD', 'ASNOW', 'SNOD_06', 'ASNOW_06'], + 'SNOD_06': {'fcst_var_names': ['SNOD', 'SNOD_06'], + 'fcst_var_levels': ['Z0','A06','A6'], + 'fcst_var_thresholds': ('>=0.0254, >=0.0508,' + + ' >=0.1016,' + + ' >=0.2032,' + + ' >=0.3048,'), + 'fcst_var_options': '', + 'obs_var_names': ['ASNOW', 'ASNOW_06'], + 'obs_var_levels': ['A06','A6'], + 'obs_var_thresholds': ('>=0.0254, >=0.0508,' + + ' >=0.1016,' + + ' >=0.2032,' + + ' >=0.3048,'), + 'obs_var_options': '', + 'plot_group':'precip'}, + 'SNOD_24': {'fcst_var_names': ['SNOD', 'SNOD_24'], + 'fcst_var_levels': ['Z0','A24'], + 'fcst_var_thresholds': ('>=0.0254, >=0.0508,' + + ' >=0.1016,' + + ' >=0.2032,' + + ' >=0.3048,'), + 'fcst_var_options': '', + 'obs_var_names': ['ASNOW', 'ASNOW_24'], + 'obs_var_levels': ['A24'], + 'obs_var_thresholds': ('>=0.0254, >=0.0508,' + + ' >=0.1016,' + + ' >=0.2032,' + + ' >=0.3048,'), + 'obs_var_options': '', + 'plot_group':'precip'}, + 'ASNOW_06': {'fcst_var_names': ['ASNOW', 'ASNOW_06'], 'fcst_var_levels': ['Z0','A06','A6'], 'fcst_var_thresholds': ('>=0.0254, >=0.0508,' + ' >=0.1016,' @@ -3902,7 +3962,7 @@ def __init__(self): + ' >=0.3048,'), 'obs_var_options': '', 'plot_group':'precip'}, - 'SNOD_24': {'fcst_var_names': ['SNOD', 'ASNOW', 'SNOD_24', 'ASNOW_24'], + 'ASNOW_24': {'fcst_var_names': ['ASNOW', 'ASNOW_24'], 'fcst_var_levels': ['Z0','A24'], 'fcst_var_thresholds': ('>=0.0254, >=0.0508,' + ' >=0.1016,' diff --git a/ush/cam/stat_by_level.py b/ush/cam/stat_by_level.py index 8cf3427a7a..26b0dfa3a2 100755 --- a/ush/cam/stat_by_level.py +++ b/ush/cam/stat_by_level.py @@ -4,7 +4,6 @@ # Name: stat_by_level.py # Contact(s): Marcel Caron # Developed: Oct. 14, 2021 by Marcel Caron -# Last Modified: Dec. 02, 2022 by Marcel Caron # Title: Line plot of pressure level as a function of # verification metric # Abstract: Plots METplus output (e.g., BCRMSE) as a line plot, @@ -104,7 +103,7 @@ def plot_stat_by_level(df: pd.DataFrame, logger: logging.Logger, frange_save_string = 'ALL_LEADS' pass elif isinstance(flead, list): - if len(flead) <= 8: + if len(flead) <= 3: if len(flead) > 1: frange_phrase = 's '+', '.join([str(f) for f in flead]) else: @@ -926,8 +925,10 @@ def plot_stat_by_level(df: pd.DataFrame, logger: logging.Logger, var_savename = df['FCST_VAR'].tolist()[0] if 'APCP' in var_savename.upper(): var_savename = 'APCP' - elif any(field in var_savename.upper() for field in ['ASNOW','SNOD']): + elif any(field in var_savename.upper() for field in ['ASNOW']): var_savename = 'ASNOW' + elif any(field in var_savename.upper() for field in ['SNOD']): + var_savename = 'SNOD' elif str(df['OBS_VAR'].tolist()[0]).upper() in ['HPBL']: var_savename = 'HPBL' elif str(df['OBS_VAR'].tolist()[0]).upper() in ['MSLET','MSLMA','PRMSL']: diff --git a/ush/cam/threshold_average.py b/ush/cam/threshold_average.py index 793753a6a5..637f9bf7c2 100755 --- a/ush/cam/threshold_average.py +++ b/ush/cam/threshold_average.py @@ -4,7 +4,6 @@ # Name: threshold_average.py # Contact(s): Marcel Caron # Developed: Nov. 22, 2021 by Marcel Caron -# Last Modified: Dec. 1, 2021 by Marcel Caron # Title: Line plot of verification metric as a function of # forecast threshold # Abstract: Plots METplus output (e.g., BCRMSE) as a line plot, @@ -107,7 +106,7 @@ def plot_threshold_average(df: pd.DataFrame, logger: logging.Logger, return None # filter by forecast lead times if isinstance(flead, list): - if len(flead) <= 8: + if len(flead) <= 3: if len(flead) > 1: frange_phrase = 's '+', '.join([str(f) for f in flead]) else: @@ -784,8 +783,10 @@ def plot_threshold_average(df: pd.DataFrame, logger: logging.Logger, var_savename = df['FCST_VAR'].tolist()[0] if any(field in var_savename.upper() for field in ['APCP']): var_savename = re.sub('[^a-zA-Z \n\.]', '', var_savename) - elif any(field in var_savename.upper() for field in ['ASNOW','SNOD']): + elif any(field in var_savename.upper() for field in ['ASNOW']): var_savename = 'ASNOW' + elif any(field in var_savename.upper() for field in ['SNOD']): + var_savename = 'SNOD' elif 'PROB_MXUPHL25_A24_GEHWT' in var_savename.upper(): var_savename = 'MXUPHL25' elif str(df['OBS_VAR'].tolist()[0]).upper() in ['HPBL']: diff --git a/ush/cam/time_series.py b/ush/cam/time_series.py index 178288b3d3..422ddc99ea 100755 --- a/ush/cam/time_series.py +++ b/ush/cam/time_series.py @@ -4,7 +4,6 @@ # Name: time_series.py # Contact(s): Marcel Caron # Developed: Oct. 14, 2021 by Marcel Caron -# Last Modified: Dec. 01, 2022 by Marcel Caron # Title: Line plot of verification metric as a function of # valid or init time # Abstract: Plots METplus output (e.g., BCRMSE) as a line plot, @@ -108,7 +107,7 @@ def plot_time_series(df: pd.DataFrame, logger: logging.Logger, # filter by forecast lead times if isinstance(flead, list): - if len(flead) <= 8: + if len(flead) <= 3: if len(flead) > 1: frange_phrase = 's '+', '.join([str(f) for f in flead]) else: @@ -984,8 +983,10 @@ def plot_time_series(df: pd.DataFrame, logger: logging.Logger, var_savename = df['FCST_VAR'].tolist()[0] if 'APCP' in var_savename.upper(): var_savename = 'APCP' - elif any(field in var_savename.upper() for field in ['ASNOW','SNOD']): + elif any(field in var_savename.upper() for field in ['ASNOW']): var_savename = 'ASNOW' + elif any(field in var_savename.upper() for field in ['SNOD']): + var_savename = 'SNOD' elif str(df['OBS_VAR'].tolist()[0]).upper() in ['HPBL']: var_savename = 'HPBL' elif str(df['OBS_VAR'].tolist()[0]).upper() in ['MSLET','MSLMA','PRMSL']: diff --git a/ush/cam/valid_hour_average.py b/ush/cam/valid_hour_average.py index fae07554c5..d15008e8ac 100755 --- a/ush/cam/valid_hour_average.py +++ b/ush/cam/valid_hour_average.py @@ -4,7 +4,6 @@ # Name: valid_hour_average.py # Contact(s): Marcel Caron # Developed: Nov. 22, 2021 by Marcel Caron -# Last Modified: Jul. 05, 2023 by Marcel Caron # Title: Line plot of verification metric as a function of # valid or init hour # Abstract: Plots METplus output (e.g., BCRMSE) as a line plot, @@ -109,7 +108,7 @@ def plot_valid_hour_average(df: pd.DataFrame, logger: logging.Logger, # filter by forecast lead times if isinstance(flead, list): - if len(flead) <= 8: + if len(flead) <= 3: if len(flead) > 1: frange_phrase = 's '+', '.join([str(f) for f in flead]) else: @@ -1100,8 +1099,10 @@ def plot_valid_hour_average(df: pd.DataFrame, logger: logging.Logger, var_savename = df['FCST_VAR'].tolist()[0] if 'APCP' in var_savename.upper(): var_savename = 'APCP' - elif any(field in var_savename.upper() for field in ['ASNOW','SNOD']): + elif any(field in var_savename.upper() for field in ['ASNOW']): var_savename = 'ASNOW' + elif any(field in var_savename.upper() for field in ['SNOD']): + var_savename = 'SNOD' elif str(df['OBS_VAR'].tolist()[0]).upper() in ['HPBL']: var_savename = 'HPBL' elif str(df['OBS_VAR'].tolist()[0]).upper() in ['MSLET','MSLMA','PRMSL']: From 37f34166c54228c45d0c661381fc06ca56a9a336 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 25 Oct 2024 15:44:07 +0000 Subject: [PATCH 05/25] Adjust precip graphics --- ...cam_plots_precip_last31days_graphx_defs.py | 270 ++++++++++++++++-- ...cam_plots_precip_last90days_graphx_defs.py | 270 ++++++++++++++++-- 2 files changed, 504 insertions(+), 36 deletions(-) diff --git a/ush/cam/cam_plots_precip_last31days_graphx_defs.py b/ush/cam/cam_plots_precip_last31days_graphx_defs.py index d108a7d62c..9ad5828ed0 100755 --- a/ush/cam/cam_plots_precip_last31days_graphx_defs.py +++ b/ush/cam/cam_plots_precip_last31days_graphx_defs.py @@ -41,7 +41,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -82,7 +82,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -134,7 +134,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -197,6 +197,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -212,6 +225,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -238,6 +264,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -499,7 +538,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -540,7 +579,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -592,7 +631,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -629,6 +668,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -644,6 +696,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -670,6 +735,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -853,7 +931,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -894,7 +972,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -946,7 +1024,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -983,6 +1061,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -998,6 +1089,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1024,6 +1128,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1207,7 +1324,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1248,7 +1365,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1300,7 +1417,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1337,6 +1454,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1352,6 +1482,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1378,6 +1521,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1561,7 +1717,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1602,7 +1758,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1654,7 +1810,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1691,6 +1847,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1706,6 +1875,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1732,6 +1914,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1917,7 +2112,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1958,7 +2153,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -2010,7 +2205,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -2073,6 +2268,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -2088,6 +2296,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -2114,6 +2335,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], diff --git a/ush/cam/cam_plots_precip_last90days_graphx_defs.py b/ush/cam/cam_plots_precip_last90days_graphx_defs.py index fe179491d3..3c6d4b8484 100755 --- a/ush/cam/cam_plots_precip_last90days_graphx_defs.py +++ b/ush/cam/cam_plots_precip_last90days_graphx_defs.py @@ -41,7 +41,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -82,7 +82,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -134,7 +134,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -171,6 +171,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -186,6 +199,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -212,6 +238,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -395,7 +434,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -436,7 +475,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -488,7 +527,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -525,6 +564,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -540,6 +592,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -566,6 +631,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -749,7 +827,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -790,7 +868,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -842,7 +920,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -879,6 +957,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -894,6 +985,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -920,6 +1024,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1103,7 +1220,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1144,7 +1261,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1196,7 +1313,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1233,6 +1350,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1248,6 +1378,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1274,6 +1417,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1457,7 +1613,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1498,7 +1654,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1550,7 +1706,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1587,6 +1743,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1602,6 +1771,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1628,6 +1810,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1813,7 +2008,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fbias','ets'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1854,7 +2049,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['fss'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1906,7 +2101,7 @@ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], 'STATSs':['sratio,pod,csi'], - 'FCST_LEADS':['24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48','36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60'], + 'FCST_LEADS':['3,6,9,12,15,18,21,24','15,18,21,24,27,30,33,36','27,30,33,36,39,42,45,48'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], @@ -1943,6 +2138,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fbias','ets'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1958,6 +2166,19 @@ }, }, 'nbrcnt':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['fss'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NBRHD_SQUARE', + 'INTERP_PNTSs':['1','25','81','441','961','3969'], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1984,6 +2205,19 @@ 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'VARIABLES':{ 'ctc':{ + 'APCP_03':{ + 'FCST_VALID_HOURS':[''], + 'FCST_INIT_HOURS':['0','12'], + 'STATSs':['sratio,pod,csi'], + 'FCST_LEADS':['39,42,45,48,51,54,57,60'], + 'FCST_LEVEL':'A3', + 'OBS_LEVEL':'A3', + 'FCST_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'OBS_THRESHs':['>=0.254,>=1.27,>=2.54,>=6.35,>=12.7,>=19.05,>=25.4,>=50.8'], + 'CONFIDENCE_INTERVALS':'False', + 'INTERP':'NEAREST', + 'INTERP_PNTSs':[''], + }, 'APCP_24':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], From 64772461dce15ebdb87130bee116bf8e9f4c2c88 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 25 Oct 2024 16:23:04 +0000 Subject: [PATCH 06/25] Fix HRRR TCDC field configuration --- ush/cam/cam_stats_grid2obs_var_defs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/cam/cam_stats_grid2obs_var_defs.py b/ush/cam/cam_stats_grid2obs_var_defs.py index c11f0faa86..b7d1e4edfc 100644 --- a/ush/cam/cam_stats_grid2obs_var_defs.py +++ b/ush/cam/cam_stats_grid2obs_var_defs.py @@ -1540,7 +1540,7 @@ 'var1_fcst_name': 'TCDC', 'var1_fcst_levels': 'L0', 'var1_fcst_thresholds': 'lt10,gt10,gt50,gt90', - 'var1_fcst_options': 'set_attr_level = \\"TOTAL\\"; GRIB2_ipdtmpl_index = [ 9 ]; GRIB2_ipdtmpl_val = [ 211 ];', + 'var1_fcst_options': 'set_attr_level = \\"TOTAL\\"; GRIB2_ipdtmpl_index = [ 9 ]; GRIB2_ipdtmpl_val = [ 10 ];', 'var1_obs_name': 'TCDC', 'var1_obs_levels': 'L0', 'var1_obs_thresholds': 'lt10,gt10,gt50,gt90', From f3788c9131d7f31a6ef1de12d79995dd44f47a91 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 25 Oct 2024 16:28:48 +0000 Subject: [PATCH 07/25] Add CONUS E/W/C/S graphics --- ush/cam/cam_plots_grid2obs_graphx_defs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/cam/cam_plots_grid2obs_graphx_defs.py b/ush/cam/cam_plots_grid2obs_graphx_defs.py index 31fa67fab7..71ddc25a04 100755 --- a/ush/cam/cam_plots_grid2obs_graphx_defs.py +++ b/ush/cam/cam_plots_grid2obs_graphx_defs.py @@ -861,7 +861,7 @@ 'VALID_END':'', 'INIT_BEG':'', 'INIT_END':'', - 'VX_MASK_LISTs':['CONUS'], + 'VX_MASK_LISTs':['CONUS','CONUS_East','CONUS_West','CONUS_Central','CONUS_South'], 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], @@ -1085,7 +1085,7 @@ 'VALID_END':'', 'INIT_BEG':'', 'INIT_END':'', - 'VX_MASK_LISTs':['CONUS'], + 'VX_MASK_LISTs':['CONUS','CONUS_East','CONUS_West','CONUS_Central','CONUS_South'], 'EVAL_PERIODS':[os.environ['EVAL_PERIOD']], 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], From 8c721da82f16c8cd802d14900f8ec5a91e809c1d Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 25 Oct 2024 17:21:07 +0000 Subject: [PATCH 08/25] update walltime --- .../scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.sh | 2 +- .../scripts/plots/cam/jevs_cam_grid2obs_plots_last90days.sh | 2 +- ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.ecf | 2 +- ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last90days.ecf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.sh b/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.sh index b31ed0836d..2a2847a37d 100755 --- a/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.sh +++ b/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.sh @@ -4,7 +4,7 @@ #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=2:30:00 +#PBS -l walltime=8:00:00 #PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=220GB #PBS -l debug=true diff --git a/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last90days.sh b/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last90days.sh index 93f8158c83..6e422172e3 100755 --- a/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last90days.sh +++ b/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last90days.sh @@ -4,7 +4,7 @@ #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=4:30:00 +#PBS -l walltime=8:00:00 #PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=330GB #PBS -l debug=true diff --git a/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.ecf b/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.ecf index 03557cc3c8..1170de7fc5 100755 --- a/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.ecf +++ b/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=2:30:00 +#PBS -l walltime=8:00:00 #PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=220GB #PBS -l debug=true diff --git a/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last90days.ecf b/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last90days.ecf index 5fb0fe57de..ebb2d6e188 100755 --- a/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last90days.ecf +++ b/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last90days.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=4:30:00 +#PBS -l walltime=8:00:00 #PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=330GB #PBS -l debug=true From 15fea2ffc60abe7730349788ce946bbfbde820d2 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 25 Oct 2024 17:42:40 +0000 Subject: [PATCH 09/25] Adjust severe prep resource allocation + match ecf with dev --- dev/drivers/scripts/prep/cam/jevs_cam_hireswarw_severe_prep.sh | 2 +- .../scripts/prep/cam/jevs_cam_hireswarwmem2_severe_prep.sh | 2 +- dev/drivers/scripts/prep/cam/jevs_cam_hireswfv3_severe_prep.sh | 2 +- dev/drivers/scripts/prep/cam/jevs_cam_hrrr_severe_prep.sh | 2 +- dev/drivers/scripts/prep/cam/jevs_cam_namnest_severe_prep.sh | 2 +- .../prep/cam/jevs_cam_hireswarw_severe_prep_vhr_master.ecf | 2 +- .../prep/cam/jevs_cam_hireswarwmem2_severe_prep_vhr_master.ecf | 2 +- .../prep/cam/jevs_cam_hireswfv3_severe_prep_vhr_master.ecf | 2 +- ecf/scripts/prep/cam/jevs_cam_href_severe_prep_vhr_master.ecf | 2 +- ecf/scripts/prep/cam/jevs_cam_hrrr_severe_prep_vhr_master.ecf | 2 +- .../prep/cam/jevs_cam_namnest_severe_prep_vhr_master.ecf | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dev/drivers/scripts/prep/cam/jevs_cam_hireswarw_severe_prep.sh b/dev/drivers/scripts/prep/cam/jevs_cam_hireswarw_severe_prep.sh index 9b4b22e18b..2f5fbe87c0 100644 --- a/dev/drivers/scripts/prep/cam/jevs_cam_hireswarw_severe_prep.sh +++ b/dev/drivers/scripts/prep/cam/jevs_cam_hireswarw_severe_prep.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=shared,select=1:ncpus=1:mem=10GB +#PBS -l place=shared,select=1:ncpus=1:mem=15GB #PBS -l debug=true diff --git a/dev/drivers/scripts/prep/cam/jevs_cam_hireswarwmem2_severe_prep.sh b/dev/drivers/scripts/prep/cam/jevs_cam_hireswarwmem2_severe_prep.sh index dff569b135..34448866f2 100644 --- a/dev/drivers/scripts/prep/cam/jevs_cam_hireswarwmem2_severe_prep.sh +++ b/dev/drivers/scripts/prep/cam/jevs_cam_hireswarwmem2_severe_prep.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=shared,select=1:ncpus=1:mem=10GB +#PBS -l place=shared,select=1:ncpus=1:mem=15GB #PBS -l debug=true diff --git a/dev/drivers/scripts/prep/cam/jevs_cam_hireswfv3_severe_prep.sh b/dev/drivers/scripts/prep/cam/jevs_cam_hireswfv3_severe_prep.sh index 17c85f7471..10387d628e 100644 --- a/dev/drivers/scripts/prep/cam/jevs_cam_hireswfv3_severe_prep.sh +++ b/dev/drivers/scripts/prep/cam/jevs_cam_hireswfv3_severe_prep.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=shared,select=1:ncpus=1:mem=10GB +#PBS -l place=shared,select=1:ncpus=1:mem=15GB #PBS -l debug=true diff --git a/dev/drivers/scripts/prep/cam/jevs_cam_hrrr_severe_prep.sh b/dev/drivers/scripts/prep/cam/jevs_cam_hrrr_severe_prep.sh index b3497ed6b3..97c973322f 100644 --- a/dev/drivers/scripts/prep/cam/jevs_cam_hrrr_severe_prep.sh +++ b/dev/drivers/scripts/prep/cam/jevs_cam_hrrr_severe_prep.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=shared,select=1:ncpus=1:mem=10GB +#PBS -l place=shared,select=1:ncpus=1:mem=15GB #PBS -l debug=true diff --git a/dev/drivers/scripts/prep/cam/jevs_cam_namnest_severe_prep.sh b/dev/drivers/scripts/prep/cam/jevs_cam_namnest_severe_prep.sh index 729cc14dbb..11a6cfd86d 100644 --- a/dev/drivers/scripts/prep/cam/jevs_cam_namnest_severe_prep.sh +++ b/dev/drivers/scripts/prep/cam/jevs_cam_namnest_severe_prep.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=shared,select=1:ncpus=1:mem=10GB +#PBS -l place=shared,select=1:ncpus=1:mem=75GB #PBS -l debug=true diff --git a/ecf/scripts/prep/cam/jevs_cam_hireswarw_severe_prep_vhr_master.ecf b/ecf/scripts/prep/cam/jevs_cam_hireswarw_severe_prep_vhr_master.ecf index 2876916b2c..7694e7f8a4 100644 --- a/ecf/scripts/prep/cam/jevs_cam_hireswarw_severe_prep_vhr_master.ecf +++ b/ecf/scripts/prep/cam/jevs_cam_hireswarw_severe_prep_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=shared,select=1:ncpus=1:mem=40GB +#PBS -l place=shared,select=1:ncpus=1:mem=15GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/prep/cam/jevs_cam_hireswarwmem2_severe_prep_vhr_master.ecf b/ecf/scripts/prep/cam/jevs_cam_hireswarwmem2_severe_prep_vhr_master.ecf index 2a1bc61a93..f1955a7bce 100644 --- a/ecf/scripts/prep/cam/jevs_cam_hireswarwmem2_severe_prep_vhr_master.ecf +++ b/ecf/scripts/prep/cam/jevs_cam_hireswarwmem2_severe_prep_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=shared,select=1:ncpus=1:mem=40GB +#PBS -l place=shared,select=1:ncpus=1:mem=15GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/prep/cam/jevs_cam_hireswfv3_severe_prep_vhr_master.ecf b/ecf/scripts/prep/cam/jevs_cam_hireswfv3_severe_prep_vhr_master.ecf index 0a126268e7..871f4cb2b0 100644 --- a/ecf/scripts/prep/cam/jevs_cam_hireswfv3_severe_prep_vhr_master.ecf +++ b/ecf/scripts/prep/cam/jevs_cam_hireswfv3_severe_prep_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=shared,select=1:ncpus=1:mem=10GB +#PBS -l place=shared,select=1:ncpus=1:mem=15GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/prep/cam/jevs_cam_href_severe_prep_vhr_master.ecf b/ecf/scripts/prep/cam/jevs_cam_href_severe_prep_vhr_master.ecf index 3d9b816747..97c5e1a314 100644 --- a/ecf/scripts/prep/cam/jevs_cam_href_severe_prep_vhr_master.ecf +++ b/ecf/scripts/prep/cam/jevs_cam_href_severe_prep_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=shared,select=1:ncpus=1:mem=20GB +#PBS -l place=shared,select=1:ncpus=1:mem=10GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/prep/cam/jevs_cam_hrrr_severe_prep_vhr_master.ecf b/ecf/scripts/prep/cam/jevs_cam_hrrr_severe_prep_vhr_master.ecf index 681e328cb3..4ad62690c1 100644 --- a/ecf/scripts/prep/cam/jevs_cam_hrrr_severe_prep_vhr_master.ecf +++ b/ecf/scripts/prep/cam/jevs_cam_hrrr_severe_prep_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=shared,select=1:ncpus=1:mem=30GB +#PBS -l place=shared,select=1:ncpus=1:mem=15GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/prep/cam/jevs_cam_namnest_severe_prep_vhr_master.ecf b/ecf/scripts/prep/cam/jevs_cam_namnest_severe_prep_vhr_master.ecf index 58e8273f86..76ac99d79d 100644 --- a/ecf/scripts/prep/cam/jevs_cam_namnest_severe_prep_vhr_master.ecf +++ b/ecf/scripts/prep/cam/jevs_cam_namnest_severe_prep_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:15:00 -#PBS -l place=shared,select=1:ncpus=1:mem=80GB +#PBS -l place=shared,select=1:ncpus=1:mem=75GB #PBS -l debug=true export model=evs From a7ea44c51ccf9eacf98747e46d365afe4c9b6520 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 25 Oct 2024 21:41:07 +0000 Subject: [PATCH 10/25] Fix input template for merged ptype file --- ush/cam/cam_stats_grid2obs_create_merged_ptype.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/cam/cam_stats_grid2obs_create_merged_ptype.py b/ush/cam/cam_stats_grid2obs_create_merged_ptype.py index 0e8fd82a3e..b5b86573a6 100644 --- a/ush/cam/cam_stats_grid2obs_create_merged_ptype.py +++ b/ush/cam/cam_stats_grid2obs_create_merged_ptype.py @@ -56,7 +56,7 @@ + f"process.") else: regrid_dir = os.path.join( - DATA, VERIF_CASE, 'METplus_output', 'workdirs', job_type, f'job{njob}', + DATA, VERIF_CASE, 'METplus_output', 'workdirs', job_type, f'{job_name}', VERIF_TYPE, 'regrid_data_plane', f'{MODELNAME}.{VDATE}' ) regrid_fname = (f'regrid_data_plane_{MODELNAME}_t{VHOUR}z_{VERIF_TYPE}_{NEST}_' From a86d9ff5113d2e1e84a209f9e09e1317b27932c4 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 25 Oct 2024 21:41:37 +0000 Subject: [PATCH 11/25] Fix metric config --- ush/cam/cam_plots_precip_last31days_graphx_defs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/cam/cam_plots_precip_last31days_graphx_defs.py b/ush/cam/cam_plots_precip_last31days_graphx_defs.py index 9ad5828ed0..c8c1877037 100755 --- a/ush/cam/cam_plots_precip_last31days_graphx_defs.py +++ b/ush/cam/cam_plots_precip_last31days_graphx_defs.py @@ -228,7 +228,7 @@ 'APCP_03':{ 'FCST_VALID_HOURS':[''], 'FCST_INIT_HOURS':['0','12'], - 'STATSs':['sratio,pod,csi'], + 'STATSs':['fss'], 'FCST_LEADS':['39,42,45,48,51,54,57,60'], 'FCST_LEVEL':'A3', 'OBS_LEVEL':'A3', From 01928c7467ee72a20dbd2ba2d678a46764536014 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Mon, 28 Oct 2024 15:19:52 +0000 Subject: [PATCH 12/25] Use verbose option for cp commands --- scripts/plots/cam/exevs_cam_grid2obs_plots.sh | 2 +- scripts/plots/cam/exevs_cam_headline_plots.sh | 2 +- scripts/plots/cam/exevs_cam_precip_plots.sh | 2 +- scripts/plots/cam/exevs_cam_radar_plots.sh | 2 +- scripts/plots/cam/exevs_cam_severe_plots.sh | 2 +- scripts/plots/cam/exevs_cam_snowfall_plots.sh | 2 +- scripts/stats/cam/exevs_cam_radar_stats.sh | 2 +- scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh | 10 +++++----- scripts/stats/cam/exevs_hireswarw_precip_stats.sh | 10 +++++----- scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh | 10 +++++----- .../stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh | 10 +++++----- scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh | 10 +++++----- .../stats/cam/exevs_hireswarwmem2_snowfall_stats.sh | 10 +++++----- scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh | 10 +++++----- scripts/stats/cam/exevs_hireswfv3_precip_stats.sh | 10 +++++----- scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh | 10 +++++----- scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh | 10 +++++----- scripts/stats/cam/exevs_hrrr_precip_stats.sh | 10 +++++----- scripts/stats/cam/exevs_hrrr_snowfall_stats.sh | 10 +++++----- scripts/stats/cam/exevs_namnest_grid2obs_stats.sh | 10 +++++----- scripts/stats/cam/exevs_namnest_precip_stats.sh | 10 +++++----- scripts/stats/cam/exevs_namnest_snowfall_stats.sh | 10 +++++----- 22 files changed, 82 insertions(+), 82 deletions(-) diff --git a/scripts/plots/cam/exevs_cam_grid2obs_plots.sh b/scripts/plots/cam/exevs_cam_grid2obs_plots.sh index 0e770f992e..dee57688a6 100755 --- a/scripts/plots/cam/exevs_cam_grid2obs_plots.sh +++ b/scripts/plots/cam/exevs_cam_grid2obs_plots.sh @@ -77,7 +77,7 @@ fi # Copy Plots Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/out/workdirs/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. export err=$?; err_chk done diff --git a/scripts/plots/cam/exevs_cam_headline_plots.sh b/scripts/plots/cam/exevs_cam_headline_plots.sh index 64f4d9012e..28ae2a04d4 100755 --- a/scripts/plots/cam/exevs_cam_headline_plots.sh +++ b/scripts/plots/cam/exevs_cam_headline_plots.sh @@ -76,7 +76,7 @@ fi # Copy Plots Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/out/workdirs/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. export err=$?; err_chk done diff --git a/scripts/plots/cam/exevs_cam_precip_plots.sh b/scripts/plots/cam/exevs_cam_precip_plots.sh index 7c56ccb2e4..2dbf930a95 100755 --- a/scripts/plots/cam/exevs_cam_precip_plots.sh +++ b/scripts/plots/cam/exevs_cam_precip_plots.sh @@ -77,7 +77,7 @@ fi # Copy Plots Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/out/workdirs/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. export err=$?; err_chk done diff --git a/scripts/plots/cam/exevs_cam_radar_plots.sh b/scripts/plots/cam/exevs_cam_radar_plots.sh index b42d8ea5fa..33d71da034 100755 --- a/scripts/plots/cam/exevs_cam_radar_plots.sh +++ b/scripts/plots/cam/exevs_cam_radar_plots.sh @@ -179,7 +179,7 @@ fi # Copy Plots Output to Main Directory ################################################################### for CHILD_DIR in ${DATA}/out/workdirs/*; do - cp -ru $CHILD_DIR/* ${DATA}/out/. + cp -ruv $CHILD_DIR/* ${DATA}/out/. export err=$?; err_chk done diff --git a/scripts/plots/cam/exevs_cam_severe_plots.sh b/scripts/plots/cam/exevs_cam_severe_plots.sh index d2b7665deb..dd9571da78 100755 --- a/scripts/plots/cam/exevs_cam_severe_plots.sh +++ b/scripts/plots/cam/exevs_cam_severe_plots.sh @@ -193,7 +193,7 @@ fi # Copy Plots Output to Main Directory ################################################################### for CHILD_DIR in ${DATA}/out/workdirs/*; do - cp -ru $CHILD_DIR/* ${DATA}/out/. + cp -ruv $CHILD_DIR/* ${DATA}/out/. export err=$?; err_chk done diff --git a/scripts/plots/cam/exevs_cam_snowfall_plots.sh b/scripts/plots/cam/exevs_cam_snowfall_plots.sh index 08836404d2..17e868cd3a 100755 --- a/scripts/plots/cam/exevs_cam_snowfall_plots.sh +++ b/scripts/plots/cam/exevs_cam_snowfall_plots.sh @@ -76,7 +76,7 @@ fi # Copy Plots Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/out/workdirs/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/out/. export err=$?; err_chk done diff --git a/scripts/stats/cam/exevs_cam_radar_stats.sh b/scripts/stats/cam/exevs_cam_radar_stats.sh index 8b44d33efe..ff7d2c4f0d 100755 --- a/scripts/stats/cam/exevs_cam_radar_stats.sh +++ b/scripts/stats/cam/exevs_cam_radar_stats.sh @@ -111,7 +111,7 @@ fi # Copy child output to main output directory ################################################################### for CHILD_DIR in ${DATA}/workdirs/*; do - cp -ru ${CHILD_DIR}/* ${DATA}/. + cp -ruv ${CHILD_DIR}/* ${DATA}/. export err=$?; err_chk done diff --git a/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh b/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh index 0946a61d8a..7bf32edc07 100755 --- a/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hireswarw_grid2obs_stats.sh @@ -116,7 +116,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -195,7 +195,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -263,7 +263,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -328,7 +328,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -405,7 +405,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_hireswarw_precip_stats.sh b/scripts/stats/cam/exevs_hireswarw_precip_stats.sh index 678efe3643..f80b599280 100755 --- a/scripts/stats/cam/exevs_hireswarw_precip_stats.sh +++ b/scripts/stats/cam/exevs_hireswarw_precip_stats.sh @@ -121,7 +121,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -211,7 +211,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -275,7 +275,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -337,7 +337,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -424,7 +424,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh b/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh index 7a104987b1..7f77fec504 100755 --- a/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hireswarw_snowfall_stats.sh @@ -120,7 +120,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -207,7 +207,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -269,7 +269,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -328,7 +328,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -402,7 +402,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh b/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh index 4d1840331d..9b0ab3b19a 100755 --- a/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hireswarwmem2_grid2obs_stats.sh @@ -116,7 +116,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -195,7 +195,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -263,7 +263,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -329,7 +329,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -406,7 +406,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh b/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh index 15611e667e..f12321f806 100755 --- a/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh +++ b/scripts/stats/cam/exevs_hireswarwmem2_precip_stats.sh @@ -121,7 +121,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -211,7 +211,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -275,7 +275,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -336,7 +336,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -423,7 +423,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh b/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh index 271eb42ea0..5915edfdbc 100755 --- a/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hireswarwmem2_snowfall_stats.sh @@ -120,7 +120,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -207,7 +207,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -269,7 +269,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -328,7 +328,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -402,7 +402,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh b/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh index d82ffb6c66..6da3bf9b0e 100755 --- a/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hireswfv3_grid2obs_stats.sh @@ -116,7 +116,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -195,7 +195,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -263,7 +263,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -329,7 +329,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -406,7 +406,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh b/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh index 8cac5b9916..6616b4df5c 100755 --- a/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh +++ b/scripts/stats/cam/exevs_hireswfv3_precip_stats.sh @@ -121,7 +121,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -211,7 +211,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -275,7 +275,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -336,7 +336,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -423,7 +423,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh b/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh index 055511fbfe..ac2b776247 100755 --- a/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hireswfv3_snowfall_stats.sh @@ -120,7 +120,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -207,7 +207,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -269,7 +269,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -328,7 +328,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -402,7 +402,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh b/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh index 6f625e729c..13d3ca595e 100755 --- a/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_hrrr_grid2obs_stats.sh @@ -116,7 +116,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -195,7 +195,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -263,7 +263,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -329,7 +329,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -406,7 +406,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_hrrr_precip_stats.sh b/scripts/stats/cam/exevs_hrrr_precip_stats.sh index 0482a9b7d4..4a98fce57b 100755 --- a/scripts/stats/cam/exevs_hrrr_precip_stats.sh +++ b/scripts/stats/cam/exevs_hrrr_precip_stats.sh @@ -121,7 +121,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -210,7 +210,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -275,7 +275,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -337,7 +337,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -424,7 +424,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh b/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh index 4b3316c999..2336a7d5a9 100755 --- a/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_hrrr_snowfall_stats.sh @@ -120,7 +120,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -207,7 +207,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -269,7 +269,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -328,7 +328,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -402,7 +402,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh b/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh index f7a56c1da9..ff98a692e0 100755 --- a/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh +++ b/scripts/stats/cam/exevs_namnest_grid2obs_stats.sh @@ -116,7 +116,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -195,7 +195,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -263,7 +263,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -329,7 +329,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -406,7 +406,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_namnest_precip_stats.sh b/scripts/stats/cam/exevs_namnest_precip_stats.sh index 2e44aaaae0..c785ae61e2 100755 --- a/scripts/stats/cam/exevs_namnest_precip_stats.sh +++ b/scripts/stats/cam/exevs_namnest_precip_stats.sh @@ -122,7 +122,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -213,7 +213,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -277,7 +277,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -338,7 +338,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -425,7 +425,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi diff --git a/scripts/stats/cam/exevs_namnest_snowfall_stats.sh b/scripts/stats/cam/exevs_namnest_snowfall_stats.sh index 35852b01ea..98dc92feee 100755 --- a/scripts/stats/cam/exevs_namnest_snowfall_stats.sh +++ b/scripts/stats/cam/exevs_namnest_snowfall_stats.sh @@ -120,7 +120,7 @@ fi # Copy Reformat Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -207,7 +207,7 @@ fi # Copy Generate Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -269,7 +269,7 @@ fi # Copy Gather Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -328,7 +328,7 @@ fi # Copy Gather 2 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done @@ -402,7 +402,7 @@ if [ "$vhr" -ge "$last_cyc" ]; then # Copy Gather 3 Output to Main Directory for CHILD_DIR in ${DATA}/${VERIF_CASE}/METplus_output/workdirs/${job_type}/*; do - cp -ru $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/METplus_output/. export err=$?; err_chk done fi From 8ddb098e1148ac3bce369b13824f9f012eea6051 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Mon, 28 Oct 2024 16:13:12 +0000 Subject: [PATCH 13/25] Bugzilla 1547: Run child processes in separate working directories (precip prep jobs) --- scripts/prep/cam/exevs_hireswarw_precip_prep.sh | 10 ++++++++++ .../prep/cam/exevs_hireswarwmem2_precip_prep.sh | 10 ++++++++++ scripts/prep/cam/exevs_hireswfv3_precip_prep.sh | 10 ++++++++++ scripts/prep/cam/exevs_hrrr_precip_prep.sh | 10 ++++++++++ scripts/prep/cam/exevs_namnest_precip_prep.sh | 10 ++++++++++ ush/cam/cam_create_child_workdirs.py | 16 +++++++++++++--- ush/cam/cam_create_output_dirs.py | 11 +++++++---- ush/cam/cam_prep_precip_create_job_script.py | 6 ++++-- 8 files changed, 74 insertions(+), 9 deletions(-) diff --git a/scripts/prep/cam/exevs_hireswarw_precip_prep.sh b/scripts/prep/cam/exevs_hireswarw_precip_prep.sh index 3c89db0333..918d8ac351 100755 --- a/scripts/prep/cam/exevs_hireswarw_precip_prep.sh +++ b/scripts/prep/cam/exevs_hireswarw_precip_prep.sh @@ -51,6 +51,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run all Hires Window ARW precip/prep jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/prep_job_scripts/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/prep_job_scripts/job* |wc -l) @@ -81,6 +85,12 @@ else done fi +# Copy Prep Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/data/workdirs/*; do + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/data/. + export err=$?; err_chk +done + for NEST in "conus" "ak" "pr" "hi"; do export NEST=$NEST source $config diff --git a/scripts/prep/cam/exevs_hireswarwmem2_precip_prep.sh b/scripts/prep/cam/exevs_hireswarwmem2_precip_prep.sh index f2fa3b9607..ab92c8baab 100755 --- a/scripts/prep/cam/exevs_hireswarwmem2_precip_prep.sh +++ b/scripts/prep/cam/exevs_hireswarwmem2_precip_prep.sh @@ -51,6 +51,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run all Hires Window ARW Member 2 precip/prep jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/prep_job_scripts/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/prep_job_scripts/job* |wc -l) @@ -81,6 +85,12 @@ else done fi +# Copy Prep Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/data/workdirs/*; do + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/data/. + export err=$?; err_chk +done + for NEST in "conus" "ak" "pr" "hi"; do export NEST=$NEST source $config diff --git a/scripts/prep/cam/exevs_hireswfv3_precip_prep.sh b/scripts/prep/cam/exevs_hireswfv3_precip_prep.sh index eedf62b9f2..d32efdbdd7 100755 --- a/scripts/prep/cam/exevs_hireswfv3_precip_prep.sh +++ b/scripts/prep/cam/exevs_hireswfv3_precip_prep.sh @@ -51,6 +51,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run all Hires Window FV3 precip/prep jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/prep_job_scripts/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/prep_job_scripts/job* |wc -l) @@ -81,6 +85,12 @@ else done fi +# Copy Prep Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/data/workdirs/*; do + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/data/. + export err=$?; err_chk +done + for NEST in "conus" "ak" "pr" "hi"; do export NEST=$NEST source $config diff --git a/scripts/prep/cam/exevs_hrrr_precip_prep.sh b/scripts/prep/cam/exevs_hrrr_precip_prep.sh index 0ace840380..038948dca0 100755 --- a/scripts/prep/cam/exevs_hrrr_precip_prep.sh +++ b/scripts/prep/cam/exevs_hrrr_precip_prep.sh @@ -50,6 +50,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run all HRRR precip/prep jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/prep_job_scripts/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/prep_job_scripts/job* |wc -l) @@ -80,6 +84,12 @@ else done fi +# Copy Prep Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/data/workdirs/*; do + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/data/. + export err=$?; err_chk +done + for NEST in "conus" "ak"; do export NEST=$NEST source $config diff --git a/scripts/prep/cam/exevs_namnest_precip_prep.sh b/scripts/prep/cam/exevs_namnest_precip_prep.sh index 78da06213e..94077e532a 100755 --- a/scripts/prep/cam/exevs_namnest_precip_prep.sh +++ b/scripts/prep/cam/exevs_namnest_precip_prep.sh @@ -50,6 +50,10 @@ if [ $USE_CFP = YES ]; then export err=$?; err_chk fi +# Create Working Directories +python $USHevs/cam/cam_create_child_workdirs.py +export err=$?; err_chk + # Run all NAM Nest precip/prep jobs chmod u+x ${DATA}/${VERIF_CASE}/${STEP}/prep_job_scripts/* ncount_job=$(ls -l ${DATA}/${VERIF_CASE}/${STEP}/prep_job_scripts/job* |wc -l) @@ -80,6 +84,12 @@ else done fi +# Copy Prep Output to Main Directory +for CHILD_DIR in ${DATA}/${VERIF_CASE}/data/workdirs/*; do + cp -ruv $CHILD_DIR/* ${DATA}/${VERIF_CASE}/data/. + export err=$?; err_chk +done + for NEST in "conus" "ak" "pr" "hi"; do export NEST=$NEST source $config diff --git a/ush/cam/cam_create_child_workdirs.py b/ush/cam/cam_create_child_workdirs.py index 1426de6093..e05e9c3c32 100644 --- a/ush/cam/cam_create_child_workdirs.py +++ b/ush/cam/cam_create_child_workdirs.py @@ -17,8 +17,14 @@ job_type = os.environ['job_type'] if STEP == 'prep': - print( - "WARNING: Child working directories are not being made for prep." + jobdir = os.path.join( + DATA, VERIF_CASE, STEP, 'prep_job_scripts' + ) + outdir = os.path.join( + DATA, VERIF_CASE, 'data' + ) + workdirs = os.path.join( + outdir, 'workdirs' ) elif STEP == 'stats': jobdir = os.path.join( @@ -65,7 +71,11 @@ '\"*workdirs*\"', '-not', '-path', '\"*job*\"', '-exec', 'mkdir', '-p', os.path.join(workdir,'{}'), '\\;' ]) - if STEP == "stats": + if STEP == "prep": + print( + "Done making working directories for child prcoesses." + ) + elif STEP == "stats": print( "Done making working directories for child processes " + f"({job_type} jobs)." diff --git a/ush/cam/cam_create_output_dirs.py b/ush/cam/cam_create_output_dirs.py index d6ac03aefc..151bd0dfc7 100644 --- a/ush/cam/cam_create_output_dirs.py +++ b/ush/cam/cam_create_output_dirs.py @@ -162,11 +162,11 @@ working_dir_list = [] COMOUT_dir_list = [] if STEP == 'prep': + working_output_base_dir = os.path.join( + DATA, VERIF_CASE + ) + working_dir_list.append(working_output_base_dir) if VERIF_CASE == 'grid2obs': - working_output_base_dir = os.path.join( - DATA, VERIF_CASE - ) - working_dir_list.append(working_output_base_dir) working_dir_list.append(os.path.join( working_output_base_dir, 'data', NEST+'.'+vdate_dt.strftime('%Y%m%d') @@ -183,6 +183,9 @@ COMOUT, NEST+'.'+vdate_dt.strftime('%Y%m%d') )) + working_dir_list.append(os.path.join( + working_output_base_dir, 'data', 'workdirs' + )) elif STEP == 'stats': if VERIF_CASE == 'precip': if job_type == 'reformat': diff --git a/ush/cam/cam_prep_precip_create_job_script.py b/ush/cam/cam_prep_precip_create_job_script.py index 3b474e5b1c..b24afbf6a8 100644 --- a/ush/cam/cam_prep_precip_create_job_script.py +++ b/ush/cam/cam_prep_precip_create_job_script.py @@ -56,7 +56,8 @@ VDATEm = VDATEHOURm.strftime('%Y%m%d') VHOURm = VDATEHOURm.strftime('%H') COMOUTobs = os.path.join( - DATA, VERIF_CASE, 'data', 'ccpa', f'ccpa.{VDATEm}' + DATA, VERIF_CASE, 'data', 'workdirs', f'job{njob}', + 'ccpa', f'ccpa.{VDATEm}' ) if not os.path.isfile(os.path.join( COMOUTobs, @@ -109,7 +110,8 @@ VMINm = VDATEHOURm.strftime('%M') VSECm = VDATEHOURm.strftime('%S') COMOUTobs = os.path.join( - DATA, VERIF_CASE, 'data', 'mrms', f'mrms.{VDATEm}' + DATA, VERIF_CASE, 'data', 'workdirs', f'job{njob}', + 'mrms', f'mrms.{VDATEm}' ) outfilename=f'mrms.t{VHOURm}z.{OBS_ACC}h.{NEST}.gb2' if not os.path.isfile(os.path.join( From 1b49174e5a7214c55f6893b061d22b419bd5dcbc Mon Sep 17 00:00:00 2001 From: marcel caron Date: Tue, 29 Oct 2024 17:02:05 +0000 Subject: [PATCH 14/25] use cfp + reduce resources --- .../scripts/prep/cam/jevs_cam_hireswarw_precip_prep.sh | 6 +++--- .../scripts/prep/cam/jevs_cam_hireswarwmem2_precip_prep.sh | 6 +++--- .../scripts/prep/cam/jevs_cam_hireswfv3_precip_prep.sh | 6 +++--- dev/drivers/scripts/prep/cam/jevs_cam_hrrr_precip_prep.sh | 6 +++--- .../scripts/prep/cam/jevs_cam_namnest_precip_prep.sh | 6 +++--- .../prep/cam/jevs_cam_hireswarw_precip_prep_vhr_master.ecf | 6 +++--- .../cam/jevs_cam_hireswarwmem2_precip_prep_vhr_master.ecf | 6 +++--- .../prep/cam/jevs_cam_hireswfv3_precip_prep_vhr_master.ecf | 6 +++--- .../prep/cam/jevs_cam_hrrr_precip_prep_vhr_master.ecf | 6 +++--- .../prep/cam/jevs_cam_namnest_precip_prep_vhr_master.ecf | 6 +++--- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/dev/drivers/scripts/prep/cam/jevs_cam_hireswarw_precip_prep.sh b/dev/drivers/scripts/prep/cam/jevs_cam_hireswarw_precip_prep.sh index fbabdcfb3a..d34acb74de 100755 --- a/dev/drivers/scripts/prep/cam/jevs_cam_hireswarw_precip_prep.sh +++ b/dev/drivers/scripts/prep/cam/jevs_cam_hireswarw_precip_prep.sh @@ -5,7 +5,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=00:15:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:ompthreads=1:mem=128GB #PBS -l debug=true set -x @@ -21,8 +21,8 @@ export SENDDBN_NTC= export job=${PBS_JOBNAME:-jevs_cam_hireswarw_precip_prep} export jobid=$job.${PBS_JOBID:-$$} export SITE=$(cat /etc/cluster_name) -export USE_CFP=NO -export nproc=128 +export USE_CFP=YES +export nproc=3 # General Verification Settings export NET="evs" diff --git a/dev/drivers/scripts/prep/cam/jevs_cam_hireswarwmem2_precip_prep.sh b/dev/drivers/scripts/prep/cam/jevs_cam_hireswarwmem2_precip_prep.sh index 3a129e9398..10e556af01 100755 --- a/dev/drivers/scripts/prep/cam/jevs_cam_hireswarwmem2_precip_prep.sh +++ b/dev/drivers/scripts/prep/cam/jevs_cam_hireswarwmem2_precip_prep.sh @@ -5,7 +5,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=00:15:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:ompthreads=1:mem=128GB #PBS -l debug=true set -x @@ -21,8 +21,8 @@ export SENDDBN_NTC= export job=${PBS_JOBNAME:-jevs_cam_hireswarwmem2_precip_prep} export jobid=$job.${PBS_JOBID:-$$} export SITE=$(cat /etc/cluster_name) -export USE_CFP=NO -export nproc=128 +export USE_CFP=YES +export nproc=3 # General Verification Settings export NET="evs" diff --git a/dev/drivers/scripts/prep/cam/jevs_cam_hireswfv3_precip_prep.sh b/dev/drivers/scripts/prep/cam/jevs_cam_hireswfv3_precip_prep.sh index 5c4eed042e..f8868ff390 100755 --- a/dev/drivers/scripts/prep/cam/jevs_cam_hireswfv3_precip_prep.sh +++ b/dev/drivers/scripts/prep/cam/jevs_cam_hireswfv3_precip_prep.sh @@ -5,7 +5,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=00:15:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:ompthreads=1:mem=128GB #PBS -l debug=true set -x @@ -21,8 +21,8 @@ export SENDDBN_NTC= export job=${PBS_JOBNAME:-jevs_cam_hireswfv3_precip_prep} export jobid=$job.${PBS_JOBID:-$$} export SITE=$(cat /etc/cluster_name) -export USE_CFP=NO -export nproc=128 +export USE_CFP=YES +export nproc=3 # General Verification Settings export NET="evs" diff --git a/dev/drivers/scripts/prep/cam/jevs_cam_hrrr_precip_prep.sh b/dev/drivers/scripts/prep/cam/jevs_cam_hrrr_precip_prep.sh index d82fcebc18..d96850fb06 100755 --- a/dev/drivers/scripts/prep/cam/jevs_cam_hrrr_precip_prep.sh +++ b/dev/drivers/scripts/prep/cam/jevs_cam_hrrr_precip_prep.sh @@ -5,7 +5,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=00:15:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:ompthreads=1:mem=128GB #PBS -l debug=true set -x @@ -21,8 +21,8 @@ export SENDDBN_NTC= export job=${PBS_JOBNAME:-jevs_cam_hrrr_precip_prep} export jobid=$job.${PBS_JOBID:-$$} export SITE=$(cat /etc/cluster_name) -export USE_CFP=NO -export nproc=128 +export USE_CFP=YES +export nproc=3 # General Verification Settings export NET="evs" diff --git a/dev/drivers/scripts/prep/cam/jevs_cam_namnest_precip_prep.sh b/dev/drivers/scripts/prep/cam/jevs_cam_namnest_precip_prep.sh index 0fa37eb6eb..77e3697282 100755 --- a/dev/drivers/scripts/prep/cam/jevs_cam_namnest_precip_prep.sh +++ b/dev/drivers/scripts/prep/cam/jevs_cam_namnest_precip_prep.sh @@ -5,7 +5,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=00:15:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:ompthreads=1:mem=128GB #PBS -l debug=true set -x @@ -21,8 +21,8 @@ export SENDDBN_NTC= export job=${PBS_JOBNAME:-jevs_cam_namnest_precip_prep} export jobid=$job.${PBS_JOBID:-$$} export SITE=$(cat /etc/cluster_name) -export USE_CFP=NO -export nproc=128 +export USE_CFP=YES +export nproc=3 # General Verification Settings export NET="evs" diff --git a/ecf/scripts/prep/cam/jevs_cam_hireswarw_precip_prep_vhr_master.ecf b/ecf/scripts/prep/cam/jevs_cam_hireswarw_precip_prep_vhr_master.ecf index aba92ab012..a202eb5d87 100755 --- a/ecf/scripts/prep/cam/jevs_cam_hireswarw_precip_prep_vhr_master.ecf +++ b/ecf/scripts/prep/cam/jevs_cam_hireswarw_precip_prep_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:ompthreads=1:mem=128GB #PBS -l debug=true export model=evs @@ -42,13 +42,13 @@ module list ############################################################ # Specify environment variables ############################################################ -export nproc=128 +export nproc=3 if [ -n "%VHR:%" ]; then export vhr=${vhr:-%VHR:%} else export vhr=00 fi -export USE_CFP=NO +export USE_CFP=YES export NET="evs" export RUN="atmos" export VERIF_CASE="precip" diff --git a/ecf/scripts/prep/cam/jevs_cam_hireswarwmem2_precip_prep_vhr_master.ecf b/ecf/scripts/prep/cam/jevs_cam_hireswarwmem2_precip_prep_vhr_master.ecf index 56692614e0..f3eadbd227 100755 --- a/ecf/scripts/prep/cam/jevs_cam_hireswarwmem2_precip_prep_vhr_master.ecf +++ b/ecf/scripts/prep/cam/jevs_cam_hireswarwmem2_precip_prep_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:ompthreads=1:mem=128GB #PBS -l debug=true export model=evs @@ -42,13 +42,13 @@ module list ############################################################ # Specify environment variables ############################################################ -export nproc=128 +export nproc=3 if [ -n "%VHR:%" ]; then export vhr=${vhr:-%VHR:%} else export vhr=00 fi -export USE_CFP=NO +export USE_CFP=YES export NET="evs" export RUN="atmos" export VERIF_CASE="precip" diff --git a/ecf/scripts/prep/cam/jevs_cam_hireswfv3_precip_prep_vhr_master.ecf b/ecf/scripts/prep/cam/jevs_cam_hireswfv3_precip_prep_vhr_master.ecf index b5c1c2576c..990a968b55 100755 --- a/ecf/scripts/prep/cam/jevs_cam_hireswfv3_precip_prep_vhr_master.ecf +++ b/ecf/scripts/prep/cam/jevs_cam_hireswfv3_precip_prep_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:ompthreads=1:mem=128GB #PBS -l debug=true export model=evs @@ -42,13 +42,13 @@ module list ############################################################ # Specify environment variables ############################################################ -export nproc=128 +export nproc=3 if [ -n "%VHR:%" ]; then export vhr=${vhr:-%VHR:%} else export vhr=00 fi -export USE_CFP=NO +export USE_CFP=YES export NET="evs" export RUN="atmos" export VERIF_CASE="precip" diff --git a/ecf/scripts/prep/cam/jevs_cam_hrrr_precip_prep_vhr_master.ecf b/ecf/scripts/prep/cam/jevs_cam_hrrr_precip_prep_vhr_master.ecf index 5f0b5b7bdf..bf8980700e 100755 --- a/ecf/scripts/prep/cam/jevs_cam_hrrr_precip_prep_vhr_master.ecf +++ b/ecf/scripts/prep/cam/jevs_cam_hrrr_precip_prep_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:ompthreads=1:mem=128GB #PBS -l debug=true export model=evs @@ -42,13 +42,13 @@ module list ############################################################ # Specify environment variables ############################################################ -export nproc=128 +export nproc=3 if [ -n "%VHR:%" ]; then export vhr=${vhr:-%VHR:%} else export vhr=00 fi -export USE_CFP=NO +export USE_CFP=YES export NET="evs" export RUN="atmos" export VERIF_CASE="precip" diff --git a/ecf/scripts/prep/cam/jevs_cam_namnest_precip_prep_vhr_master.ecf b/ecf/scripts/prep/cam/jevs_cam_namnest_precip_prep_vhr_master.ecf index e65a34e8a1..eddc6d00be 100755 --- a/ecf/scripts/prep/cam/jevs_cam_namnest_precip_prep_vhr_master.ecf +++ b/ecf/scripts/prep/cam/jevs_cam_namnest_precip_prep_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:10:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:ompthreads=1:mem=128GB #PBS -l debug=true export model=evs @@ -42,13 +42,13 @@ module list ############################################################ # Specify environment variables ############################################################ -export nproc=128 +export nproc=3 if [ -n "%VHR:%" ]; then export vhr=${vhr:-%VHR:%} else export vhr=00 fi -export USE_CFP=NO +export USE_CFP=YES export NET="evs" export RUN="atmos" export VERIF_CASE="precip" From c2ab45d281ce566c89e82c1867274af0352323a6 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Wed, 30 Oct 2024 20:01:52 +0000 Subject: [PATCH 15/25] Look in main output dir for firewx mask input --- ush/cam/cam_stats_grid2obs_create_job_script.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/cam/cam_stats_grid2obs_create_job_script.py b/ush/cam/cam_stats_grid2obs_create_job_script.py index 683deb6ada..b5e0844d4a 100644 --- a/ush/cam/cam_stats_grid2obs_create_job_script.py +++ b/ush/cam/cam_stats_grid2obs_create_job_script.py @@ -276,7 +276,8 @@ 'dependent_vars': { 'names': ['MASK_POLY_LIST'], 'values': [( - f'{MET_PLUS_OUT}/{VERIF_TYPE}/genvxmask/{NEST}.' + f'{DATA}/{VERIF_CASE}/METplus_output/{VERIF_TYPE}' + + f'/genvxmask/{NEST}.' + '${VDATE}'+ f'/{NEST}_t{VHOUR}z_'+ 'f${FHR}.nc' )], } From 601f0ca922c3d355283bf95285318cb066c8ef66 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Thu, 31 Oct 2024 15:27:02 +0000 Subject: [PATCH 16/25] Adjust walltimes + match ecf with dev walltimes --- .../scripts/stats/cam/jevs_cam_hireswfv3_grid2obs_stats.sh | 2 +- .../scripts/stats/cam/jevs_cam_namnest_grid2obs_stats.sh | 2 +- .../stats/cam/jevs_cam_hireswarw_grid2obs_stats_vhr_master.ecf | 2 +- .../cam/jevs_cam_hireswarwmem2_grid2obs_stats_vhr_master.ecf | 2 +- .../stats/cam/jevs_cam_hireswfv3_grid2obs_stats_vhr_master.ecf | 2 +- .../stats/cam/jevs_cam_namnest_grid2obs_stats_vhr_master.ecf | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_grid2obs_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_grid2obs_stats.sh index 660cb01650..73c0de19d7 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_grid2obs_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_grid2obs_stats.sh @@ -4,7 +4,7 @@ #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=00:45:00 +#PBS -l walltime=01:00:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=150GB #PBS -l debug=true diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_namnest_grid2obs_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_namnest_grid2obs_stats.sh index 82a2b00b32..562c8f0638 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_namnest_grid2obs_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_namnest_grid2obs_stats.sh @@ -4,7 +4,7 @@ #PBS -S /bin/bash #PBS -q dev #PBS -A VERF-DEV -#PBS -l walltime=02:15:00 +#PBS -l walltime=02:50:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=256GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswarw_grid2obs_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswarw_grid2obs_stats_vhr_master.ecf index 2d06f5553f..b66c2eb11e 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswarw_grid2obs_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswarw_grid2obs_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:30:00 +#PBS -l walltime=00:45:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_grid2obs_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_grid2obs_stats_vhr_master.ecf index 1456dc8da1..0c0264030b 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_grid2obs_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_grid2obs_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:30:00 +#PBS -l walltime=00:45:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_grid2obs_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_grid2obs_stats_vhr_master.ecf index 31bad5183f..8b31bc9817 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_grid2obs_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_grid2obs_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:30:00 +#PBS -l walltime=01:00:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_namnest_grid2obs_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_namnest_grid2obs_stats_vhr_master.ecf index 26b5ca721b..5873ecdc0c 100755 --- a/ecf/scripts/stats/cam/jevs_cam_namnest_grid2obs_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_namnest_grid2obs_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=02:15:00 +#PBS -l walltime=02:50:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=256GB #PBS -l debug=true From a07e669418f09fbba1fedcedf4a0e9c3aadd8981 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Thu, 31 Oct 2024 16:40:52 +0000 Subject: [PATCH 17/25] Match ecf with dev driver walltime --- .../stats/cam/jevs_cam_hireswfv3_precip_stats_vhr_master.ecf | 2 +- ecf/scripts/stats/cam/jevs_cam_hrrr_precip_stats_vhr_master.ecf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_precip_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_precip_stats_vhr_master.ecf index 7ee5d8ddec..fbf2ea27ce 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_precip_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_precip_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:55:00 +#PBS -l walltime=01:00:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_hrrr_precip_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hrrr_precip_stats_vhr_master.ecf index 3370cbca8c..4039e0c5c4 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hrrr_precip_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hrrr_precip_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=02:25:00 +#PBS -l walltime=02:00:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=256GB #PBS -l debug=true From 14fc4c679124185faa98f43cf985b8af646c254c Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 1 Nov 2024 14:07:15 +0000 Subject: [PATCH 18/25] set and adjust mem allocation --- dev/drivers/scripts/stats/cam/jevs_cam_namnest_precip_stats.sh | 2 +- .../stats/cam/jevs_cam_namnest_precip_stats_vhr_master.ecf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_namnest_precip_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_namnest_precip_stats.sh index 22f3c1479c..8b144d90ce 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_namnest_precip_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_namnest_precip_stats.sh @@ -5,7 +5,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=01:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1 +#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=125GB #PBS -l debug=true set -x diff --git a/ecf/scripts/stats/cam/jevs_cam_namnest_precip_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_namnest_precip_stats_vhr_master.ecf index 34a48ee235..e20c3201e1 100755 --- a/ecf/scripts/stats/cam/jevs_cam_namnest_precip_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_namnest_precip_stats_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=01:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=125GB #PBS -l debug=true export model=evs From 00386046bb2593896ff662eef79c3a7f130b5298 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 1 Nov 2024 14:22:33 +0000 Subject: [PATCH 19/25] Match ecf with dev resource requests --- .../stats/cam/jevs_cam_hireswarw_radar_stats_vhr_master.ecf | 2 +- .../cam/jevs_cam_hireswarw_snowfall_stats_vhr_master.ecf | 2 +- .../cam/jevs_cam_hireswarwmem2_radar_stats_vhr_master.ecf | 2 +- .../jevs_cam_hireswarwmem2_snowfall_stats_vhr_master.ecf | 2 +- .../stats/cam/jevs_cam_hireswfv3_radar_stats_vhr_master.ecf | 2 +- .../cam/jevs_cam_hireswfv3_snowfall_stats_vhr_master.ecf | 2 +- .../stats/cam/jevs_cam_href_radar_stats_vhr_master.ecf | 2 +- .../stats/cam/jevs_cam_hrrr_radar_stats_vhr_master.ecf | 6 +++--- .../stats/cam/jevs_cam_hrrr_snowfall_stats_vhr_master.ecf | 2 +- .../stats/cam/jevs_cam_namnest_radar_stats_vhr_master.ecf | 2 +- .../cam/jevs_cam_namnest_snowfall_stats_vhr_master.ecf | 4 ++-- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswarw_radar_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswarw_radar_stats_vhr_master.ecf index df526d1422..9b65c54a1b 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswarw_radar_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswarw_radar_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:10:00 +#PBS -l walltime=00:30:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswarw_snowfall_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswarw_snowfall_stats_vhr_master.ecf index 88cd8ed6d7..9f9b5ccc39 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswarw_snowfall_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswarw_snowfall_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:35:00 +#PBS -l walltime=00:45:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats_vhr_master.ecf index 22991005e3..c3c97287c4 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:10:00 +#PBS -l walltime=00:30:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_snowfall_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_snowfall_stats_vhr_master.ecf index 0074287e4c..19c0473962 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_snowfall_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_snowfall_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:35:00 +#PBS -l walltime=00:45:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats_vhr_master.ecf index 01f797f0e1..080a0a6084 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:10:00 +#PBS -l walltime=00:30:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_snowfall_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_snowfall_stats_vhr_master.ecf index 2dd23bf4bd..aabd0727f5 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_snowfall_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_snowfall_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:35:00 +#PBS -l walltime=00:45:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_href_radar_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_href_radar_stats_vhr_master.ecf index 3540c0db88..31140ca995 100755 --- a/ecf/scripts/stats/cam/jevs_cam_href_radar_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_href_radar_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:15:00 +#PBS -l walltime=00:30:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=9:mem=500GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_hrrr_radar_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hrrr_radar_stats_vhr_master.ecf index c6476ea45d..3c537c51c3 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hrrr_radar_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hrrr_radar_stats_vhr_master.ecf @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:15:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=12:mem=500GB +#PBS -l walltime=00:30:00 +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB #PBS -l debug=true export model=evs @@ -53,7 +53,7 @@ export VERIF_CASE=radar export MODELNAME=hrrr export modsys=hrrr export USE_CFP=YES -export nproc=12 +export nproc=3 export OMP_NUM_THREADS=1 ############################################################ diff --git a/ecf/scripts/stats/cam/jevs_cam_hrrr_snowfall_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hrrr_snowfall_stats_vhr_master.ecf index 29ac3507d1..5ecd722f91 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hrrr_snowfall_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hrrr_snowfall_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=02:10:00 +#PBS -l walltime=02:45:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=256GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_namnest_radar_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_namnest_radar_stats_vhr_master.ecf index 50392312f0..cf0b77553f 100755 --- a/ecf/scripts/stats/cam/jevs_cam_namnest_radar_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_namnest_radar_stats_vhr_master.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:15:00 +#PBS -l walltime=00:30:00 #PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_namnest_snowfall_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_namnest_snowfall_stats_vhr_master.ecf index 62bbc60a96..0825486083 100755 --- a/ecf/scripts/stats/cam/jevs_cam_namnest_snowfall_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_namnest_snowfall_stats_vhr_master.ecf @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=01:15:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=500GB +#PBS -l walltime=01:30:00 +#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=128GB #PBS -l debug=true export model=evs From 0c95388c00764a00b0f538d2a86a95455cd6a01f Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 1 Nov 2024 14:57:24 +0000 Subject: [PATCH 20/25] Reduce mem allocation --- dev/drivers/scripts/stats/cam/jevs_cam_hireswarw_radar_stats.sh | 2 +- .../scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats.sh | 2 +- dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats.sh | 2 +- dev/drivers/scripts/stats/cam/jevs_cam_hrrr_radar_stats.sh | 2 +- dev/drivers/scripts/stats/cam/jevs_cam_namnest_radar_stats.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_hireswarw_radar_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_hireswarw_radar_stats.sh index 275e415250..ef65943ee2 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_hireswarw_radar_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_hireswarw_radar_stats.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=50GB #PBS -l debug=true diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats.sh index 85f81fa651..7e4ac50ee1 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=50GB #PBS -l debug=true diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats.sh index 544c2ca64b..f91e84776b 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=50GB #PBS -l debug=true diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_hrrr_radar_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_hrrr_radar_stats.sh index 760f316cb4..e5b02a193f 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_hrrr_radar_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_hrrr_radar_stats.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=50GB #PBS -l debug=true diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_namnest_radar_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_namnest_radar_stats.sh index e2d77c0d20..294d5b1e33 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_namnest_radar_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_namnest_radar_stats.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=00:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=50GB #PBS -l debug=true From d1859e042c04e9fffff661c9a2c87cd55da7a7a5 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 1 Nov 2024 14:58:47 +0000 Subject: [PATCH 21/25] Reduce mem allocation (ecf) --- .../stats/cam/jevs_cam_hireswarw_radar_stats_vhr_master.ecf | 2 +- .../stats/cam/jevs_cam_hireswarwmem2_radar_stats_vhr_master.ecf | 2 +- .../stats/cam/jevs_cam_hireswfv3_radar_stats_vhr_master.ecf | 2 +- ecf/scripts/stats/cam/jevs_cam_hrrr_radar_stats_vhr_master.ecf | 2 +- .../stats/cam/jevs_cam_namnest_radar_stats_vhr_master.ecf | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswarw_radar_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswarw_radar_stats_vhr_master.ecf index 9b65c54a1b..6a7c062396 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswarw_radar_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswarw_radar_stats_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=50GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats_vhr_master.ecf index c3c97287c4..19137b2834 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_radar_stats_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=50GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats_vhr_master.ecf index 080a0a6084..59f9d40331 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_radar_stats_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=50GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/stats/cam/jevs_cam_hrrr_radar_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_hrrr_radar_stats_vhr_master.ecf index 3c537c51c3..188d98eec6 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hrrr_radar_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hrrr_radar_stats_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=50GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/stats/cam/jevs_cam_namnest_radar_stats_vhr_master.ecf b/ecf/scripts/stats/cam/jevs_cam_namnest_radar_stats_vhr_master.ecf index cf0b77553f..aef7f3c05f 100755 --- a/ecf/scripts/stats/cam/jevs_cam_namnest_radar_stats_vhr_master.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_namnest_radar_stats_vhr_master.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=00:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=500GB +#PBS -l place=vscatter:exclhost,select=1:ncpus=3:mem=50GB #PBS -l debug=true export model=evs From dd80739b53f289b4bcb21ae59dc4f0496a475bc8 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 1 Nov 2024 19:38:22 +0000 Subject: [PATCH 22/25] Match ecf with dev driver allocations (plots) --- ecf/scripts/plots/cam/jevs_cam_headline_plots.ecf | 4 ++-- ecf/scripts/plots/cam/jevs_cam_precip_plots_last90days.ecf | 2 +- ecf/scripts/plots/cam/jevs_cam_snowfall_plots.ecf | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ecf/scripts/plots/cam/jevs_cam_headline_plots.ecf b/ecf/scripts/plots/cam/jevs_cam_headline_plots.ecf index 5671d2c4a1..62dabe93dc 100755 --- a/ecf/scripts/plots/cam/jevs_cam_headline_plots.ecf +++ b/ecf/scripts/plots/cam/jevs_cam_headline_plots.ecf @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:40:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=150GB +#PBS -l walltime=00:30:00 +#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=50GB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/plots/cam/jevs_cam_precip_plots_last90days.ecf b/ecf/scripts/plots/cam/jevs_cam_precip_plots_last90days.ecf index 3786bd5226..44b1d29396 100755 --- a/ecf/scripts/plots/cam/jevs_cam_precip_plots_last90days.ecf +++ b/ecf/scripts/plots/cam/jevs_cam_precip_plots_last90days.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=04:35:00 +#PBS -l walltime=04:30:00 #PBS -l place=vscatter:exclhost,select=4:ncpus=128:mem=200GB #PBS -l debug=true diff --git a/ecf/scripts/plots/cam/jevs_cam_snowfall_plots.ecf b/ecf/scripts/plots/cam/jevs_cam_snowfall_plots.ecf index fbe77c8043..b840cb2d82 100755 --- a/ecf/scripts/plots/cam/jevs_cam_snowfall_plots.ecf +++ b/ecf/scripts/plots/cam/jevs_cam_snowfall_plots.ecf @@ -3,7 +3,7 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=04:30:00 +#PBS -l walltime=05:30:00 #PBS -l place=vscatter:exclhost,select=4:ncpus=128:mem=150GB #PBS -l debug=true From c426889505b007c88c734eb9b47c6de4ffad456a Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 1 Nov 2024 19:39:09 +0000 Subject: [PATCH 23/25] Add memory allocation --- dev/drivers/scripts/plots/cam/jevs_cam_headline_plots.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/drivers/scripts/plots/cam/jevs_cam_headline_plots.sh b/dev/drivers/scripts/plots/cam/jevs_cam_headline_plots.sh index f0854c4fa9..69cdbcfa55 100755 --- a/dev/drivers/scripts/plots/cam/jevs_cam_headline_plots.sh +++ b/dev/drivers/scripts/plots/cam/jevs_cam_headline_plots.sh @@ -5,7 +5,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=00:30:00 -#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1 +#PBS -l place=vscatter:exclhost,select=1:ncpus=128:ompthreads=1:mem=50GB #PBS -l debug=true set -x From cf37bafec88af81a90f4c64317de84a9145f53c5 Mon Sep 17 00:00:00 2001 From: marcel caron Date: Fri, 1 Nov 2024 21:48:28 +0000 Subject: [PATCH 24/25] Increase mem allocation --- .../scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.sh | 2 +- ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.ecf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.sh b/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.sh index 2a2847a37d..3e09779d8a 100755 --- a/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.sh +++ b/dev/drivers/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.sh @@ -5,7 +5,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=8:00:00 -#PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=220GB +#PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=300GB #PBS -l debug=true set -x diff --git a/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.ecf b/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.ecf index 1170de7fc5..8c0f5418b6 100755 --- a/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.ecf +++ b/ecf/scripts/plots/cam/jevs_cam_grid2obs_plots_last31days.ecf @@ -4,7 +4,7 @@ #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% #PBS -l walltime=8:00:00 -#PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=220GB +#PBS -l place=vscatter:exclhost,select=12:ncpus=128:mem=300GB #PBS -l debug=true export model=evs From dfc000f9cc6aba3dc56ab9f29656ebec0851f03e Mon Sep 17 00:00:00 2001 From: Marcel Caron Date: Mon, 4 Nov 2024 15:42:55 +0000 Subject: [PATCH 25/25] Use single cpu + match ecf with dev resources --- .../scripts/stats/cam/jevs_cam_hireswarw_severe_stats.sh | 2 +- .../scripts/stats/cam/jevs_cam_hireswarwmem2_severe_stats.sh | 2 +- .../scripts/stats/cam/jevs_cam_hireswfv3_severe_stats.sh | 2 +- dev/drivers/scripts/stats/cam/jevs_cam_href_severe_stats.sh | 2 +- dev/drivers/scripts/stats/cam/jevs_cam_hrrr_severe_stats.sh | 2 +- .../scripts/stats/cam/jevs_cam_namnest_severe_stats.sh | 2 +- ecf/scripts/stats/cam/jevs_cam_hireswarw_severe_stats.ecf | 4 ++-- ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_severe_stats.ecf | 4 ++-- ecf/scripts/stats/cam/jevs_cam_hireswfv3_severe_stats.ecf | 4 ++-- ecf/scripts/stats/cam/jevs_cam_href_severe_stats.ecf | 4 ++-- ecf/scripts/stats/cam/jevs_cam_hrrr_severe_stats.ecf | 4 ++-- ecf/scripts/stats/cam/jevs_cam_namnest_severe_stats.ecf | 4 ++-- 12 files changed, 18 insertions(+), 18 deletions(-) diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_hireswarw_severe_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_hireswarw_severe_stats.sh index 1f0de18d48..e9a5efb20d 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_hireswarw_severe_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_hireswarw_severe_stats.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=exclhost,select=1:ncpus=5:mem=500MB +#PBS -l place=exclhost,select=1:ncpus=1:mem=500MB #PBS -l debug=true diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_hireswarwmem2_severe_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_hireswarwmem2_severe_stats.sh index a917016ef5..323e73232a 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_hireswarwmem2_severe_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_hireswarwmem2_severe_stats.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=exclhost,select=1:ncpus=5:mem=500MB +#PBS -l place=exclhost,select=1:ncpus=1:mem=500MB #PBS -l debug=true diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_severe_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_severe_stats.sh index 4a329cdced..f53a4db00b 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_severe_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_hireswfv3_severe_stats.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=exclhost,select=1:ncpus=5:mem=500MB +#PBS -l place=exclhost,select=1:ncpus=1:mem=500MB #PBS -l debug=true diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_href_severe_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_href_severe_stats.sh index ec22054cb4..24abc032a0 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_href_severe_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_href_severe_stats.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=exclhost,select=1:ncpus=5:mem=500MB +#PBS -l place=exclhost,select=1:ncpus=1:mem=500MB #PBS -l debug=true diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_hrrr_severe_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_hrrr_severe_stats.sh index 8cb987cc69..2ed558e6eb 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_hrrr_severe_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_hrrr_severe_stats.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=0:30:00 -#PBS -l place=exclhost,select=1:ncpus=5:mem=500MB +#PBS -l place=exclhost,select=1:ncpus=1:mem=500MB #PBS -l debug=true diff --git a/dev/drivers/scripts/stats/cam/jevs_cam_namnest_severe_stats.sh b/dev/drivers/scripts/stats/cam/jevs_cam_namnest_severe_stats.sh index 7e236a02c6..7fd6689ec5 100755 --- a/dev/drivers/scripts/stats/cam/jevs_cam_namnest_severe_stats.sh +++ b/dev/drivers/scripts/stats/cam/jevs_cam_namnest_severe_stats.sh @@ -4,7 +4,7 @@ #PBS -q dev #PBS -A VERF-DEV #PBS -l walltime=00:30:00 -#PBS -l place=exclhost,select=1:ncpus=5:mem=500MB +#PBS -l place=exclhost,select=1:ncpus=1:mem=500MB #PBS -l debug=true diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswarw_severe_stats.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswarw_severe_stats.ecf index a427ca0a6f..eddb1268d3 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswarw_severe_stats.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswarw_severe_stats.ecf @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:10:00 -#PBS -l place=shared,select=1:ncpus=5:mem=5GB +#PBS -l walltime=00:30:00 +#PBS -l place=shared,select=1:ncpus=1:mem=500MB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_severe_stats.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_severe_stats.ecf index 7d595c5cef..f35bbfe7d5 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_severe_stats.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswarwmem2_severe_stats.ecf @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:10:00 -#PBS -l place=shared,select=1:ncpus=5:mem=5GB +#PBS -l walltime=00:30:00 +#PBS -l place=shared,select=1:ncpus=1:mem=500MB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_severe_stats.ecf b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_severe_stats.ecf index eca270dfe2..ab964b74ca 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hireswfv3_severe_stats.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hireswfv3_severe_stats.ecf @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:10:00 -#PBS -l place=shared,select=1:ncpus=5:mem=5GB +#PBS -l walltime=00:30:00 +#PBS -l place=shared,select=1:ncpus=1:mem=500MB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/stats/cam/jevs_cam_href_severe_stats.ecf b/ecf/scripts/stats/cam/jevs_cam_href_severe_stats.ecf index 6a3bd3ab09..6ac9481e79 100755 --- a/ecf/scripts/stats/cam/jevs_cam_href_severe_stats.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_href_severe_stats.ecf @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:10:00 -#PBS -l place=shared,select=1:ncpus=5:mem=5GB +#PBS -l walltime=00:30:00 +#PBS -l place=shared,select=1:ncpus=1:mem=500MB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/stats/cam/jevs_cam_hrrr_severe_stats.ecf b/ecf/scripts/stats/cam/jevs_cam_hrrr_severe_stats.ecf index 2d98ad6c1a..c9068850b1 100755 --- a/ecf/scripts/stats/cam/jevs_cam_hrrr_severe_stats.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_hrrr_severe_stats.ecf @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:10:00 -#PBS -l place=shared,select=1:ncpus=5:mem=5GB +#PBS -l walltime=00:30:00 +#PBS -l place=shared,select=1:ncpus=1:mem=500MB #PBS -l debug=true export model=evs diff --git a/ecf/scripts/stats/cam/jevs_cam_namnest_severe_stats.ecf b/ecf/scripts/stats/cam/jevs_cam_namnest_severe_stats.ecf index 6391f1dea8..7a770a1f77 100755 --- a/ecf/scripts/stats/cam/jevs_cam_namnest_severe_stats.ecf +++ b/ecf/scripts/stats/cam/jevs_cam_namnest_severe_stats.ecf @@ -3,8 +3,8 @@ #PBS -S /bin/bash #PBS -q %QUEUE% #PBS -A %PROJ%-%PROJENVIR% -#PBS -l walltime=00:10:00 -#PBS -l place=shared,select=1:ncpus=5:mem=5GB +#PBS -l walltime=00:30:00 +#PBS -l place=shared,select=1:ncpus=1:mem=500MB #PBS -l debug=true export model=evs