From ba647d27cc5c957d0effac1e9cb0649974d686f9 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 4 Apr 2023 07:17:23 -0600 Subject: [PATCH 01/11] Bugfix #2082 main_v5.0 regrid.convert/censor_thresh/censor_val (#2101) --- .github/jobs/setup_and_run_diff.py | 6 +- docs/Users_Guide/glossary.rst | 165 +++++++++++++++++ docs/Users_Guide/wrappers.rst | 87 +++++++++ .../test_ensemble_stat_wrapper.py | 17 +- .../gen_ens_prod/test_gen_ens_prod_wrapper.py | 124 +++++++------ .../wrappers/grid_diag/test_grid_diag.py | 167 +++++++++++++++++ .../grid_stat/test_grid_stat_wrapper.py | 17 +- .../wrappers/mode/test_mode_wrapper.py | 18 +- .../pytests/wrappers/mtd/test_mtd_wrapper.py | 174 ++++++++++++++++++ .../point_stat/test_point_stat_wrapper.py | 18 +- .../series_analysis/test_series_analysis.py | 39 ++++ metplus/util/met_config.py | 15 +- metplus/wrappers/command_builder.py | 3 + metplus/wrappers/gen_ens_prod_wrapper.py | 8 +- metplus/wrappers/mtd_wrapper.py | 5 +- metplus/wrappers/tc_pairs_wrapper.py | 7 +- .../EnsembleStat/EnsembleStat.conf | 3 + .../GenEnsProd/GenEnsProd.conf | 3 + .../met_tool_wrapper/GridDiag/GridDiag.conf | 3 + .../met_tool_wrapper/GridStat/GridStat.conf | 7 + .../use_cases/met_tool_wrapper/MODE/MODE.conf | 3 + parm/use_cases/met_tool_wrapper/MTD/MTD.conf | 8 + .../Point2Grid/Point2Grid.conf | 8 +- .../met_tool_wrapper/PointStat/PointStat.conf | 5 + .../SeriesAnalysis/SeriesAnalysis.conf | 3 + .../met_tool_wrapper/TCRMW/TCRMW.conf | 3 + ...rod_fcstHRRR_fcstOnly_SurrogateSevere.conf | 2 + 27 files changed, 838 insertions(+), 80 deletions(-) diff --git a/.github/jobs/setup_and_run_diff.py b/.github/jobs/setup_and_run_diff.py index 995c71f2b4..1a4c908b14 100755 --- a/.github/jobs/setup_and_run_diff.py +++ b/.github/jobs/setup_and_run_diff.py @@ -4,6 +4,7 @@ import sys import subprocess import shlex +import re from docker_utils import VERSION_EXT, get_branch_name @@ -33,8 +34,9 @@ output_data_branch = os.environ.get('GITHUB_BASE_REF') else: branch_name = get_branch_name() - if branch_name.startswith('main_v'): - output_data_branch = branch_name + match = re.match(r'.*(main_v\d+\.\d+).*', branch_name) + if match: + output_data_branch = match.group(1) else: output_data_branch = 'develop' diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 8fcefc43ff..4a1f8564da 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -10253,3 +10253,168 @@ METplus Configuration Glossary Specify the value for 'seeps_p1_thresh' in the MET configuration file for PointStat. | *Used by:* PointStat + + ENSEMBLE_STAT_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for EnsembleStat. + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for EnsembleStat. + + | *Used by:* EnsembleStat + + ENSEMBLE_STAT_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for EnsembleStat. + + | *Used by:* EnsembleStat + + GEN_ENS_PROD_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for GenEnsProd. + + | *Used by:* GenEnsProd + + GEN_ENS_PROD_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for GenEnsProd. + + | *Used by:* GenEnsProd + + GEN_ENS_PROD_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for GenEnsProd. + + | *Used by:* GenEnsProd + + GRID_DIAG_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for GridDiag. + + | *Used by:* GridDiag + + GRID_DIAG_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for GridDiag. + + | *Used by:* GridDiag + + GRID_DIAG_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for GridDiag. + + | *Used by:* GridDiag + + GRID_STAT_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + MODE_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for MODE. + + | *Used by:* MODE + + MTD_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for MTD. + + | *Used by:* MTD + + MTD_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for MTD. + + | *Used by:* MTD + + MTD_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for MTD. + + | *Used by:* MTD + + POINT2GRID_REGRID_WIDTH + Specify the value for 'regrid.width' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_REGRID_VLD_THRESH + Specify the value for 'regrid.vld_thresh' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_REGRID_SHAPE + Specify the value for 'regrid.shape' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT2GRID_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for Point2Grid. + + | *Used by:* Point2Grid + + POINT_STAT_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + SERIES_ANALYSIS_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for SeriesAnalysis. + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for SeriesAnalysis. + + | *Used by:* SeriesAnalysis + + SERIES_ANALYSIS_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for SeriesAnalysis. + + | *Used by:* SeriesAnalysis + + TC_RMW_REGRID_CONVERT + Specify the value for 'regrid.convert' in the MET configuration file for TCRMW. + + | *Used by:* TCRMW + + TC_RMW_REGRID_CENSOR_THRESH + Specify the value for 'regrid.censor_thresh' in the MET configuration file for TCRMW. + + | *Used by:* TCRMW + + TC_RMW_REGRID_CENSOR_VAL + Specify the value for 'regrid.censor_val' in the MET configuration file for TCRMW. + + | *Used by:* TCRMW diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 118e1d8723..170a073d30 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -204,6 +204,9 @@ METplus Configuration | :term:`ENSEMBLE_STAT_REGRID_WIDTH` | :term:`ENSEMBLE_STAT_REGRID_VLD_THRESH` | :term:`ENSEMBLE_STAT_REGRID_SHAPE` +| :term:`ENSEMBLE_STAT_REGRID_CONVERT` +| :term:`ENSEMBLE_STAT_REGRID_CENSOR_THRESH` +| :term:`ENSEMBLE_STAT_REGRID_CENSOR_VAL` | :term:`ENSEMBLE_STAT_CONFIG_FILE` | :term:`ENSEMBLE_STAT_MET_OBS_ERR_TABLE` | :term:`ENSEMBLE_STAT_N_MEMBERS` @@ -394,6 +397,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`ENSEMBLE_STAT_REGRID_TO_GRID` - regrid.to_grid + * - :term:`ENSEMBLE_STAT_REGRID_CONVERT` + - regrid.convert + * - :term:`ENSEMBLE_STAT_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`ENSEMBLE_STAT_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_CENSOR_THRESH}** @@ -1050,6 +1059,9 @@ METplus Configuration | :term:`GEN_ENS_PROD_REGRID_WIDTH` | :term:`GEN_ENS_PROD_REGRID_VLD_THRESH` | :term:`GEN_ENS_PROD_REGRID_SHAPE` +| :term:`GEN_ENS_PROD_REGRID_CONVERT` +| :term:`GEN_ENS_PROD_REGRID_CENSOR_THRESH` +| :term:`GEN_ENS_PROD_REGRID_CENSOR_VAL` | :term:`GEN_ENS_PROD_CENSOR_THRESH` | :term:`GEN_ENS_PROD_CENSOR_VAL` | :term:`GEN_ENS_PROD_CAT_THRESH` @@ -1174,6 +1186,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`GEN_ENS_PROD_REGRID_TO_GRID` - regrid.to_grid + * - :term:`GEN_ENS_PROD_REGRID_CONVERT` + - regrid.convert + * - :term:`GEN_ENS_PROD_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`GEN_ENS_PROD_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_CENSOR_THRESH}** @@ -2669,6 +2687,9 @@ METplus Configuration | :term:`GRID_DIAG_REGRID_VLD_THRESH` | :term:`GRID_DIAG_REGRID_SHAPE` | :term:`GRID_DIAG_REGRID_TO_GRID` +| :term:`GRID_DIAG_REGRID_CONVERT` +| :term:`GRID_DIAG_REGRID_CENSOR_THRESH` +| :term:`GRID_DIAG_REGRID_CENSOR_VAL` | :term:`GRID_DIAG_DESC` | :term:`GRID_DIAG_SKIP_IF_OUTPUT_EXISTS` | :term:`GRID_DIAG_RUNTIME_FREQ` @@ -2725,6 +2746,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`GRID_DIAG_REGRID_TO_GRID` - regrid.to_grid + * - :term:`GRID_DIAG_REGRID_CONVERT` + - regrid.convert + * - :term:`GRID_DIAG_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`GRID_DIAG_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_CENSOR_THRESH}** @@ -2826,6 +2853,9 @@ METplus Configuration | :term:`GRID_STAT_REGRID_WIDTH` | :term:`GRID_STAT_REGRID_VLD_THRESH` | :term:`GRID_STAT_REGRID_SHAPE` +| :term:`GRID_STAT_REGRID_CONVERT` +| :term:`GRID_STAT_REGRID_CENSOR_THRESH` +| :term:`GRID_STAT_REGRID_CENSOR_VAL` | :term:`GRID_STAT_CLIMO_CDF_BINS` | :term:`GRID_STAT_CLIMO_CDF_CENTER_BINS` | :term:`GRID_STAT_CLIMO_CDF_WRITE_BINS` @@ -3025,6 +3055,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`GRID_STAT_REGRID_TO_GRID` - regrid.to_grid + * - :term:`GRID_STAT_REGRID_CONVERT` + - regrid.convert + * - :term:`GRID_STAT_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`GRID_STAT_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_FCST_FIELD}** @@ -3944,6 +3980,9 @@ METplus Configuration | :term:`MODE_REGRID_WIDTH` | :term:`MODE_REGRID_VLD_THRESH` | :term:`MODE_REGRID_SHAPE` +| :term:`MODE_REGRID_CONVERT` +| :term:`MODE_REGRID_CENSOR_THRESH` +| :term:`MODE_REGRID_CENSOR_VAL` | :term:`MODE_CONFIG_FILE` | :term:`FCST_MODE_INPUT_DATATYPE` | :term:`OBS_MODE_INPUT_DATATYPE` @@ -4105,6 +4144,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`MODE_REGRID_TO_GRID` - regrid.to_grid + * - :term:`MODE_REGRID_CONVERT` + - regrid.convert + * - :term:`MODE_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`MODE_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_GRID_RES}** @@ -4639,6 +4684,9 @@ METplus Configuration | :term:`MTD_REGRID_WIDTH` | :term:`MTD_REGRID_VLD_THRESH` | :term:`MTD_REGRID_SHAPE` +| :term:`MTD_REGRID_CONVERT` +| :term:`MTD_REGRID_CENSOR_THRESH` +| :term:`MTD_REGRID_CENSOR_VAL` | :term:`MTD_MET_CONFIG_OVERRIDES` | :term:`FCST_MTD_IS_PROB` | :term:`FCST_MTD_PROB_IN_GRIB_PDS` @@ -4730,6 +4778,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`MTD_REGRID_TO_GRID` - regrid.to_grid + * - :term:`MTD_REGRID_CONVERT` + - regrid.convert + * - :term:`MTD_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`MTD_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_FCST_FILE_TYPE}** @@ -5684,6 +5738,12 @@ METplus Configuration | :term:`POINT2GRID_QC_FLAGS` | :term:`POINT2GRID_ADP` | :term:`POINT2GRID_REGRID_METHOD` +| :term:`POINT2GRID_REGRID_WIDTH` +| :term:`POINT2GRID_REGRID_VLD_THRESH` +| :term:`POINT2GRID_REGRID_SHAPE` +| :term:`POINT2GRID_REGRID_CONVERT` +| :term:`POINT2GRID_REGRID_CENSOR_THRESH` +| :term:`POINT2GRID_REGRID_CENSOR_VAL` | :term:`POINT2GRID_GAUSSIAN_DX` | :term:`POINT2GRID_GAUSSIAN_RADIUS` | :term:`POINT2GRID_PROB_CAT_THRESH` @@ -5728,6 +5788,9 @@ Configuration | :term:`POINT_STAT_REGRID_WIDTH` | :term:`POINT_STAT_REGRID_VLD_THRESH` | :term:`POINT_STAT_REGRID_SHAPE` +| :term:`POINT_STAT_REGRID_CONVERT` +| :term:`POINT_STAT_REGRID_CENSOR_THRESH` +| :term:`POINT_STAT_REGRID_CENSOR_VAL` | :term:`POINT_STAT_MASK_GRID` | :term:`POINT_STAT_MASK_POLY` | :term:`POINT_STAT_MASK_SID` @@ -5913,6 +5976,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`POINT_STAT_REGRID_TO_GRID` - regrid.to_grid + * - :term:`POINT_STAT_REGRID_CONVERT` + - regrid.convert + * - :term:`POINT_STAT_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`POINT_STAT_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_FCST_FIELD}** @@ -6400,6 +6469,9 @@ METplus Configuration | :term:`SERIES_ANALYSIS_REGRID_WIDTH` | :term:`SERIES_ANALYSIS_REGRID_VLD_THRESH` | :term:`SERIES_ANALYSIS_REGRID_SHAPE` +| :term:`SERIES_ANALYSIS_REGRID_CONVERT` +| :term:`SERIES_ANALYSIS_REGRID_CENSOR_THRESH` +| :term:`SERIES_ANALYSIS_REGRID_CENSOR_VAL` | :term:`SERIES_ANALYSIS_STAT_LIST` | :term:`SERIES_ANALYSIS_IS_PAIRED` | :term:`SERIES_ANALYSIS_CUSTOM_LOOP_LIST` @@ -6563,6 +6635,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.vld_thresh * - :term:`SERIES_ANALYSIS_REGRID_TO_GRID` - regrid.to_grid + * - :term:`SERIES_ANALYSIS_REGRID_CONVERT` + - regrid.convert + * - :term:`SERIES_ANALYSIS_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`SERIES_ANALYSIS_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_CAT_THRESH}** @@ -8678,6 +8756,9 @@ METplus Configuration | :term:`TC_RMW_REGRID_WIDTH` | :term:`TC_RMW_REGRID_VLD_THRESH` | :term:`TC_RMW_REGRID_SHAPE` +| :term:`TC_RMW_REGRID_CONVERT` +| :term:`TC_RMW_REGRID_CENSOR_THRESH` +| :term:`TC_RMW_REGRID_CENSOR_VAL` | :term:`TC_RMW_N_RANGE` | :term:`TC_RMW_N_AZIMUTH` | :term:`TC_RMW_MAX_RANGE_KM` @@ -8885,6 +8966,12 @@ see :ref:`How METplus controls MET config file settings`. - regrid.width * - :term:`TC_RMW_REGRID_VLD_THRESH` - regrid.vld_thresh + * - :term:`TC_RMW_REGRID_CONVERT` + - regrid.convert + * - :term:`TC_RMW_REGRID_CENSOR_THRESH` + - regrid.censor_thresh + * - :term:`TC_RMW_REGRID_CENSOR_VAL` + - regrid.censor_val **${METPLUS_N_RANGE}** diff --git a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py index 3737e8ec72..bc5ac37554 100644 --- a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py @@ -244,14 +244,29 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, }, {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + ({'ENSEMBLE_STAT_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'ENSEMBLE_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'ENSEMBLE_STAT_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + ({'ENSEMBLE_STAT_REGRID_TO_GRID': 'FCST', 'ENSEMBLE_STAT_REGRID_METHOD': 'NEAREST', 'ENSEMBLE_STAT_REGRID_WIDTH': '1', 'ENSEMBLE_STAT_REGRID_VLD_THRESH': '0.5', 'ENSEMBLE_STAT_REGRID_SHAPE': 'SQUARE', + 'ENSEMBLE_STAT_REGRID_CONVERT': '2*x', + 'ENSEMBLE_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', + 'ENSEMBLE_STAT_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' - 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' ), 'REGRID_TO_GRID': 'FCST'}), diff --git a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py index 2bcba73a17..572130433f 100644 --- a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -87,74 +87,90 @@ def handle_input_dir(config): }, {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), # 8 + ({'GEN_ENS_PROD_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + # 9 + ({'GEN_ENS_PROD_REGRID_CENSOR_THRESH': '>12000,<5000', }, + { + 'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + # 10 + ({'GEN_ENS_PROD_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + # 11 ({'GEN_ENS_PROD_REGRID_TO_GRID': 'FCST', 'GEN_ENS_PROD_REGRID_METHOD': 'NEAREST', 'GEN_ENS_PROD_REGRID_WIDTH': '1', 'GEN_ENS_PROD_REGRID_VLD_THRESH': '0.5', 'GEN_ENS_PROD_REGRID_SHAPE': 'SQUARE', + 'GEN_ENS_PROD_REGRID_CONVERT': '2*x', + 'GEN_ENS_PROD_REGRID_CENSOR_THRESH': '>12000,<5000', + 'GEN_ENS_PROD_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' - 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' )}), - # 9 + # 12 ({'GEN_ENS_PROD_CLIMO_MEAN_INPUT_TEMPLATE': '/some/path/climo/filename.nc', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {file_name = ["/some/path/climo/filename.nc"];}', }), - # 10 + # 13 ({'GEN_ENS_PROD_CLIMO_STDEV_INPUT_TEMPLATE': '/some/path/climo/stdfile.nc', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {file_name = ["/some/path/climo/stdfile.nc"];}', }), - # 11 ensemble_flag + # 14 ensemble_flag ({'GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {latlon = FALSE;}'}), - # 12 + # 15 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {mean = FALSE;}'}), - # 13 + # 16 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_STDEV': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {stdev = FALSE;}'}), - # 14 + # 17 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MINUS': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {minus = FALSE;}'}), - # 15 + # 18 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_PLUS': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {plus = FALSE;}'}), - # 16 + # 19 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MIN': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {min = FALSE;}'}), - # 17 + # 20 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_MAX': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {max = FALSE;}'}), - # 18 + # 21 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_RANGE': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {range = FALSE;}'}), - # 19 + # 22 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_VLD_COUNT': 'FALSE', }, { 'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {vld_count = FALSE;}'}), - # 20 + # 23 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_FREQUENCY': 'FALSE', }, { 'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {frequency = FALSE;}'}), - # 21 + # 24 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_NEP': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {nep = FALSE;}'}), - # 22 + # 25 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_NMEP': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {nmep = FALSE;}'}), - # 23 + # 26 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {climo = FALSE;}'}), - # 24 + # 27 ({'GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDP': 'FALSE', }, {'METPLUS_ENSEMBLE_FLAG_DICT': 'ensemble_flag = {climo_cdp = FALSE;}'}), - # 25 + # 28 ({ 'GEN_ENS_PROD_ENSEMBLE_FLAG_LATLON': 'FALSE', 'GEN_ENS_PROD_ENSEMBLE_FLAG_MEAN': 'FALSE', @@ -180,40 +196,40 @@ def handle_input_dir(config): 'frequency = FALSE;nep = FALSE;' 'nmep = FALSE;climo = FALSE;' 'climo_cdp = FALSE;}')}), - # 26 + # 29 ({'GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', }, {'METPLUS_CLIMO_MEAN_DICT': ('climo_mean = {file_name = ' '["/some/climo_mean/file.txt"];}'),}), - # 27 + # 30 ({'GEN_ENS_PROD_CLIMO_MEAN_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {field = [{name="CLM_NAME"; level="(0,0,*,*)";}];}'}), - # 28 + # 31 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_METHOD': 'NEAREST', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {method = NEAREST;}}'}), - # 29 + # 32 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_WIDTH': '1', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {width = 1;}}'}), - # 30 + # 33 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_VLD_THRESH': '0.5', }, { 'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {vld_thresh = 0.5;}}'}), - # 31 + # 34 ({'GEN_ENS_PROD_CLIMO_MEAN_REGRID_SHAPE': 'SQUARE', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {regrid = {shape = SQUARE;}}'}), - # 32 + # 35 ({'GEN_ENS_PROD_CLIMO_MEAN_TIME_INTERP_METHOD': 'NEAREST', }, { 'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {time_interp_method = NEAREST;}'}), - # 33 + # 36 ({'GEN_ENS_PROD_CLIMO_MEAN_MATCH_MONTH': 'True', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {match_month = TRUE;}'}), - # 34 + # 37 ({'GEN_ENS_PROD_CLIMO_MEAN_DAY_INTERVAL': '30', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {day_interval = 30;}'}), - # 35 + # 38 ({'GEN_ENS_PROD_CLIMO_MEAN_HOUR_INTERVAL': '12', }, {'METPLUS_CLIMO_MEAN_DICT': 'climo_mean = {hour_interval = 12;}'}), - # 36 + # 39 ({ 'GEN_ENS_PROD_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', 'GEN_ENS_PROD_CLIMO_MEAN_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', @@ -234,42 +250,42 @@ def handle_input_dir(config): 'time_interp_method = NEAREST;' 'match_month = TRUE;day_interval = 30;' 'hour_interval = 12;}')}), - # 37 climo stdev + # 40 climo stdev ({'GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME': '/some/climo_stdev/file.txt', }, {'METPLUS_CLIMO_STDEV_DICT': ('climo_stdev = {file_name = ' '["/some/climo_stdev/file.txt"];}')}), - # 38 + # 41 ({'GEN_ENS_PROD_CLIMO_STDEV_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {field = [{name="CLM_NAME"; level="(0,0,*,*)";}];}'}), - # 39 + # 42 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_METHOD': 'NEAREST', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {method = NEAREST;}}'}), - # 40 + # 43 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_WIDTH': '1', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {width = 1;}}'}), - # 41 + # 44 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_VLD_THRESH': '0.5', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {vld_thresh = 0.5;}}'}), - # 42 + # 45 ({'GEN_ENS_PROD_CLIMO_STDEV_REGRID_SHAPE': 'SQUARE', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {regrid = {shape = SQUARE;}}'}), - # 43 + # 46 ({'GEN_ENS_PROD_CLIMO_STDEV_TIME_INTERP_METHOD': 'NEAREST', }, { 'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {time_interp_method = NEAREST;}'}), - # 44 + # 47 ({'GEN_ENS_PROD_CLIMO_STDEV_MATCH_MONTH': 'True', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {match_month = TRUE;}'}), - # 45 + # 48 ({'GEN_ENS_PROD_CLIMO_STDEV_DAY_INTERVAL': '30', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {day_interval = 30;}'}), - # 46 + # 49 ({'GEN_ENS_PROD_CLIMO_STDEV_HOUR_INTERVAL': '12', }, {'METPLUS_CLIMO_STDEV_DICT': 'climo_stdev = {hour_interval = 12;}'}), - # 47 + # 50 ({ 'GEN_ENS_PROD_CLIMO_STDEV_FILE_NAME': '/some/climo_stdev/file.txt', 'GEN_ENS_PROD_CLIMO_STDEV_FIELD': '{name="CLM_NAME"; level="(0,0,*,*)";}', @@ -290,16 +306,16 @@ def handle_input_dir(config): 'time_interp_method = NEAREST;' 'match_month = TRUE;day_interval = 30;' 'hour_interval = 12;}')}), - # 48 + # 51 ({'GEN_ENS_PROD_NBRHD_PROB_WIDTH': '5', }, {'METPLUS_NBRHD_PROB_DICT': 'nbrhd_prob = {width = [5];}'}), - # 49 + # 52 ({'GEN_ENS_PROD_NBRHD_PROB_SHAPE': 'circle', }, {'METPLUS_NBRHD_PROB_DICT': 'nbrhd_prob = {shape = CIRCLE;}'}), - # 50 + # 53 ({'GEN_ENS_PROD_NBRHD_PROB_VLD_THRESH': '0.0', }, {'METPLUS_NBRHD_PROB_DICT': 'nbrhd_prob = {vld_thresh = 0.0;}'}), - # 51 + # 54 ({ 'GEN_ENS_PROD_NBRHD_PROB_WIDTH': '5', 'GEN_ENS_PROD_NBRHD_PROB_SHAPE': 'CIRCLE', @@ -311,26 +327,26 @@ def handle_input_dir(config): 'vld_thresh = 0.0;}' ) }), - # 52 + # 55 ({'GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH': '0.0', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {vld_thresh = 0.0;}'}), - # 53 + # 56 ({'GEN_ENS_PROD_NMEP_SMOOTH_SHAPE': 'circle', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {shape = CIRCLE;}'}), - # 54 + # 57 ({'GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_DX': '81.27', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {gaussian_dx = 81.27;}'}), - # 55 + # 58 ({'GEN_ENS_PROD_NMEP_SMOOTH_GAUSSIAN_RADIUS': '120', }, { 'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {gaussian_radius = 120;}'}), - # 56 + # 59 ({'GEN_ENS_PROD_NMEP_SMOOTH_TYPE_METHOD': 'GAUSSIAN', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {type = [{method = GAUSSIAN;}];}'}), - # 57 + # 60 ({'GEN_ENS_PROD_NMEP_SMOOTH_TYPE_WIDTH': '1', }, {'METPLUS_NMEP_SMOOTH_DICT': 'nmep_smooth = {type = [{width = 1;}];}'}), - # 58 + # 61 ({ 'GEN_ENS_PROD_NMEP_SMOOTH_VLD_THRESH': '0.0', 'GEN_ENS_PROD_NMEP_SMOOTH_SHAPE': 'circle', @@ -346,13 +362,13 @@ def handle_input_dir(config): 'type = [{method = GAUSSIAN;width = 1;}];}' ) }), - # 59 + # 62 ({'GEN_ENS_PROD_ENS_MEMBER_IDS': '1,2,3,4', }, {'METPLUS_ENS_MEMBER_IDS': 'ens_member_ids = ["1", "2", "3", "4"];'}), - # 60 + # 63 ({'GEN_ENS_PROD_CONTROL_ID': '0', }, {'METPLUS_CONTROL_ID': 'control_id = "0";'}), - # 61 + # 64 ({'GEN_ENS_PROD_NORMALIZE': 'CLIMO_STD_ANOM', }, {'METPLUS_NORMALIZE': 'normalize = CLIMO_STD_ANOM;'}), diff --git a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py index 72e3a878d7..40c6e2eee2 100644 --- a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py +++ b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py @@ -9,6 +9,54 @@ from metplus.wrappers.grid_diag_wrapper import GridDiagWrapper +time_fmt = '%Y%m%d%H' +run_times = ['2016092900', '2016092906'] +data_dir = '/some/path/data' + +data_name_1 = 'APCP' +data_level = 'L0' +data_options_1 = 'n_bins = 55; range = [0, 55];' +data_name_2 = 'PWAT' +data_options_2 = 'n_bins = 35; range = [35, 70];' +data_fmt = ('data = {field = [ ' + f'{{ name="{data_name_1}"; level="{data_level}"; {data_options_1} }},' + f'{{ name="{data_name_2}"; level="{data_level}"; {data_options_2} }}' + ' ];}') + + +def set_minimum_config_settings(config): + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'GridDiag') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', time_fmt) + config.set('config', 'INIT_BEG', run_times[0]) + config.set('config', 'INIT_END', run_times[-1]) + config.set('config', 'INIT_INCREMENT', '6H') + config.set('config', 'LEAD_SEQ', '141, 144, 147') + config.set('config', 'GRID_DIAG_RUNTIME_FREQ', + 'RUN_ONCE_PER_INIT_OR_VALID') + config.set('config', 'GRID_DIAG_CONFIG_FILE', + '{PARM_BASE}/met_config/GridDiagConfig_wrapped') + config.set('config', 'GRID_DIAG_INPUT_DIR', data_dir) + config.set('config', 'GRID_DIAG_INPUT_TEMPLATE', + ('gfs.subset.t00z.pgrb2.0p25.f{lead?fmt=%H}, ' + 'gfs.subset.t00z.pgrb2.0p25.f{lead?fmt=%H}')) + config.set('config', 'GRID_DIAG_OUTPUT_DIR', + '{OUTPUT_BASE}/grid_diag/output') + config.set('config', 'GRID_DIAG_OUTPUT_TEMPLATE', + 'grid_diag.{valid?fmt=%Y%m%d%H}.nc') + config.set('config', 'BOTH_VAR1_NAME', data_name_1) + config.set('config', 'BOTH_VAR1_LEVELS', data_level) + config.set('config', 'BOTH_VAR1_OPTIONS', data_options_1) + config.set('config', 'BOTH_VAR2_NAME', data_name_2) + config.set('config', 'BOTH_VAR2_LEVELS', data_level) + config.set('config', 'BOTH_VAR2_OPTIONS', data_options_2) + @pytest.mark.parametrize( 'time_info, expected_subset', [ @@ -188,3 +236,122 @@ def test_get_config_file(metplus_config): config.set('config', 'GRID_DIAG_CONFIG_FILE', fake_config_name) wrapper = GridDiagWrapper(config) assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name + + +@pytest.mark.parametrize( + 'config_overrides, env_var_values', [ + ({'GRID_DIAG_DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + + ({'GRID_DIAG_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_CENSOR_THRESH': 'censor_thresh = [>12000, <5000];'}), + + ({'GRID_DIAG_CENSOR_VAL': '12000, 5000', }, + {'METPLUS_CENSOR_VAL': 'censor_val = [12000, 5000];'}), + + ({'GRID_DIAG_MASK_GRID': 'FULL', }, + {'METPLUS_MASK_DICT': 'mask = {grid = "FULL";}'}), + + ({'GRID_DIAG_MASK_POLY': 'MET_BASE/poly/EAST.poly', }, + {'METPLUS_MASK_DICT': 'mask = {poly = "MET_BASE/poly/EAST.poly";}'}), + + ({'GRID_DIAG_MASK_GRID': 'FULL', + 'GRID_DIAG_MASK_POLY': 'MET_BASE/poly/EAST.poly',}, + {'METPLUS_MASK_DICT': ('mask = {grid = "FULL";' + 'poly = "MET_BASE/poly/EAST.poly";}')}), + + ({'GRID_DIAG_REGRID_TO_GRID': 'FCST',}, + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + + ({'GRID_DIAG_REGRID_METHOD': 'NEAREST',}, + {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), + + ({'GRID_DIAG_REGRID_WIDTH': '1',}, + {'METPLUS_REGRID_DICT': 'regrid = {width = 1;}'}), + + ({'GRID_DIAG_REGRID_VLD_THRESH': '0.5',}, + {'METPLUS_REGRID_DICT': 'regrid = {vld_thresh = 0.5;}'}), + + ({'GRID_DIAG_REGRID_SHAPE': 'SQUARE',}, + {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + + ({'GRID_DIAG_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'GRID_DIAG_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'GRID_DIAG_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + + ({'GRID_DIAG_REGRID_TO_GRID': 'FCST', + 'GRID_DIAG_REGRID_METHOD': 'NEAREST', + 'GRID_DIAG_REGRID_WIDTH': '1', + 'GRID_DIAG_REGRID_VLD_THRESH': '0.5', + 'GRID_DIAG_REGRID_SHAPE': 'SQUARE', + 'GRID_DIAG_REGRID_CONVERT': '2*x', + 'GRID_DIAG_REGRID_CENSOR_THRESH': '>12000,<5000', + 'GRID_DIAG_REGRID_CENSOR_VAL': '12000,5000', + }, + {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' + )}), + ] +) +@pytest.mark.wrapper +def test_grid_diag(metplus_config, config_overrides, env_var_values): + config = metplus_config + set_minimum_config_settings(config) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = GridDiagWrapper(config) + assert wrapper.isOK + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR') + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + + expected_cmds = [ + (f"{app_path} -data {file_list_dir}/grid_diag_files_input0" + "_init_20160929000000_valid_ALL_lead_ALL.txt " + f"-data {file_list_dir}/grid_diag_files_input1" + "_init_20160929000000_valid_ALL_lead_ALL.txt " + f"-config {config_file} -out {out_dir}/grid_diag.all.nc {verbosity}"), + (f"{app_path} -data {file_list_dir}/grid_diag_files_input0" + "_init_20160929060000_valid_ALL_lead_ALL.txt " + f"-data {file_list_dir}/grid_diag_files_input1" + "_init_20160929060000_valid_ALL_lead_ALL.txt " + f"-config {config_file} -out {out_dir}/grid_diag.all.nc {verbosity}"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + assert len(all_cmds) == len(expected_cmds) + + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd + + # check that environment variables were set properly + # including deprecated env vars (not in wrapper env var keys) + for env_var_key in env_var_keys: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + actual_value = match.split('=', 1)[1] + if env_var_key == 'METPLUS_DATA_DICT': + assert actual_value == data_fmt + else: + assert env_var_values.get(env_var_key, '') == actual_value diff --git a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py index 458c515e3c..0b1416a2ea 100644 --- a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py @@ -193,14 +193,29 @@ def test_handle_climo_file_variables(metplus_config, config_overrides, }, {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + ({'GRID_STAT_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'GRID_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'GRID_STAT_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + ({'GRID_STAT_REGRID_TO_GRID': 'FCST', 'GRID_STAT_REGRID_METHOD': 'NEAREST', 'GRID_STAT_REGRID_WIDTH': '1', 'GRID_STAT_REGRID_VLD_THRESH': '0.5', 'GRID_STAT_REGRID_SHAPE': 'SQUARE', + 'GRID_STAT_REGRID_CONVERT': '2*x', + 'GRID_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', + 'GRID_STAT_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' - 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' ), 'REGRID_TO_GRID': 'FCST'}), diff --git a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py index fb9759face..07ff4bb110 100644 --- a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py +++ b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py @@ -93,14 +93,30 @@ def set_minimum_config_settings(config): }, {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + ({'MODE_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'MODE_REGRID_CENSOR_THRESH': '>12000,<5000', }, + { + 'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'MODE_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + ({'MODE_REGRID_TO_GRID': 'FCST', 'MODE_REGRID_METHOD': 'NEAREST', 'MODE_REGRID_WIDTH': '1', 'MODE_REGRID_VLD_THRESH': '0.5', 'MODE_REGRID_SHAPE': 'SQUARE', + 'MODE_REGRID_CONVERT': '2*x', + 'MODE_REGRID_CENSOR_THRESH': '>12000,<5000', + 'MODE_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' - 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' ), 'REGRID_TO_GRID': 'FCST'}), diff --git a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py index 420e6d40fc..2101ccfefc 100644 --- a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py +++ b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py @@ -7,6 +7,19 @@ from metplus.wrappers.mtd_wrapper import MTDWrapper +fcst_dir = '/some/path/fcst' +obs_dir = '/some/path/obs' +fcst_name = 'APCP' +fcst_level = 'A03' +fcst_thresh = 'gt12.7' +obs_name = 'APCP_03' +obs_level_no_quotes = '(*,*)' +obs_level = f'"{obs_level_no_quotes}"' +obs_thresh = 'gt12.7' +fcst_fmt = f'field = {{ name="{fcst_name}"; level="{fcst_level}"; cat_thresh=[ gt12.7 ]; }};' +obs_fmt = (f'field = {{ name="{obs_name}"; ' + f'level="{obs_level_no_quotes}"; cat_thresh=[ gt12.7 ]; }};') + def get_test_data_dir(config, subdir): return os.path.join(config.getdir('METPLUS_BASE'), @@ -32,6 +45,167 @@ def mtd_wrapper(metplus_config, lead_seq=None): return MTDWrapper(config) +def set_minimum_config_settings(config): + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'MTD') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') + config.set('config', 'INIT_BEG', '2005080700') + config.set('config', 'INIT_END', '2005080700') + config.set('config', 'INIT_INCREMENT', '12H') + config.set('config', 'LEAD_SEQ', '6H, 9H, 12H') + config.set('config', 'MTD_CONFIG_FILE', + '{PARM_BASE}/met_config/MTDConfig_wrapped') + config.set('config', 'FCST_MTD_INPUT_DIR', fcst_dir) + config.set('config', 'OBS_MTD_INPUT_DIR', obs_dir) + config.set('config', 'FCST_MTD_INPUT_TEMPLATE', + '{init?fmt=%Y%m%d%H}/fcst_file_F{lead?fmt=%3H}') + config.set('config', 'OBS_MTD_INPUT_TEMPLATE', + '{valid?fmt=%Y%m%d%H}/obs_file') + config.set('config', 'MTD_OUTPUT_DIR', + '{OUTPUT_BASE}/MTD/output') + config.set('config', 'MTD_OUTPUT_TEMPLATE', '{valid?fmt=%Y%m%d%H}') + + config.set('config', 'FCST_VAR1_NAME', fcst_name) + config.set('config', 'FCST_VAR1_LEVELS', fcst_level) + config.set('config', 'FCST_VAR1_THRESH', fcst_thresh) + config.set('config', 'OBS_VAR1_NAME', obs_name) + config.set('config', 'OBS_VAR1_LEVELS', obs_level) + config.set('config', 'OBS_VAR1_THRESH', obs_thresh) + + +@pytest.mark.parametrize( + 'config_overrides, env_var_values', [ + ({'MODEL': 'my_model'}, + {'METPLUS_MODEL': 'model = "my_model";'}), + + ({'MTD_DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + + ({'DESC': 'my_desc'}, + {'METPLUS_DESC': 'desc = "my_desc";'}), + + ({'OBTYPE': 'my_obtype'}, + {'METPLUS_OBTYPE': 'obtype = "my_obtype";'}), + + ({'MTD_REGRID_TO_GRID': 'FCST',}, + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + + ({'MTD_REGRID_METHOD': 'NEAREST',}, + {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), + + ({'MTD_REGRID_WIDTH': '1',}, + {'METPLUS_REGRID_DICT': 'regrid = {width = 1;}'}), + + ({'MTD_REGRID_VLD_THRESH': '0.5',}, + {'METPLUS_REGRID_DICT': 'regrid = {vld_thresh = 0.5;}'}), + + ({'MTD_REGRID_SHAPE': 'SQUARE',}, + {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + + ({'MTD_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'MTD_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'MTD_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + + ({'MTD_REGRID_TO_GRID': 'FCST', + 'MTD_REGRID_METHOD': 'NEAREST', + 'MTD_REGRID_WIDTH': '1', + 'MTD_REGRID_VLD_THRESH': '0.5', + 'MTD_REGRID_SHAPE': 'SQUARE', + 'MTD_REGRID_CONVERT': '2*x', + 'MTD_REGRID_CENSOR_THRESH': '>12000,<5000', + 'MTD_REGRID_CENSOR_VAL': '12000,5000', + }, + {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' + )}), + + ({'FCST_MTD_CONV_RADIUS': '40.0/grid_res'}, + {'METPLUS_FCST_CONV_RADIUS': 'conv_radius = 40.0/grid_res;'}), + + ({'OBS_MTD_CONV_RADIUS': '40.0/grid_res'}, + {'METPLUS_OBS_CONV_RADIUS': 'conv_radius = 40.0/grid_res;'}), + + ({'FCST_MTD_CONV_THRESH': '>=10.0'}, + {'METPLUS_FCST_CONV_THRESH': 'conv_thresh = >=10.0;'}), + + ({'OBS_MTD_CONV_THRESH': '>=10.0'}, + {'METPLUS_OBS_CONV_THRESH': 'conv_thresh = >=10.0;'}), + + ({'MTD_MIN_VOLUME': '1000'}, + {'METPLUS_MIN_VOLUME': 'min_volume = 1000;'}), + + ({'MTD_OUTPUT_PREFIX': 'my_output_prefix'}, + {'METPLUS_OUTPUT_PREFIX': 'output_prefix = "my_output_prefix";'}), + ] +) +@pytest.mark.wrapper +def test_mode_single_field(metplus_config, config_overrides, env_var_values): + config = metplus_config + + # set config variables needed to run + set_minimum_config_settings(config) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = MTDWrapper(config) + assert wrapper.isOK + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR') + config_file = wrapper.c_dict.get('CONFIG_FILE') + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + expected_cmds = [(f"{app_path} {verbosity} " + f"-fcst {file_list_dir}/" + f"20050807060000_mtd_fcst_{fcst_name}.txt " + f"-obs {file_list_dir}/" + f"20050807060000_mtd_obs_{obs_name}.txt " + f"-config {config_file} " + f"-outdir {out_dir}/2005080706"), + ] + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + + missing_env = [item for item in env_var_values + if item not in wrapper.WRAPPER_ENV_VAR_KEYS] + env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd + + # check that environment variables were set properly + # including deprecated env vars (not in wrapper env var keys) + for env_var_key in env_var_keys: + match = next((item for item in env_vars if + item.startswith(env_var_key)), None) + assert match is not None + value = match.split('=', 1)[1] + if env_var_key == 'METPLUS_FCST_FIELD': + assert value == fcst_fmt + elif env_var_key == 'METPLUS_OBS_FIELD': + assert value == obs_fmt + else: + assert env_var_values.get(env_var_key, '') == value + + @pytest.mark.wrapper def test_mtd_by_init_all_found(metplus_config): mw = mtd_wrapper(metplus_config, '1,2,3') diff --git a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py index 0293e84302..f537605b84 100755 --- a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py @@ -87,14 +87,30 @@ def test_met_dictionary_in_var_options(metplus_config): }, {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + ({'POINT_STAT_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'POINT_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', }, + { + 'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'POINT_STAT_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + ({'POINT_STAT_REGRID_TO_GRID': 'FCST', 'POINT_STAT_REGRID_METHOD': 'NEAREST', 'POINT_STAT_REGRID_WIDTH': '1', 'POINT_STAT_REGRID_VLD_THRESH': '0.5', 'POINT_STAT_REGRID_SHAPE': 'SQUARE', + 'POINT_STAT_REGRID_CONVERT': '2*x', + 'POINT_STAT_REGRID_CENSOR_THRESH': '>12000,<5000', + 'POINT_STAT_REGRID_CENSOR_VAL': '12000,5000', }, {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' - 'width = 1;vld_thresh = 0.5;shape = SQUARE;}' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' ), 'REGRID_TO_GRID': 'FCST'}), diff --git a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index 08a83e771c..e5815a5b4c 100644 --- a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -105,6 +105,45 @@ def set_minimum_config_settings(config): @pytest.mark.parametrize( 'config_overrides, env_var_values', [ + ({'SERIES_ANALYSIS_REGRID_TO_GRID': 'FCST', }, + {'METPLUS_REGRID_DICT': 'regrid = {to_grid = FCST;}'}), + + ({'SERIES_ANALYSIS_REGRID_METHOD': 'NEAREST',}, + {'METPLUS_REGRID_DICT': 'regrid = {method = NEAREST;}'}), + + ({'SERIES_ANALYSIS_REGRID_WIDTH': '1',}, + {'METPLUS_REGRID_DICT': 'regrid = {width = 1;}'}), + + ({'SERIES_ANALYSIS_REGRID_VLD_THRESH': '0.5',}, + {'METPLUS_REGRID_DICT': 'regrid = {vld_thresh = 0.5;}'}), + + ({'SERIES_ANALYSIS_REGRID_SHAPE': 'SQUARE',}, + {'METPLUS_REGRID_DICT': 'regrid = {shape = SQUARE;}'}), + + ({'SERIES_ANALYSIS_REGRID_CONVERT': '2*x', }, + {'METPLUS_REGRID_DICT': 'regrid = {convert(x) = 2*x;}'}), + + ({'SERIES_ANALYSIS_REGRID_CENSOR_THRESH': '>12000,<5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_thresh = [>12000, <5000];}'}), + + ({'SERIES_ANALYSIS_REGRID_CENSOR_VAL': '12000,5000', }, + {'METPLUS_REGRID_DICT': 'regrid = {censor_val = [12000, 5000];}'}), + + ({'SERIES_ANALYSIS_REGRID_TO_GRID': 'FCST', + 'SERIES_ANALYSIS_REGRID_METHOD': 'NEAREST', + 'SERIES_ANALYSIS_REGRID_WIDTH': '1', + 'SERIES_ANALYSIS_REGRID_VLD_THRESH': '0.5', + 'SERIES_ANALYSIS_REGRID_SHAPE': 'SQUARE', + 'SERIES_ANALYSIS_REGRID_CONVERT': '2*x', + 'SERIES_ANALYSIS_REGRID_CENSOR_THRESH': '>12000,<5000', + 'SERIES_ANALYSIS_REGRID_CENSOR_VAL': '12000,5000', + }, + {'METPLUS_REGRID_DICT': ('regrid = {to_grid = FCST;method = NEAREST;' + 'width = 1;vld_thresh = 0.5;shape = SQUARE;' + 'convert(x) = 2*x;' + 'censor_thresh = [>12000, <5000];' + 'censor_val = [12000, 5000];}' + )}), # climo_mean ({'SERIES_ANALYSIS_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', }, {'METPLUS_CLIMO_MEAN_DICT': ('climo_mean = {file_name = ' diff --git a/metplus/util/met_config.py b/metplus/util/met_config.py index a097dd6990..e652b4f22d 100644 --- a/metplus/util/met_config.py +++ b/metplus/util/met_config.py @@ -525,6 +525,8 @@ def set_met_config_string(config, c_dict, mp_config, met_config_name, Default value is False @param default (Optional) if set, use this value as default if config is not set + @param add_x if True, add (x) to variable name, e.g. convert(x) + Default value is False """ mp_config_name = config.get_mp_config_name(mp_config) conf_value = _get_config_or_default( @@ -548,7 +550,10 @@ def set_met_config_string(config, c_dict, mp_config, met_config_name, c_key = c_dict_key if c_dict_key else met_config_name.upper() if met_config_name: - conf_value = f'{met_config_name} = {conf_value};' + config_name = met_config_name + if kwargs.get('add_x'): + config_name = f'{config_name}(x)' + conf_value = f'{config_name} = {conf_value};' c_dict[c_key] = conf_value return True @@ -639,7 +644,8 @@ def set_met_config_thresh(config, c_dict, mp_config, met_config_name, out_value = str(conf_value) c_dict[c_key] = out_value - return True + + return True def set_met_config_bool(config, c_dict, mp_config, met_config_name, @@ -738,8 +744,8 @@ def _parse_item_info(item_info): def _parse_extra_args(extra): """! Check string for extra option keywords and set them to True in - dictionary if they are found. Supports 'remove_quotes', 'uppercase' - and 'allow_empty' + dictionary if they are found. Supports 'remove_quotes', 'uppercase', + 'allow_empty', 'to_grid', 'default', and 'add_x'. @param extra string to parse for keywords @returns dictionary with extra args set if found in string @@ -754,6 +760,7 @@ def _parse_extra_args(extra): 'allow_empty', 'to_grid', 'default', + 'add_x', ) for extra_option in VALID_EXTRAS: if extra_option in extra: diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 3595bc6b05..df7354c54f 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -1308,6 +1308,9 @@ def handle_regrid(self, c_dict, set_to_grid=True): dict_items['width'] = 'int' dict_items['vld_thresh'] = 'float' dict_items['shape'] = ('string', 'uppercase,remove_quotes') + dict_items['convert'] = ('string', 'remove_quotes,add_x') + dict_items['censor_thresh'] = ('list', 'remove_quotes') + dict_items['censor_val'] = ('list', 'remove_quotes') self.add_met_config_dict('regrid', dict_items) def handle_description(self, is_list=False): diff --git a/metplus/wrappers/gen_ens_prod_wrapper.py b/metplus/wrappers/gen_ens_prod_wrapper.py index 21e07305b4..74c87ffcc9 100755 --- a/metplus/wrappers/gen_ens_prod_wrapper.py +++ b/metplus/wrappers/gen_ens_prod_wrapper.py @@ -125,13 +125,7 @@ def create_c_dict(self): metplus_configs=['DESC', 'GEN_ENS_PROD_DESC'], ) - self.add_met_config_dict('regrid', { - 'to_grid': ('string', 'to_grid'), - 'method': ('string', 'uppercase,remove_quotes'), - 'width': 'int', - 'vld_thresh': 'float', - 'shape': ('string', 'uppercase,remove_quotes'), - }) + self.handle_regrid(c_dict) self.add_met_config(name='censor_thresh', data_type='list', diff --git a/metplus/wrappers/mtd_wrapper.py b/metplus/wrappers/mtd_wrapper.py index b74663ddab..432f7f375f 100755 --- a/metplus/wrappers/mtd_wrapper.py +++ b/metplus/wrappers/mtd_wrapper.py @@ -137,10 +137,11 @@ def read_field_values(self, c_dict, read_type, write_type): ) self.add_met_config(name='conv_radius', - data_type='int', + data_type='string', env_var_name=f'METPLUS_{write_type}_CONV_RADIUS', metplus_configs=[f'{read_type}_MTD_CONV_RADIUS', - 'MTD_CONV_RADIUS']) + 'MTD_CONV_RADIUS'], + extra_args={'remove_quotes': True}) self.add_met_config(name='conv_thresh', data_type='thresh', diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index 31b9999125..743ecbe34e 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -440,7 +440,7 @@ def _handle_diag_convert_map(self): dict_items = { 'diag_source': 'string', 'key': 'list', - 'convert': ('string', 'remove_quotes'), + 'convert': ('string', 'remove_quotes,add_x'), } return_code = add_met_config_dict_list(config=self.config, app_name=self.app_name, @@ -451,11 +451,6 @@ def _handle_diag_convert_map(self): self.isOK = False return - # change "convert =" to "convert(x) =" - value = self.env_var_dict.get('METPLUS_DIAG_CONVERT_MAP_LIST', '') - value = value.replace('convert =', 'convert(x) =') - self.env_var_dict['METPLUS_DIAG_CONVERT_MAP_LIST'] = value - def _handle_diag(self, c_dict): diag_indices = list( find_indices_in_config_section(r'TC_PAIRS_DIAG_TEMPLATE(\d+)$', diff --git a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf index 03aaee0657..da1c9d5e6d 100644 --- a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf +++ b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf @@ -113,6 +113,9 @@ ENSEMBLE_STAT_REGRID_METHOD = NEAREST ENSEMBLE_STAT_REGRID_WIDTH = 1 ENSEMBLE_STAT_REGRID_VLD_THRESH = 0.5 ENSEMBLE_STAT_REGRID_SHAPE = SQUARE +#ENSEMBLE_STAT_REGRID_CONVERT = +#ENSEMBLE_STAT_REGRID_CENSOR_THRESH = +#ENSEMBLE_STAT_REGRID_CENSOR_VAL = ENSEMBLE_STAT_CENSOR_THRESH = ENSEMBLE_STAT_CENSOR_VAL = diff --git a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf index 696abe4008..3c4a300a40 100644 --- a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf +++ b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf @@ -102,6 +102,9 @@ GEN_ENS_PROD_N_MEMBERS = 6 #GEN_ENS_PROD_REGRID_WIDTH = 1 #GEN_ENS_PROD_REGRID_VLD_THRESH = 0.5 #GEN_ENS_PROD_REGRID_SHAPE = SQUARE +#GEN_ENS_PROD_REGRID_CONVERT = +#GEN_ENS_PROD_REGRID_CENSOR_THRESH = +#GEN_ENS_PROD_REGRID_CENSOR_VAL = #GEN_ENS_PROD_CENSOR_THRESH = #GEN_ENS_PROD_CENSOR_VAL = diff --git a/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf b/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf index c35fe3994a..4104ee6016 100644 --- a/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf +++ b/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf @@ -78,5 +78,8 @@ GRID_DIAG_CONFIG_FILE = {PARM_BASE}/met_config/GridDiagConfig_wrapped #GRID_DIAG_REGRID_WIDTH = 1 #GRID_DIAG_REGRID_VLD_THRESH = 0.5 #GRID_DIAG_REGRID_SHAPE = SQUARE +#GRID_DIAG_REGRID_CONVERT = +#GRID_DIAG_REGRID_CENSOR_THRESH = +#GRID_DIAG_REGRID_CENSOR_VAL = GRID_DIAG_MASK_POLY = MET_BASE/poly/SAO.poly diff --git a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf index ce378efb9b..94f563d660 100644 --- a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf +++ b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf @@ -90,6 +90,13 @@ GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped #OBS_GRID_STAT_FILE_TYPE = GRID_STAT_REGRID_TO_GRID = NONE +#GRID_STAT_REGRID_METHOD = +#GRID_STAT_REGRID_WIDTH = +#GRID_STAT_REGRID_VLD_THRESH = +#GRID_STAT_REGRID_SHAPE = +#GRID_STAT_REGRID_CONVERT = +#GRID_STAT_REGRID_CENSOR_THRESH = +#GRID_STAT_REGRID_CENSOR_VAL = #GRID_STAT_INTERP_FIELD = #GRID_STAT_INTERP_VLD_THRESH = diff --git a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf index 30ed8a2f16..07b392a36c 100644 --- a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf +++ b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf @@ -115,6 +115,9 @@ MODE_REGRID_TO_GRID = NONE #MODE_REGRID_WIDTH = #MODE_REGRID_VLD_THRESH = #MODE_REGRID_SHAPE = +#MODE_REGRID_CONVERT = +#MODE_REGRID_CENSOR_THRESH = +#MODE_REGRID_CENSOR_VAL = MODE_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_NAME}_{CURRENT_OBS_LEVEL} diff --git a/parm/use_cases/met_tool_wrapper/MTD/MTD.conf b/parm/use_cases/met_tool_wrapper/MTD/MTD.conf index 54ef200032..6f615537e0 100644 --- a/parm/use_cases/met_tool_wrapper/MTD/MTD.conf +++ b/parm/use_cases/met_tool_wrapper/MTD/MTD.conf @@ -87,6 +87,14 @@ OBTYPE = MC_PCP MTD_CONFIG_FILE = {PARM_BASE}/met_config/MTDConfig_wrapped MTD_REGRID_TO_GRID = OBS +#MTD_REGRID_METHOD = +#MTD_REGRID_WIDTH = +#MTD_REGRID_VLD_THRESH = +#MTD_REGRID_SHAPE = +#MTD_REGRID_CONVERT = +#MTD_REGRID_CENSOR_THRESH = +#MTD_REGRID_CENSOR_VAL = + MTD_MIN_VOLUME = 2000 diff --git a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf index 50646d631a..ff4730d495 100644 --- a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf +++ b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf @@ -57,6 +57,13 @@ POINT2GRID_FILE_WINDOW_END = 0 # LOG_POINT2GRID_VERBOSITY = 1 POINT2GRID_REGRID_TO_GRID = G212 +POINT2GRID_REGRID_METHOD = MAX +#POINT2GRID_REGRID_WIDTH = +#POINT2GRID_REGRID_VLD_THRESH = +#POINT2GRID_REGRID_SHAPE = +#POINT2GRID_REGRID_CONVERT = +#POINT2GRID_REGRID_CENSOR_THRESH = +#POINT2GRID_REGRID_CENSOR_VAL = POINT2GRID_INPUT_FIELD =TMP POINT2GRID_INPUT_LEVEL = @@ -65,7 +72,6 @@ POINT2GRID_QC_FLAGS = 0 POINT2GRID_ADP = -POINT2GRID_REGRID_METHOD = MAX POINT2GRID_GAUSSIAN_DX = 81.271 POINT2GRID_GAUSSIAN_RADIUS = 120 diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf index be514f6a9f..3c6de6eb1f 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf @@ -151,6 +151,11 @@ OBTYPE = POINT_STAT_REGRID_TO_GRID = NONE POINT_STAT_REGRID_METHOD = BILIN POINT_STAT_REGRID_WIDTH = 2 +#POINT_STAT_REGRID_VLD_THRESH = +#POINT_STAT_REGRID_SHAPE = +#POINT_STAT_REGRID_CONVERT = +#POINT_STAT_REGRID_CENSOR_THRESH = +#POINT_STAT_REGRID_CENSOR_VAL = POINT_STAT_OUTPUT_PREFIX = diff --git a/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf b/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf index bcaa622247..7b5fdb8edf 100644 --- a/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf +++ b/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf @@ -108,6 +108,9 @@ SERIES_ANALYSIS_IS_PAIRED = False #SERIES_ANALYSIS_REGRID_WIDTH = #SERIES_ANALYSIS_REGRID_VLD_THRESH = #SERIES_ANALYSIS_REGRID_SHAPE = +#SERIES_ANALYSIS_REGRID_CONVERT = +#SERIES_ANALYSIS_REGRID_CENSOR_THRESH = +#SERIES_ANALYSIS_REGRID_CENSOR_VAL = #SERIES_ANALYSIS_CLIMO_MEAN_FILE_NAME = #SERIES_ANALYSIS_CLIMO_MEAN_FIELD = diff --git a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf index 6d341db0a6..91c4791912 100644 --- a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf +++ b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf @@ -76,6 +76,9 @@ MODEL = fv3 #TC_RMW_REGRID_WIDTH = 1 #TC_RMW_REGRID_VLD_THRESH = 0.5 #TC_RMW_REGRID_SHAPE = SQUARE +#TC_RMW_REGRID_CONVERT = +#TC_RMW_REGRID_CENSOR_THRESH = +#TC_RMW_REGRID_CENSOR_VAL = TC_RMW_STORM_ID = AL142016 TC_RMW_BASIN = AL diff --git a/parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf b/parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf index df6ead3e55..edcfa743fd 100644 --- a/parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf +++ b/parm/use_cases/model_applications/short_range/GenEnsProd_fcstHRRR_fcstOnly_SurrogateSevere.conf @@ -123,6 +123,8 @@ GEN_ENS_PROD_REGRID_TO_GRID = G211 GEN_ENS_PROD_REGRID_METHOD = MAX GEN_ENS_PROD_REGRID_WIDTH = 27 GEN_ENS_PROD_REGRID_VLD_THRESH = 0.0 +GEN_ENS_PROD_REGRID_CENSOR_THRESH = ==-9999 +GEN_ENS_PROD_REGRID_CENSOR_VAL = 0.0 GEN_ENS_PROD_CENSOR_THRESH = ==-9999 GEN_ENS_PROD_CENSOR_VAL = 0.0 From 42db9efcedfb47a586a451a86e2c72e408e27994 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 25 Apr 2023 17:22:16 -0600 Subject: [PATCH 02/11] removed unused variable from test --- .../tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py index f537605b84..2e81d1512b 100755 --- a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py @@ -24,7 +24,6 @@ def set_minimum_config_settings(config): config.set('config', 'INIT_END', '2005080712') config.set('config', 'INIT_INCREMENT', '12H') config.set('config', 'LEAD_SEQ', '12H') - config.set('config', 'LOOP_ORDER', 'times') config.set('config', 'POINT_STAT_CONFIG_FILE', '{PARM_BASE}/met_config/PointStatConfig_wrapped') From 9ddb9c242887288d69cb337b292934a9ad6a3cab Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 25 Apr 2023 17:23:55 -0600 Subject: [PATCH 03/11] allow filename templates in grid definition and ADP, require input field, fix QC to support string e.g. 0,1 --- metplus/wrappers/point2grid_wrapper.py | 46 +++++++++++++------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/metplus/wrappers/point2grid_wrapper.py b/metplus/wrappers/point2grid_wrapper.py index 0af3fe3600..a1870616dd 100755 --- a/metplus/wrappers/point2grid_wrapper.py +++ b/metplus/wrappers/point2grid_wrapper.py @@ -67,28 +67,28 @@ def create_c_dict(self): self.config.getseconds('config', 'OBS_FILE_WINDOW_END', 0)) - c_dict['GRID'] = self.config.getstr('config', - 'POINT2GRID_REGRID_TO_GRID', - '') + c_dict['GRID_TEMPLATE'] = ( + self.config.getraw('config', 'POINT2GRID_REGRID_TO_GRID') + ) # grid is required - if not c_dict['GRID']: + if not c_dict['GRID_TEMPLATE']: self.log_error('Must specify a grid name') # optional arguments c_dict['INPUT_FIELD'] = self.config.getraw('config', 'POINT2GRID_INPUT_FIELD', '') + # input field is required + if not c_dict['INPUT_FIELD']: + self.log_error('Must specify a field with POINT2GRID_INPUT_FIELD') c_dict['INPUT_LEVEL'] = self.config.getraw('config', 'POINT2GRID_INPUT_LEVEL', '') - c_dict['QC_FLAGS'] = self.config.getbool('config', - 'POINT2GRID_QC_FLAGS', - '') - c_dict['ADP'] = self.config.getstr('config', - 'POINT2GRID_ADP', - '') + c_dict['QC_FLAGS'] = self.config.getraw('config', + 'POINT2GRID_QC_FLAGS') + c_dict['ADP'] = self.config.getraw('config', 'POINT2GRID_ADP') c_dict['REGRID_METHOD'] = self.config.getstr('config', 'POINT2GRID_REGRID_METHOD', @@ -186,11 +186,6 @@ def run_at_time_once(self, time_info): self.set_environment_variables(time_info) # build command and run - cmd = self.get_command() - if cmd is None: - self.log_error("Could not generate command") - return - self.build() def find_input_files(self, time_info): @@ -207,6 +202,9 @@ def find_input_files(self, time_info): self.infiles.append(input_path) + self.c_dict['GRID'] = do_string_sub(self.c_dict['GRID_TEMPLATE'], + **time_info) + return self.infiles def set_command_line_arguments(self, time_info): @@ -216,16 +214,14 @@ def set_command_line_arguments(self, time_info): #input_field and input_level go hand in hand. If there is a field there needs to be a level #even if it is blank + input_field = do_string_sub(self.c_dict['INPUT_FIELD'], **time_info) input_level = "" - if self.c_dict['INPUT_FIELD']: - input_field = do_string_sub(self.c_dict['INPUT_FIELD'], + if self.c_dict['INPUT_LEVEL']: + input_level = do_string_sub(self.c_dict['INPUT_LEVEL'], **time_info) - if self.c_dict['INPUT_LEVEL']: - input_level = do_string_sub(self.c_dict['INPUT_LEVEL'], - **time_info) - self.logger.info(f"Processing level: {input_level}") - #Add either the specified level above or the defauilt blank one - self.args.append(f"-field 'name=\"{input_field}\"; level=\"{input_level}\";'") + self.logger.info(f"Processing level: {input_level}") + #Add either the specified level above or the defauilt blank one + self.args.append(f"-field 'name=\"{input_field}\"; level=\"{input_level}\";'") if self.c_dict['QC_FLAGS']: self.args.append(f"-qc {self.c_dict['QC_FLAGS']}") @@ -247,3 +243,7 @@ def set_command_line_arguments(self, time_info): if self. c_dict['VLD_THRESH']: self.args.append(f"-vld_thresh {self.c_dict['VLD_THRESH']}") + + # string sub all args + for index, value in enumerate(self.args): + self.args[index] = do_string_sub(value, **time_info) From ffa5c3ede7e5c76459019ce7abc1854f643c4489 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 25 Apr 2023 17:24:22 -0600 Subject: [PATCH 04/11] write more complete Point2Grid unit tests --- .../wrappers/point2grid/test_point2grid.py | 244 ++++++++---------- 1 file changed, 107 insertions(+), 137 deletions(-) diff --git a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py index c0f51d47fd..03184b80bf 100644 --- a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py +++ b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py @@ -2,148 +2,118 @@ import pytest import datetime +import os from metplus.wrappers.point2grid_wrapper import Point2GridWrapper from metplus.util import time_util +input_dir = '/some/path/input' +input_name = 'TMP' -def p2g_wrapper(metplus_config): - """! Returns a default Point2Grid with /path/to entries in the - metplus_system.conf and metplus_runtime.conf configuration - files. Subsequent tests can customize the final METplus configuration - to over-ride these /path/to values.""" - - config = metplus_config - config.set('config', 'DO_NOT_RUN_EXE', True) - return Point2GridWrapper(config) +# grid to use if grid is not set with command line argument +grid_dir = '/some/path/grid' +grid_template = os.path.join(grid_dir, '{valid?fmt=%Y%m%d}.nc') +def set_minimum_config_settings(config): + # set config variables to prevent command from running and bypass check + # if input files actually exist + config.set('config', 'DO_NOT_RUN_EXE', True) + config.set('config', 'INPUT_MUST_EXIST', False) + + # set process and time config variables + config.set('config', 'PROCESS_LIST', 'Point2Grid') + config.set('config', 'LOOP_BY', 'INIT') + config.set('config', 'INIT_TIME_FMT', '%Y%m%d%H') + config.set('config', 'INIT_BEG', '2017060100') + config.set('config', 'INIT_END', '2017060300') + config.set('config', 'INIT_INCREMENT', '24H') + config.set('config', 'LEAD_SEQ', '12H') + + # required variables for input/output, to grid, and input field + config.set('config', 'POINT2GRID_INPUT_DIR', input_dir) + config.set('config', 'POINT2GRID_INPUT_TEMPLATE', + 'input.{init?fmt=%Y%m%d%H}_f{lead?fmt=%2H}.nc') + config.set('config', 'POINT2GRID_OUTPUT_DIR', '{OUTPUT_BASE}/out') + config.set('config', 'POINT2GRID_OUTPUT_TEMPLATE', + 'output.{valid?fmt=%Y%m%d%H}.nc') + config.set('config', 'POINT2GRID_REGRID_TO_GRID', grid_template) + config.set('config', 'POINT2GRID_INPUT_FIELD', input_name) + + +@pytest.mark.parametrize( + 'config_overrides, optional_args', [ + ({}, {}), + ({'POINT2GRID_REGRID_METHOD': 'UW_MEAN'}, ['-method UW_MEAN']), + ({'POINT2GRID_REGRID_METHOD': 'UW_MEAN', + 'POINT2GRID_GAUSSIAN_DX': '2',}, + ['-method UW_MEAN', '-gaussian_dx 2']), + ({'POINT2GRID_GAUSSIAN_RADIUS': '81.231'}, + ['-gaussian_radius 81.231']), + ({'POINT2GRID_PROB_CAT_THRESH': '1'}, ['-prob_cat_thresh 1']), + ({'POINT2GRID_VLD_THRESH': '0.5'}, ['-vld_thresh 0.5']), + ({'POINT2GRID_QC_FLAGS': '0,1'}, ['-qc 0,1']), + ({'POINT2GRID_ADP': '{valid?fmt=%Y%m}.nc'}, ['-adp 201706.nc']), + ({'POINT2GRID_REGRID_TO_GRID': 'G212'}, []), + ({'POINT2GRID_INPUT_LEVEL': '(*,*)'}, []), + ] +) @pytest.mark.wrapper -def test_set_command_line_arguments(metplus_config): - test_passed = True - wrap = p2g_wrapper(metplus_config) - - input_dict = {'valid': datetime.datetime.strptime("202003050000", '%Y%m%d%H%M'), - 'lead': 0} - time_info = time_util.ti_calculate(input_dict) - - - wrap.c_dict['REGRID_METHOD'] = 'UW_MEAN' - - expected_args = ['-method UW_MEAN',] - - wrap.set_command_line_arguments(time_info) - if wrap.args != expected_args: - test_passed = False - print("Test 0 failed") - print(f"ARGS: {wrap.args}") - print(f"EXP: {expected_args}") - - wrap.args.clear() - - wrap.c_dict['GAUSSIAN_DX'] = 2 - - expected_args = ['-method UW_MEAN', - '-gaussian_dx 2', - ] - - wrap.set_command_line_arguments(time_info) - if wrap.args != expected_args: - test_passed = False - print("Test 1 failed") - print(f"ARGS: {wrap.args}") - print(f"EXP: {expected_args}") - - wrap.args.clear() - - wrap.c_dict['PROB_CAT_THRESH'] = 1 - - expected_args = ['-method UW_MEAN', - '-gaussian_dx 2', - '-prob_cat_thresh 1', - ] - - wrap.set_command_line_arguments(time_info) - if wrap.args != expected_args: - test_passed = False - print("Test 2 failed") - print(f"ARGS: {wrap.args}") - print(f"EXP: {expected_args}") - - wrap.args.clear() - - wrap.c_dict['GAUSSIAN_RADIUS'] = 3 - - expected_args = ['-method UW_MEAN', - '-gaussian_dx 2', - '-gaussian_radius 3', - '-prob_cat_thresh 1', - ] - - wrap.set_command_line_arguments(time_info) - if wrap.args != expected_args: - test_passed = False - print("Test 3 failed") - print(f"ARGS: {wrap.args}") - print(f"EXP: {expected_args}") - - wrap.args.clear() - - wrap.c_dict['VLD_THRESH'] = .5 - - expected_args = ['-method UW_MEAN', - '-gaussian_dx 2', - '-gaussian_radius 3', - '-prob_cat_thresh 1', - '-vld_thresh 0.5', - ] - - wrap.set_command_line_arguments(time_info) - if wrap.args != expected_args: - test_passed = False - print("Test 4 failed") - print(f"ARGS: {wrap.args}") - print(f"EXP: {expected_args}") - - wrap.args.clear() - - wrap.c_dict['QC_FLAGS'] = 1 - - expected_args = ['-qc 1', - '-method UW_MEAN', - '-gaussian_dx 2', - '-gaussian_radius 3', - '-prob_cat_thresh 1', - '-vld_thresh 0.5', - ] - - wrap.set_command_line_arguments(time_info) - if wrap.args != expected_args: - test_passed = False - print("Test 5 failed") - print(f"ARGS: {wrap.args}") - print(f"EXP: {expected_args}") - - wrap.args.clear() - - wrap.c_dict['ADP'] = 'test.nc' - - expected_args = ['-qc 1', - '-adp test.nc', - '-method UW_MEAN', - '-gaussian_dx 2', - '-gaussian_radius 3', - '-prob_cat_thresh 1', - '-vld_thresh 0.5', - ] - - wrap.set_command_line_arguments(time_info) - if wrap.args != expected_args: - test_passed = False - print("Test 6 failed") - print(f"ARGS: {wrap.args}") - print(f"EXP: {expected_args}") - - wrap.args.clear() - - assert test_passed +def test_point2grid_run(metplus_config, config_overrides, optional_args): + config = metplus_config + set_minimum_config_settings(config) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = Point2GridWrapper(config) + assert wrapper.isOK + + app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name) + verbosity = f"-v {wrapper.c_dict['VERBOSITY']}" + out_dir = wrapper.c_dict.get('OUTPUT_DIR') + input_files = ( + f'{input_dir}/input.2017060100_f12.nc', + f'{input_dir}/input.2017060200_f12.nc', + f'{input_dir}/input.2017060300_f12.nc', + ) + output_files = ( + f'{out_dir}/output.2017060112.nc', + f'{out_dir}/output.2017060212.nc', + f'{out_dir}/output.2017060312.nc', + ) + + if 'POINT2GRID_REGRID_TO_GRID' in config_overrides: + grids = ( + config_overrides['POINT2GRID_REGRID_TO_GRID'], + config_overrides['POINT2GRID_REGRID_TO_GRID'], + config_overrides['POINT2GRID_REGRID_TO_GRID'] + ) + else: + grids = ( + os.path.join(grid_dir, '20170601.nc'), + os.path.join(grid_dir, '20170602.nc'), + os.path.join(grid_dir, '20170603.nc') + ) + + if 'POINT2GRID_INPUT_LEVEL' in config_overrides: + level = config_overrides['POINT2GRID_INPUT_LEVEL'] + else: + level = '' + + extra_args = " ".join(optional_args) + " " if optional_args else "" + expected_cmds = [] + for idx in range(0, 3): + expected_cmds.append( + f'{app_path} {input_files[idx]} "{grids[idx]}" {output_files[idx]}' + f' -field \'name="{input_name}"; level="{level}";\'' + f' {extra_args}{verbosity}' + ) + + all_cmds = wrapper.run_all_times() + print(f"ALL COMMANDS: {all_cmds}") + + for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): + # ensure commands are generated as expected + assert cmd == expected_cmd From 0ec1374eb4f2e6c631c0f50fbc330ba09fed390e Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 25 Apr 2023 17:26:58 -0600 Subject: [PATCH 05/11] turn on use case group to confirm diff tests now pass, ci-run-diff --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index b5787cec6b..65ab735a40 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -67,7 +67,7 @@ { "category": "marine_and_cryosphere", "index_list": "0-2", - "run": false + "run": true }, { "category": "marine_and_cryosphere", From b65b5d9a7e55356d6a453642507c3d63bad3128b Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 25 Apr 2023 17:41:00 -0600 Subject: [PATCH 06/11] removed the new config variables that aren't actually used and corrected the definition for POINT2GRID_REGRID_TO_GRID --- docs/Users_Guide/glossary.rst | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 4a1f8564da..e2e52287fd 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -995,7 +995,7 @@ METplus Configuration Glossary | *Used by:* GridStat POINT2GRID_REGRID_TO_GRID - Used to set the regrid dictionary item 'to_grid' in the MET Point2Grid config file. See the `MET User's Guide `_ for more information. + Used to set the grid definition for point2grid. | *Used by:* Point2Grid @@ -10344,36 +10344,6 @@ METplus Configuration Glossary | *Used by:* MTD - POINT2GRID_REGRID_WIDTH - Specify the value for 'regrid.width' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_REGRID_VLD_THRESH - Specify the value for 'regrid.vld_thresh' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_REGRID_SHAPE - Specify the value for 'regrid.shape' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_REGRID_CONVERT - Specify the value for 'regrid.convert' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_REGRID_CENSOR_THRESH - Specify the value for 'regrid.censor_thresh' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_REGRID_CENSOR_VAL - Specify the value for 'regrid.censor_val' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - POINT_STAT_REGRID_CONVERT Specify the value for 'regrid.convert' in the MET configuration file for PointStat. From bddd7fcc37c78a9556ed70544059b2501fa741d4 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 26 Apr 2023 09:31:28 -0600 Subject: [PATCH 07/11] removed config variables that were added accidentally --- docs/Users_Guide/wrappers.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 170a073d30..a900dcd864 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -5738,12 +5738,6 @@ METplus Configuration | :term:`POINT2GRID_QC_FLAGS` | :term:`POINT2GRID_ADP` | :term:`POINT2GRID_REGRID_METHOD` -| :term:`POINT2GRID_REGRID_WIDTH` -| :term:`POINT2GRID_REGRID_VLD_THRESH` -| :term:`POINT2GRID_REGRID_SHAPE` -| :term:`POINT2GRID_REGRID_CONVERT` -| :term:`POINT2GRID_REGRID_CENSOR_THRESH` -| :term:`POINT2GRID_REGRID_CENSOR_VAL` | :term:`POINT2GRID_GAUSSIAN_DX` | :term:`POINT2GRID_GAUSSIAN_RADIUS` | :term:`POINT2GRID_PROB_CAT_THRESH` From 6702b0e26e428a09cba9bb5192b37f67ad9f7d25 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 26 Apr 2023 12:27:30 -0600 Subject: [PATCH 08/11] turn off use case that appears to be failing the diff test due to a MET issue --- .github/parm/use_case_groups.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 65ab735a40..b5787cec6b 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -67,7 +67,7 @@ { "category": "marine_and_cryosphere", "index_list": "0-2", - "run": true + "run": false }, { "category": "marine_and_cryosphere", From 66c9158534361b3b85d0c24a5b5eb8bcd3302a6f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 26 Apr 2023 14:36:36 -0600 Subject: [PATCH 09/11] added fix to discern POINT2GRID_QC_FLAGS=0 from unset --- metplus/wrappers/point2grid_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metplus/wrappers/point2grid_wrapper.py b/metplus/wrappers/point2grid_wrapper.py index a1870616dd..e502554e72 100755 --- a/metplus/wrappers/point2grid_wrapper.py +++ b/metplus/wrappers/point2grid_wrapper.py @@ -223,7 +223,7 @@ def set_command_line_arguments(self, time_info): #Add either the specified level above or the defauilt blank one self.args.append(f"-field 'name=\"{input_field}\"; level=\"{input_level}\";'") - if self.c_dict['QC_FLAGS']: + if self.c_dict['QC_FLAGS'] != '': self.args.append(f"-qc {self.c_dict['QC_FLAGS']}") if self.c_dict['ADP']: From 57c807af2d6074b7cbb69e8ad1d606d5877ca25d Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 26 Apr 2023 14:50:31 -0600 Subject: [PATCH 10/11] remove QC flags = 0 from config because it removes useful output and causes differences in the test output --- parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf index ff4730d495..570ee1cc81 100644 --- a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf +++ b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf @@ -68,7 +68,7 @@ POINT2GRID_REGRID_METHOD = MAX POINT2GRID_INPUT_FIELD =TMP POINT2GRID_INPUT_LEVEL = -POINT2GRID_QC_FLAGS = 0 +#POINT2GRID_QC_FLAGS = POINT2GRID_ADP = From 2d70bc57c74360047e3ef6ab6ede0e082ce8a47c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 26 Apr 2023 15:54:28 -0600 Subject: [PATCH 11/11] removed config variable that now causes diffs --- .../Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf b/parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf index 5f24ac8034..7ddab10eb0 100644 --- a/parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf +++ b/parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect.conf @@ -101,8 +101,6 @@ POINT2GRID_REGRID_TO_GRID = G211 POINT2GRID_INPUT_FIELD =Fscale POINT2GRID_INPUT_LEVEL = -POINT2GRID_QC_FLAGS = 0 - POINT2GRID_ADP = POINT2GRID_REGRID_METHOD = MAX