diff --git a/.coveragerc b/.coveragerc index 04ab2270b8..4c62180950 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,4 +3,3 @@ relative_files = True source = metplus omit = metplus/wrappers/cyclone_plotter_wrapper.py - metplus/produtil/** \ No newline at end of file diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 8d12b81db2..05cc0b6196 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -276,12 +276,6 @@ "disabled": true, "run": false }, - { - "category": "short_range", - "index_list": "15", - "disabled": true, - "run":false - }, { "category": "space_weather", "index_list": "0-1", diff --git a/.github/update_truth_change_log.txt b/.github/update_truth_change_log.txt index e36117aed6..fd6a31913f 100644 --- a/.github/update_truth_change_log.txt +++ b/.github/update_truth_change_log.txt @@ -7,4 +7,3 @@ [20240905_16:45:56 develop] dtcenter/MET#2959 - Correct bugs in SEDI and BAGSS contingency table statistics found in the CTS line type [20240910_19:03:38 develop] dtcenter/MET#2967 - Fixes integer overflow bug in the computation of the CTS and NBRCTS ORSS, ORSS_NCL, and ORSS_NCU columns. [20240926_16:50:38 develop] #2701 - #2701 added a new use case (SeriesAnalysis_aggr) that produces new output -[20241014_16:47:58 develop] dtcenter/MET#2988 - dtcenter/MET#2988 -- see issue #2719 for details diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index 56f8815dbe..10fe4c9a7a 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -169,7 +169,6 @@ Use Cases that Exceed GitHub Actions Disk Space ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - *model_applications/s2s_stratosphere/UserScript_fcstGFS_obsERA_StratospherePolar* - *model_applications/s2s_stratosphere/UserScript_fcstGFS_obsERA_StratosphereQBO* -- *model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning* .. _use_case_documentation: diff --git a/docs/Contributors_Guide/documentation.rst b/docs/Contributors_Guide/documentation.rst index 89373f4804..a340774bd9 100644 --- a/docs/Contributors_Guide/documentation.rst +++ b/docs/Contributors_Guide/documentation.rst @@ -218,7 +218,7 @@ for more information. **What to bold:** - * Environment Variables (e.g. **MET_INSTALL_DIR, INPUT_BASE, METCALCPY_HOME**, etc.) + * Variables (e.g. **MET_INSTALL_DIR, INPUT_BASE, METCALCPY_HOME**, etc.) * Filenames (**line_defaults.yaml, contour_defaults.yaml, defaults.conf**, etc.) Italics @@ -395,20 +395,6 @@ resulting in the following displayed text: See `Overleaf's Guide - Matrices `_ for more information. -Literal Include ---------------- - -This feature is used often in the Release Guide, but there are other places it could be useful. -Using a literal include ensures only having to update the documentation in one place. -It is important to use a relative path (i.e. "../../") to get to the correct -directory of the file being referenced in the literal include. This will keep the -file linking to the correct version and branch. - -As an example, refer to the METplus User's Guide. See this -`line `_ -in the PointStat Wrapper Use Case documentation, which uses RST's :code:`.. literalinclude::` -and renders the text of the referenced file in this -`section `_. Literal Blocks -------------- @@ -439,54 +425,6 @@ and `literal blocks `_ for more information. -Substitution References ------------------------ - -Substitution references are replacements in the text as defined in the :code:`rst_epilog` -in the **docs/conf.py** file. For example, instead of hard coding the MET version in -the METplus documentation, a variable :code:`met_version` can be set to a value -(e.g. 12.0.0) in the **docs/conf.py** file in the following way: - -.. code-block:: ini - - met_version = 12.0.0 - -Then, this variable is used in the :code:`rst_epilog` to set up the substitution, -along with other variables. For example, in the **docs/conf.py** -file the following is defined: - -.. code-block:: ini - - rst_epilog = f""" - .. |copyright| replace:: {copyright} - .. |author_list| replace:: {author_list} - .. |release_date| replace:: {release_date} - .. |release_year| replace:: {release_year} - .. |release_info| replace:: {release_info} - .. |python_version| replace:: {python_version} - .. |met_version| replace:: {met_version} - """ - -The value 12.0.0 would then be replaced in the documentation wherever -:code:`|met_version|` is used in .rst files. For example: - -.. code-block:: ini - - MET version |met_version| or above - -There are some -`reserved substitution variables `_ -in Sphinx. These variables are -:code:`|release|`, :code:`|version|`, :code:`|today|`, and -:code:`|translation progress|`. Reserved substitution variables do not -need to be added to :code:`rst_epilog`, but other variables do. - -See -`Substitutions `_ -and -`rst_epilog `_ -for more information. - Line Blocks ----------- @@ -1475,7 +1413,8 @@ Read the Docs supports multiple versions for each repository. For the METplus components, the "latest" version will point to the latest official (stable) release. The "develop" or "development" version will point to the most up to date development code. There may also be other previous versions of the -software available in the version selector menu. +software available in the version selector menu, which is accessible by +clicking in the bottom left corner of the documentation pages. Automation rules allow project maintainers to automate actions on new branches and tags on repositories. For the METplus components, documentation is diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 2dcd640c4f..6bc690bf76 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -290,12 +290,7 @@ METplus Configuration Glossary | *Used by:* Point2Grid POINT2GRID_QC_FLAGS - .. warning:: **DEPRECATED:** Please use :term:`POINT2GRID_GOES_QC_FLAGS`. - - | *Used by:* Point2Grid - - POINT2GRID_GOES_QC_FLAGS - Sets the value for the -goes_qc command line argument for Point2Grid. + Specify the qc flags name that is read by Point2Grid. | *Used by:* Point2Grid @@ -11629,46 +11624,6 @@ METplus Configuration Glossary | *Used by:* PointStat - POINT2GRID_VALID_TIME - Specify the value for 'valid_time' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_OBS_WINDOW_BEG - Specify the value for 'obs_window.beg' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_OBS_WINDOW_END - Specify the value for 'obs_window.end' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_MESSAGE_TYPE - Specify the value for 'message_type' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_VAR_NAME_MAP_KEY - Specify the value for the nth 'var_name_map.key' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_VAR_NAME_MAP_VAL - Specify the value for the nth 'var_name_map.val' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_OBS_QUALITY_INC - Specify the value for 'obs_quality_inc' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - - POINT2GRID_OBS_QUALITY_EXC - Specify the value for 'obs_quality_exc' in the MET configuration file for Point2Grid. - - | *Used by:* Point2Grid - MADIS2NC_CUSTOM_LOOP_LIST Sets custom string loop list for a specific wrapper. See :term:`CUSTOM_LOOP_LIST`. @@ -11857,39 +11812,105 @@ METplus Configuration Glossary | *Used by:* MADIS2NC + MADIS2NC_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for MADIS2NC. + + | *Used by:* MADIS2NC + + ASCII2NC_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for ASCII2NC. + + | *Used by:* ASCII2NC + + ENSEMBLE_STAT_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for EnsembleStat. + + | *Used by:* EnsembleStat + + GEN_ENS_PROD_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for GenEnsProd. + + | *Used by:* GenEnsProd + + GRID_DIAG_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for GridDiag. + + | *Used by:* GridDiag + GRID_STAT_TIME_OFFSET_WARNING Specify the value for 'time_offset_warning' in the MET configuration file for GridStat. | *Used by:* GridStat + IODA2NC_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for IODA2NC. + + | *Used by:* IODA2NC + MODE_TIME_OFFSET_WARNING Specify the value for 'time_offset_warning' in the MET configuration file for MODE. | *Used by:* MODE + MTD_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for MTD. + + | *Used by:* MTD + + PB2NC_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for PB2NC. + + | *Used by:* PB2NC + + PLOT_POINT_OBS_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for PlotPointObs. + + | *Used by:* PlotPointObs + + POINT_STAT_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + STAT_ANALYSIS_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for StatAnalysis. + + | *Used by:* StatAnalysis + SERIES_ANALYSIS_TIME_OFFSET_WARNING Specify the value for 'time_offset_warning' in the MET configuration file for SeriesAnalysis. | *Used by:* SeriesAnalysis - WAVELET_STAT_TIME_OFFSET_WARNING - Specify the value for 'time_offset_warning' in the MET configuration file for WaveletStat. + TC_DIAG_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for TCDiag. - | *Used by:* WaveletStat + | *Used by:* TCDiag - POINT2GRID_MET_CONFIG_OVERRIDES - Override any variables in the MET configuration file that are not - supported by the wrapper. This should be set to the full variable name - and value that you want to override, including the equal sign and the - ending semi-colon. The value is directly appended to the end of the - wrapped MET config file. + TC_GEN_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for TCGen. - Example: - POINT2GRID_MET_CONFIG_OVERRIDES = desc = "override_desc"; model = "override_model"; + | *Used by:* TCGen - See :ref:`Overriding Unsupported MET config file settings` for more information + TC_PAIRS_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for TCPairs. - | *Used by:* Point2Grid + | *Used by:* TCPairs + + TC_RMW_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for TCRMW. + + | *Used by:* TCRMW + + TC_STAT_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for TCStat. + + | *Used by:* TCStat + + WAVELET_STAT_TIME_OFFSET_WARNING + Specify the value for 'time_offset_warning' in the MET configuration file for WaveletStat. + + | *Used by:* WaveletStat ASCII2NC_VALID_BEG Specify the value for the command line argument '-valid_beg' for ASCII2NC. diff --git a/docs/Users_Guide/quicksearch.rst b/docs/Users_Guide/quicksearch.rst index c51d262d3d..30aaca532e 100644 --- a/docs/Users_Guide/quicksearch.rst +++ b/docs/Users_Guide/quicksearch.rst @@ -156,7 +156,6 @@ Use Cases by METplus Feature: | `METplotpy <../search.html?q=METplotpyUseCase&check_keywords=yes&area=default>`_ | `MET_PYTHON_EXE Environment Variable <../search.html?q=MET_PYTHON_EXEUseCase&check_keywords=yes&area=default>`_ | `Multiple Conf File Use <../search.html?q=MultiConfUseCase&check_keywords=yes&area=default>`_ - | `MultivariateMODE <../search.html?q=MvMODEUseCase&check_keywords=yes&area=default>`_ | `Observation Time Summary <../search.html?q=ObsTimeSummaryUseCase&check_keywords=yes&area=default>`_ | `Observation Uncertainty <../search.html?q=ObsUncertaintyUseCase&check_keywords=yes&area=default>`_ | `Python Embedding Ingest <../search.html?q=PyEmbedIngestToolUseCase&check_keywords=yes&area=default>`_ @@ -188,7 +187,6 @@ Use Cases by METplus Feature: | **METplotpy**: *METplotpyUseCase* | **MET_PYTHON_EXE Environment Variable**: *MET_PYTHON_EXEUseCase* | **Multiple Conf File Use**: *MultiConfUseCase* - | **MultivariateMODE**: *MvMODEUseCase* | **Observation Time Summary**: *ObsTimeSummaryUseCase* | **Observation Uncertainty**: *ObsUncertaintyUseCase* | **Python Embedding Ingest**: *PyEmbedIngestToolUseCase* diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index aa2794377c..c4da8506ad 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -64,6 +64,7 @@ METplus Configuration | :term:`ASCII2NC_INC_VALID_TIMES` | :term:`ASCII2NC_SKIP_INIT_TIMES` | :term:`ASCII2NC_INC_INIT_TIMES` +| :term:`ASCII2NC_TIME_OFFSET_WARNING` | .. _ascii2nc-met-conf: @@ -123,6 +124,18 @@ ${METPLUS_TIME_SUMMARY_DICT} * - :term:`ASCII2NC_TIME_SUMMARY_VALID_THRESH` - time_summary.vld_thresh +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`ASCII2NC_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -378,6 +391,7 @@ METplus Configuration | :term:`ENSEMBLE_STAT_INC_VALID_TIMES` | :term:`ENSEMBLE_STAT_SKIP_INIT_TIMES` | :term:`ENSEMBLE_STAT_INC_INIT_TIMES` +| :term:`ENSEMBLE_STAT_TIME_OFFSET_WARNING` | .. warning:: **DEPRECATED:** @@ -1103,6 +1117,18 @@ ${METPLUS_OUTPUT_PREFIX} * - :term:`ENSEMBLE_STAT_OUTPUT_PREFIX` - output_prefix +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`ENSEMBLE_STAT_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -1326,6 +1352,7 @@ METplus Configuration | :term:`GEN_ENS_PROD_INC_VALID_TIMES` | :term:`GEN_ENS_PROD_SKIP_INIT_TIMES` | :term:`GEN_ENS_PROD_INC_INIT_TIMES` +| :term:`GEN_ENS_PROD_TIME_OFFSET_WARNING` .. _gen-ens-prod-met-conf: @@ -1679,6 +1706,18 @@ ${METPLUS_ENSEMBLE_FLAG_DICT} * - :term:`GEN_ENS_PROD_ENSEMBLE_FLAG_CLIMO_CDP` - ensemble_flag.climo_cdp +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`GEN_ENS_PROD_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -3028,6 +3067,7 @@ METplus Configuration | :term:`GRID_DIAG_INC_VALID_TIMES` | :term:`GRID_DIAG_SKIP_INIT_TIMES` | :term:`GRID_DIAG_INC_INIT_TIMES` +| :term:`GRID_DIAG_TIME_OFFSET_WARNING` | .. _grid-diag-met-conf: @@ -3152,6 +3192,18 @@ ${METPLUS_MASK_DICT} .. note:: Since the default value in the MET config file for 'grid' is grid = [ "FULL" ];, setting GRID_DIAG_MASK_GRID to an empty string will result in a value of grid = []; in the MET config file. +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`GRID_DIAG_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -4143,6 +4195,7 @@ METplus Configuration | :term:`IODA2NC_INC_VALID_TIMES` | :term:`IODA2NC_SKIP_INIT_TIMES` | :term:`IODA2NC_INC_INIT_TIMES` +| :term:`IODA2NC_TIME_OFFSET_WARNING` .. _ioda2nc-met-conf: @@ -4365,6 +4418,18 @@ ${METPLUS_TIME_SUMMARY_DICT} * - :term:`IODA2NC_TIME_SUMMARY_VLD_THRESH` - time_summary.vld_thresh +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`IODA2NC_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -4425,6 +4490,7 @@ METplus Configuration | :term:`MADIS2NC_INC_VALID_TIMES` | :term:`MADIS2NC_SKIP_INIT_TIMES` | :term:`MADIS2NC_INC_INIT_TIMES` +| :term:`MADIS2NC_TIME_OFFSET_WARNING` | .. _madis2nc-met-conf: @@ -4484,6 +4550,18 @@ ${METPLUS_TIME_SUMMARY_DICT} * - :term:`MADIS2NC_TIME_SUMMARY_VLD_THRESH` - time_summary.vld_thresh +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`MADIS2NC_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -5612,6 +5690,7 @@ METplus Configuration | :term:`MTD_INC_VALID_TIMES` | :term:`MTD_SKIP_INIT_TIMES` | :term:`MTD_INC_INIT_TIMES` +| :term:`MTD_TIME_OFFSET_WARNING` | .. warning:: **DEPRECATED:** @@ -5844,6 +5923,18 @@ ${METPLUS_OUTPUT_PREFIX} * - :term:`MTD_OUTPUT_PREFIX` - output_prefix +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`MTD_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -5904,6 +5995,7 @@ METplus Configuration | :term:`PB2NC_INC_VALID_TIMES` | :term:`PB2NC_SKIP_INIT_TIMES` | :term:`PB2NC_INC_INIT_TIMES` +| :term:`PB2NC_TIME_OFFSET_WARNING` .. warning:: **DEPRECATED:** @@ -6108,6 +6200,18 @@ ${METPLUS_TIME_SUMMARY_DICT} * - :term:`PB2NC_TIME_SUMMARY_VALID_THRESH` - time_summary.vld_thresh +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`PB2NC_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -6380,6 +6484,7 @@ Configuration | :term:`PLOT_POINT_OBS_INC_VALID_TIMES` | :term:`PLOT_POINT_OBS_SKIP_INIT_TIMES` | :term:`PLOT_POINT_OBS_INC_INIT_TIMES` +| :term:`PLOT_POINT_OBS_TIME_OFFSET_WARNING` .. _plot-point-obs-met-conf: @@ -6709,6 +6814,18 @@ ${METPLUS_POINT_DATA} * - :term:`PLOT_POINT_OBS_POINT_DATA` - point_data +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`PLOT_POINT_OBS_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -6745,7 +6862,7 @@ METplus Configuration | :term:`POINT2GRID_REGRID_TO_GRID` | :term:`POINT2GRID_INPUT_FIELD` | :term:`POINT2GRID_INPUT_LEVEL` -| :term:`POINT2GRID_GOES_QC_FLAGS` +| :term:`POINT2GRID_QC_FLAGS` | :term:`POINT2GRID_ADP` | :term:`POINT2GRID_REGRID_METHOD` | :term:`POINT2GRID_GAUSSIAN_DX` @@ -6758,134 +6875,8 @@ METplus Configuration | :term:`POINT2GRID_INC_VALID_TIMES` | :term:`POINT2GRID_SKIP_INIT_TIMES` | :term:`POINT2GRID_INC_INIT_TIMES` -| :term:`POINT2GRID_VALID_TIME` -| :term:`POINT2GRID_OBS_WINDOW_BEG` -| :term:`POINT2GRID_OBS_WINDOW_END` -| :term:`POINT2GRID_MESSAGE_TYPE` -| :term:`POINT2GRID_VAR_NAME_MAP_KEY` -| :term:`POINT2GRID_VAR_NAME_MAP_VAL` -| :term:`POINT2GRID_OBS_QUALITY_INC` -| :term:`POINT2GRID_OBS_QUALITY_EXC` -| :term:`POINT2GRID_MET_CONFIG_OVERRIDES` | -.. warning:: **DEPRECATED:** - - | :term:`POINT2GRID_QC_FLAGS` - -.. _point2grid-met-conf: - -MET Configuration ------------------ - -Below is the wrapped MET configuration file used for this wrapper. -Environment variables are used to control entries in this configuration file. -The default value for each environment variable is obtained from -(except where noted below): - -`MET_INSTALL_DIR/share/met/config/Point2GridConfig_default `_ - -Below the file contents are descriptions of each environment variable -referenced in this file and the corresponding METplus configuration item used -to set the value of the environment variable. For detailed examples showing -how METplus sets the values of these environment variables, -see :ref:`How METplus controls MET config file settings`. - -.. dropdown:: Click to view parm/met_config/Point2GridConfig_wrapped - - .. literalinclude:: ../../parm/met_config/Point2GridConfig_wrapped - -Environment variables in wrapped MET config -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -${METPLUS_VALID_TIME} -^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`POINT2GRID_VALID_TIME` - - valid_time - -${METPLUS_OBS_WINDOW_DICT} -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`POINT2GRID_OBS_WINDOW_BEG` - - obs_window.beg - * - :term:`POINT2GRID_OBS_WINDOW_END` - - obs_window.end - -${METPLUS_MESSAGE_TYPE} -^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`POINT2GRID_MESSAGE_TYPE` - - message_type - -${METPLUS_VAR_NAME_MAP_LIST} -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`POINT2GRID_VAR_NAME_MAP_KEY` - - var_name_map.key - * - :term:`POINT2GRID_VAR_NAME_MAP_VAL` - - var_name_map.val - -${METPLUS_OBS_QUALITY_INC} -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`POINT2GRID_OBS_QUALITY_INC` - - obs_quality_inc - -${METPLUS_OBS_QUALITY_EXC} -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`POINT2GRID_OBS_QUALITY_EXC` - - obs_quality_exc - -${METPLUS_MET_CONFIG_OVERRIDES} -""""""""""""""""""""""""""""""" - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`POINT2GRID_MET_CONFIG_OVERRIDES` - - n/a - .. _point_stat_wrapper: PointStat @@ -7097,6 +7088,7 @@ Configuration | :term:`POINT_STAT_UGRID_MAX_DISTANCE_KM` | :term:`POINT_STAT_UGRID_COORDINATES_FILE` | :term:`POINT_STAT_UGRID_CONFIG_FILE` +| :term:`POINT_STAT_TIME_OFFSET_WARNING` | .. warning:: **DEPRECATED:** @@ -7787,6 +7779,18 @@ ${METPLUS_OUTPUT_PREFIX} * - :term:`POINT_STAT_OUTPUT_PREFIX` - output_prefix +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT_STAT_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -8848,6 +8852,7 @@ The following values are optional in the METplus configuration file: | :term:`STAT_ANALYSIS_INC_VALID_TIMES` | :term:`STAT_ANALYSIS_SKIP_INIT_TIMES` | :term:`STAT_ANALYSIS_INC_INIT_TIMES` +| :term:`STAT_ANALYSIS_TIME_OFFSET_WARNING` .. warning:: **DEPRECATED:** @@ -9310,6 +9315,18 @@ ${METPLUS_HSS_EC_VALUE} * - :term:`STAT_ANALYSIS_HSS_EC_VALUE` - hss_ec_value +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`STAT_ANALYSIS_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -9390,6 +9407,7 @@ METplus Configuration | :term:`TC_DIAG_INC_VALID_TIMES` | :term:`TC_DIAG_SKIP_INIT_TIMES` | :term:`TC_DIAG_INC_INIT_TIMES` +| :term:`TC_DIAG_TIME_OFFSET_WARNING` | .. _tc-diag-met-conf: @@ -9817,6 +9835,18 @@ ${METPLUS_OUTPUT_PREFIX} * - :term:`TC_DIAG_OUTPUT_BASE_FORMAT` - output_base_format +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_DIAG_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -9922,6 +9952,7 @@ METplus Configuration | :term:`TC_GEN_DLAND_FILE` | :term:`TC_GEN_BASIN_FILE` | :term:`TC_GEN_NC_PAIRS_GRID` +| :term:`TC_GEN_TIME_OFFSET_WARNING` .. warning:: **DEPRECATED:** @@ -10488,6 +10519,18 @@ ${METPLUS_NC_PAIRS_GRID} * - :term:`TC_GEN_NC_PAIRS_GRID` - nc_pairs_grid +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_GEN_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -10583,6 +10626,7 @@ METplus Configuration | :term:`TC_PAIRS_INC_VALID_TIMES` | :term:`TC_PAIRS_SKIP_INIT_TIMES` | :term:`TC_PAIRS_INC_INIT_TIMES` +| :term:`TC_PAIRS_TIME_OFFSET_WARNING` | .. warning:: **DEPRECATED:** @@ -10921,6 +10965,18 @@ ${METPLUS_DIAG_CONVERT_MAP_LIST} * - :term:`TC_PAIRS_DIAG_CONVERT_MAP_CONVERT` - diag_convert_map.convert +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_PAIRS_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -10986,6 +11042,7 @@ METplus Configuration | :term:`TC_RMW_INC_VALID_TIMES` | :term:`TC_RMW_SKIP_INIT_TIMES` | :term:`TC_RMW_INC_INIT_TIMES` +| :term:`TC_RMW_TIME_OFFSET_WARNING` | .. _tc-rmw-met-conf: @@ -11247,6 +11304,18 @@ ${METPLUS_RMW_SCALE} * - :term:`TC_RMW_SCALE` - rmw_scale +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_RMW_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" @@ -11331,6 +11400,7 @@ METplus Configuration | :term:`TC_STAT_INC_VALID_TIMES` | :term:`TC_STAT_SKIP_INIT_TIMES` | :term:`TC_STAT_INC_INIT_TIMES` +| :term:`TC_STAT_TIME_OFFSET_WARNING` | .. warning:: **DEPRECATED:** @@ -11954,6 +12024,18 @@ ${METPLUS_JOBS} * - :term:`TC_STAT_JOBS_LIST` - jobs +${METPLUS_TIME_OFFSET_WARNING} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`TC_STAT_TIME_OFFSET_WARNING` + - time_offset_warning + ${METPLUS_MET_CONFIG_OVERRIDES} """"""""""""""""""""""""""""""" diff --git a/docs/_static/short_range-MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.png b/docs/_static/short_range-MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.png deleted file mode 100644 index 34cffb6993..0000000000 Binary files a/docs/_static/short_range-MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.png and /dev/null differ diff --git a/docs/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.py b/docs/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.py deleted file mode 100644 index 77209d9cfe..0000000000 --- a/docs/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.py +++ /dev/null @@ -1,179 +0,0 @@ -""" -MODEMultivar: Create objects of brightness temps and radar reflectivity -======================================================================= - -model_applications/ -short_range/ -MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf - -""" -############################################################################## -# .. contents:: -# :depth: 1 -# :local: -# :backlinks: none - -############################################################################## -# Scientific Objective -# -------------------- -# -# This use case identifies convective objects, which are defined by -# the intersection of: 1) satellite infrared brightness temperature < 235 K and -# 2) radar reflectivity > 40 dBZ. -# Satellite brightness temperatures are used in conjunction with radar reflectivity -# to capture both the cloud top (satellite) and in-cloud (radar) characteristics. -# Convective objects are also defined as lightning thresholds exceeding the 10th percentile. -# A percentile threshold is used for lightning data as RRFS lightning has units -# which are “non-dimensional” and therefore cannot be directly compared to the -# Geostationary Lightning Mapper. - -############################################################################## -# Version Added -# ------------- -# -# METplus Version 6.0 - -############################################################################## -# Datasets -# -------- -# -# **Forecast:** Rapid Refresh Forecast System (RRFS) 3km resolution, -# channel 13 brightness temperature, -# composite reflectivity, and lightning strike density -# -# **Observation:** Geostationary Operational Environmental Satellites (GOES) 3km resolution, -# channel 13 brightness temperature; -# Multi-radar Multi-sensor (MRMS) 3km resolution, composite reflectivity; -# GOES Global Lightning Mapper (GLM) 3km resolution, flash_extent_density -# -# **Climatology:** None -# -# **Location:** All of the input data required for this use case can be found in a sample data -# tarball. Each use case category will have one or more sample data tarballs. It is only -# necessary to download the tarball with the use case’s dataset and not the entire -# collection of sample data. Click here to access the METplus releases page and download -# sample data for the appropriate release: https://github.com/dtcenter/METplus/releases -# This tarball should be unpacked into the directory that you will set the value of -# INPUT_BASE. See :ref:`running-metplus` for more information. - -############################################################################## -# METplus Components -# ------------------ -# -# The only tool this use case calls is MODE, which will identify super-objects -# by intersection of the multiple variable fields. -# - -############################################################################## -# METplus Workflow -# ---------------- -# -# | **Beginning Time (INIT_BEG):** 2024-01-09 05:00 UTC -# | **End Time (INIT_END):** 2024-01-09 05:00 UTC -# | **Increment between beginning and end times (VALID_INCREMENT):** 1 Hour -# | **Sequence of forecast leads to process (LEAD_SEQ):** 9,10 -# -# This use case runs twice, once for each forecast lead time provided. It -# creates objects valid at 14UTC and 15UTC from 09 January 2024 are compared to -# the 9h and 10h forecasts initialized at 05UTC on 9 January 2024. -# Convective objects are identified with thresholds of -# satellite brightness temperature < 235 K and radar reflectivity > 40 dBZ, -# or lightning > 10th percentile. -# In this use case, MODE super-object intensity statistics are output for both -# radar reflectivity and lightning. Using the MODE_MULTIVAR_INTENSITY_FLAG, -# users can control for which variables super object intensity statistics will be output. -# If all are set to False, then no intensity information will be output -# and only statistics relative to the super-object geometry will be available. - - - -############################################################################## -# METplus Configuration -# --------------------- -# -# METplus first loads all of the configuration files found in parm/metplus_config, -# then it loads any configuration files passed to METplus via the command line, i.e. -# parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf -# -# .. highlight:: bash -# .. literalinclude:: ../../../../parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf - -############################################################################## -# MET Configuration -# --------------------- -# -# METplus sets environment variables based on user settings in the METplus configuration file. -# See :ref:`How METplus controls MET config file settings` for more details. -# -# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** -# -# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to: -# :ref:`Overriding Unsupported MET config file settings` -# -# .. dropdown:: MODEConfig_wrapped -# -# .. literalinclude:: ../../../../parm/met_config/MODEConfig_wrapped - -############################################################################## -# Python Embedding -# ---------------- -# -# This use case does not use any Python Embedding. - -############################################################################## -# Python Scripting -# ---------------- -# -# This use case does not use any Python Scripting. - -############################################################################## -# Running METplus -# --------------- -# -# Pass the use case configuration file to the run_metplus.py script -# along with any user-specific system configuration files if desired:: -# -# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf /path/to/user_system.conf -# -# See :ref:`running-metplus` for more information. - -############################################################################## -# Expected Output -# --------------- -# -# A successful run will output the following both to the screen and to the logfile:: -# -# INFO: METplus has successfully finished running. -# -# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. -# Output for this use case will be found in -# {OUTPUT_BASE}/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning/f??, -# where the '??' characters will reflect the two forecast leads (09 and 10). -# Each of these directories will contain the following files with their appropriate -# verification times:: -# -# * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_cts.txt -# * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.nc -# * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.txt -# * mode_Fcst_LTNG_entireatmosphere_all_all_Obs_flash_extent_density_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A.ps -# * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_cts.txt -# * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.nc -# * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A_obj.txt -# * mode_Fcst_REFC_entireatmosphere_consideredasinglelayer_all_all_Obs_MergedReflectivityQCComposite_all_all_RRFS_or_ANALYSIS_090000L_20240109_140000V_000000A.ps - -############################################################################## -# Keywords -# -------- -# -# .. note:: -# -# * MODEToolUseCase -# * ShortRangeAppUseCase -# * NetCDFFileUseCase -# * MvMODEUseCase -# -# Navigate to the :ref:`quick-search` page to discover other similar use cases. -# -# -# sphinx_gallery_thumbnail_path = '_static/short_range-MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.png' -# diff --git a/internal/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile index 2df83a89c5..e9f9c3ecbb 100644 --- a/internal/scripts/docker/Dockerfile +++ b/internal/scripts/docker/Dockerfile @@ -58,5 +58,5 @@ RUN if [ ${OBTAIN_SOURCE_CODE} != "none" ]; then \ sed -i 's|MET_INSTALL_DIR = /path/to|MET_INSTALL_DIR = /usr/local|g' parm/metplus_config/*.conf; \ sed -i 's|OUTPUT_BASE = /path/to|OUTPUT_BASE = /data/output|g' parm/metplus_config/*.conf; \ sed -i 's|INPUT_BASE = /path/to|INPUT_BASE = /data/input/METplus_Data|g' parm/metplus_config/*.conf; \ - pip install .; \ + python3 setup.py install; \ fi diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index ab3a8ae341..1504a3f161 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -3,8 +3,8 @@ sonar.projectKey=METplus sonar.projectName=METplus sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME -sonar.sources=docs,internal,manage_externals,metplus,parm,ush -sonar.coverage.exclusions=internal/tests/**,parm/**,metplus/parm/**,internal/scripts/**,manage_externals/**,docs/**,metplus/produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py +sonar.sources=docs,internal,manage_externals,metplus,parm,produtil,ush +sonar.coverage.exclusions=internal/tests/**,parm/**,internal/scripts/**,manage_externals/**,docs/**,produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 diff --git a/internal/tests/pytests/util/run_util/test_run_util.py b/internal/tests/pytests/util/run_util/test_run_util.py index 9b95e891e5..6815e0b9d2 100644 --- a/internal/tests/pytests/util/run_util/test_run_util.py +++ b/internal/tests/pytests/util/run_util/test_run_util.py @@ -4,6 +4,7 @@ import os import re +import produtil import metplus.util.run_util as ru import metplus.util.wrapper_init as wi from metplus.wrappers.ensemble_stat_wrapper import EnsembleStatWrapper diff --git a/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py b/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py index 79ea282ac9..dfffe82f22 100644 --- a/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py +++ b/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py @@ -354,7 +354,7 @@ def test_filename_does_not_match_template(): template = "{init?fmt=%Y%m%d%H}_dog_A{lead?fmt=%HH}h" filepath = "1987020103_cat_A03h" out = parse_template(template, filepath) - assert out is None + assert out == None @pytest.mark.util @@ -362,7 +362,7 @@ def test_filename_does_not_match_template_end(): template = "{init?fmt=%Y%m%d%H}_dog_A{lead?fmt=%HH}h" filepath = "1987020103_dog_A03d" out = parse_template(template, filepath) - assert out is None + assert out == None @pytest.mark.util @@ -379,7 +379,7 @@ def test_get_tags(): # value is the formatted output string, like 01 # ttype is the unit to check, i.e. 'H', 'M', 'S', 'd', 's' @pytest.mark.parametrize( - 'fmt, key, value, ttype', [ + 'format, key, value, ttype', [ ('H', 1, '01', 'H'), ('1H', 1, '1', 'H'), ('2H', 1, '01', 'H'), @@ -418,15 +418,15 @@ def test_get_tags(): ] ) @pytest.mark.util -def test_format_one_time_item(fmt, key ,value, ttype): - assert format_one_time_item(fmt, key, ttype) == value +def test_format_one_time_item(format, key ,value, ttype): + assert format_one_time_item(format, key, ttype) == value # format is the time format to use, like, %M or %H%M # seconds is the integer number of seconds of the offset to use, i.e. 3601 # value is the formatted output string, like 010001 @pytest.mark.parametrize( - 'fmt, seconds, value', [ + 'format, seconds, value', [ ('%H', 1, '00'), ('%M', 1, '00'), ('%S', 1, '01'), @@ -448,9 +448,9 @@ def test_format_one_time_item(fmt, key ,value, ttype): ] ) @pytest.mark.util -def test_format_hms(fmt, seconds, value): +def test_format_hms(format, seconds, value): # format should be something like %M or %H%M - assert format_hms(fmt, seconds == value) + assert format_hms(format, seconds == value) @pytest.mark.util @@ -539,6 +539,7 @@ def test_populate_match_dict(template, filepath, expected_match_dict, expected_v elif match_dict is None: # if expected is not None, fail if actual is None assert False + return num_keys = len(match_dict.keys()) expected_num_keys = len(expected_match_dict.keys()) @@ -561,6 +562,8 @@ def test_populate_match_dict(template, filepath, expected_match_dict, expected_v print(f"Incorrect valid shift. Actual {valid_shift}, Expected: {expected_valid_shift}") assert False + assert True + except TypeError: assert expected_match_dict is None and expected_valid_shift is None @@ -588,6 +591,8 @@ def test_get_fmt_info(fmt, filepath, identifier, expected_fmt_len, expected_matc print(f"Expected Dictionary: {expected_match_dict}") assert False + assert True + @pytest.mark.parametrize( 'templ, expected_filename', [ diff --git a/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py b/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py index 0be6a1b369..4209c2e047 100644 --- a/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py @@ -183,6 +183,25 @@ def test_ascii2nc_missing_inputs(metplus_config, get_test_data_dir, 'type = ["min", "max", "range", "mean", "stdev", "median", "p80"];' 'vld_freq = 0;vld_thresh = 0.5;}')}), + ({'TIME_OFFSET_WARNING': '3'}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;', + 'METPLUS_TIME_SUMMARY_DICT': + ('time_summary = {flag = FALSE;raw_data = FALSE;beg = "000000";' + 'end = "235959";step = 300;width = 600;' + 'grib_code = [11, 204, 211];obs_var = [];' + 'type = ["min", "max", "range", "mean", "stdev", "median", "p80"];' + 'vld_freq = 0;vld_thresh = 0.0;}') + }), + + ({'TIME_OFFSET_WARNING': '3', 'ASCII2NC_TIME_OFFSET_WARNING': '4'}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;', + 'METPLUS_TIME_SUMMARY_DICT': + ('time_summary = {flag = FALSE;raw_data = FALSE;beg = "000000";' + 'end = "235959";step = 300;width = 600;' + 'grib_code = [11, 204, 211];obs_var = [];' + 'type = ["min", "max", "range", "mean", "stdev", "median", "p80"];' + 'vld_freq = 0;vld_thresh = 0.0;}') + }), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py index b17171495d..b905a79705 100644 --- a/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py @@ -637,6 +637,13 @@ def test_ensemble_stat_field_info(metplus_config, config_overrides, 'OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE': obs_point_template}, {}), + ({'ENSEMBLE_STAT_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + ({'TIME_OFFSET_WARNING': 2, 'ENSEMBLE_STAT_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), + # fcst climo_mean ({'ENSEMBLE_STAT_FCST_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', }, {'METPLUS_FCST_CLIMO_MEAN_DICT': 'climo_mean = {file_name = ["/some/climo_mean/file.txt"];}'}), diff --git a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py index 8c7cf09863..43f22587bf 100644 --- a/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -446,6 +446,15 @@ def test_gen_ens_prod_missing_inputs(metplus_config, get_test_data_dir, allow_mi # 68 ({'GEN_ENS_PROD_NORMALIZE': 'CLIMO_STD_ANOM', }, {'METPLUS_NORMALIZE': 'normalize = CLIMO_STD_ANOM;'}), + # 69 + ({'GEN_ENS_PROD_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + # 70 + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + # 71 + ({'TIME_OFFSET_WARNING': 2, 'GEN_ENS_PROD_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), # ens climo_mean (quietly supported) ({'GEN_ENS_PROD_ENS_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', }, {'METPLUS_ENS_CLIMO_MEAN_DICT': 'climo_mean = {file_name = ["/some/climo_mean/file.txt"];}'}), diff --git a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py index d190d37f06..7179109ddc 100644 --- a/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py +++ b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py @@ -345,6 +345,12 @@ def test_get_config_file(metplus_config): 'censor_val = [12000, 5000];}' )}), + ({'GRID_DIAG_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + ({'TIME_OFFSET_WARNING': 2, 'GRID_DIAG_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py index c6e89fbd25..cc49602b90 100644 --- a/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py @@ -230,7 +230,15 @@ def test_ioda2nc_missing_inputs(metplus_config, get_test_data_dir, missing, 'IODA2NC_NMSG': '10', }, {}, ' -iodafile *INPUT_DIR*/other/file.nc -valid_beg 20200309_12 -valid_end 20200310_12 -nmsg 10'), - + # 40 + ({'IODA2NC_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}, ''), + # 41 + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}, ''), + # 42 + ({'TIME_OFFSET_WARNING': 2, 'IODA2NC_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}, ''), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py b/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py index 86a7e14a57..59c2346968 100644 --- a/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py @@ -139,6 +139,8 @@ def test_madis2nc_missing_inputs(metplus_config, get_test_data_dir, 'MADIS2NC_MASK_GRID': 'mask_grid', 'MADIS2NC_MASK_POLY': '/some/path/to/mask/poly', 'MADIS2NC_MASK_SID': 'mask_sid,/some/path/to/mask/sid'}, {}), + ({'MADIS2NC_TIME_OFFSET_WARNING': '4', }, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) diff --git a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py index 6506afbff5..97c9b6bd6c 100644 --- a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py +++ b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py @@ -200,6 +200,12 @@ def test_mtd_missing_inputs(metplus_config, get_test_data_dir, ({'MTD_OUTPUT_PREFIX': 'my_output_prefix'}, {'METPLUS_OUTPUT_PREFIX': 'output_prefix = "my_output_prefix";'}), + ({'MTD_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + ({'TIME_OFFSET_WARNING': 2, 'MTD_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py index 09f1807253..958806cb2d 100644 --- a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py @@ -252,6 +252,13 @@ def test_find_input_files(metplus_config, offsets, offset_to_find): ({'PB2NC_VALID_BEGIN': valid_beg}, {}), ({'PB2NC_VALID_END': valid_end}, {}), ({'PB2NC_VALID_BEGIN': valid_beg, 'PB2NC_VALID_END': valid_end}, {}), + + ({'PB2NC_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + ({'TIME_OFFSET_WARNING': 2, 'PB2NC_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), # 1 extra file ({'PB2NC_INPUT_TEMPLATE': ('ndas.t{da_init?fmt=%H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}.nr,' 'another_file.nr')}, {}), diff --git a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py index e69a324418..337d2d55ed 100644 --- a/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py +++ b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py @@ -248,6 +248,12 @@ def test_plot_point_obs_missing_inputs(metplus_config, get_test_data_dir, ({'PLOT_POINT_OBS_POINT_DATA': point_data_input, }, {'METPLUS_POINT_DATA': f'point_data = {point_data_format}'}), + ({'PLOT_POINT_OBS_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + ({'TIME_OFFSET_WARNING': 2, 'PLOT_POINT_OBS_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper_c diff --git a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py index c9a869ff28..87a980f41e 100644 --- a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py +++ b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py @@ -79,56 +79,25 @@ def test_point2grid_missing_inputs(metplus_config, get_test_data_dir, @pytest.mark.parametrize( - 'config_overrides, env_var_values, optional_args', [ - ({}, {}, []), - ({'POINT2GRID_REGRID_METHOD': 'UW_MEAN'}, {}, ['-method UW_MEAN']), + 'config_overrides, optional_args', [ + ({}, {}), + ({'POINT2GRID_REGRID_METHOD': 'UW_MEAN'}, ['-method UW_MEAN']), ({'POINT2GRID_REGRID_METHOD': 'UW_MEAN', - 'POINT2GRID_GAUSSIAN_DX': '2',}, {}, + 'POINT2GRID_GAUSSIAN_DX': '2',}, ['-method UW_MEAN', '-gaussian_dx 2']), - ({'POINT2GRID_GAUSSIAN_RADIUS': '81.231'}, {}, + ({'POINT2GRID_GAUSSIAN_RADIUS': '81.231'}, ['-gaussian_radius 81.231']), - ({'POINT2GRID_PROB_CAT_THRESH': '1'}, {}, ['-prob_cat_thresh 1']), - ({'POINT2GRID_VLD_THRESH': '0.5'}, {}, ['-vld_thresh 0.5']), - ({'POINT2GRID_ADP': '{valid?fmt=%Y%m}.nc'}, {}, ['-adp 201706.nc']), - ({'POINT2GRID_REGRID_TO_GRID': 'G212'}, {}, []), - ({'POINT2GRID_REGRID_TO_GRID': 'lambert 614 428 12.190 -133.459 -95.0 12.19058 6367.47 25.0 N'}, {}, []), - ({'POINT2GRID_INPUT_LEVEL': '(*,*)'}, {}, []), - ({'POINT2GRID_VALID_TIME': '20240509_120800', }, - {'METPLUS_VALID_TIME': 'valid_time = "20240509_120800";'}, []), - - ({'POINT2GRID_OBS_WINDOW_BEG': '-5400', }, - {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -5400;}'}, []), - - ({'POINT2GRID_OBS_WINDOW_END': '3600', }, - {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {end = 3600;}'}, []), - - ({'POINT2GRID_OBS_WINDOW_BEG': '-3600', 'POINT2GRID_OBS_WINDOW_END': '5400'}, - {'METPLUS_OBS_WINDOW_DICT': 'obs_window = {beg = -3600;end = 5400;}'}, []), - ({'POINT2GRID_MESSAGE_TYPE': 'ADPSFC, ADPUPA'}, - {'METPLUS_MESSAGE_TYPE': 'message_type = ["ADPSFC", "ADPUPA"];'}, []), - - ({'POINT2GRID_VAR_NAME_MAP1_KEY': '3', 'POINT2GRID_VAR_NAME_MAP1_VAL': 'MAGIC'}, - {'METPLUS_VAR_NAME_MAP_LIST': 'var_name_map = [{key = "3";val = "MAGIC";}];'}, []), - - ({'POINT2GRID_VAR_NAME_MAP1_KEY': '13', 'POINT2GRID_VAR_NAME_MAP1_VAL': 'LUCKY', - 'POINT2GRID_VAR_NAME_MAP2_KEY': '3', 'POINT2GRID_VAR_NAME_MAP2_VAL': 'MAGIC' - }, - {'METPLUS_VAR_NAME_MAP_LIST': 'var_name_map = [{key = "13";val = "LUCKY";},{key = "3";val = "MAGIC";}];'}, []), - - ({'POINT2GRID_OBS_QUALITY_INC': '0, 1, 2', }, - {'METPLUS_OBS_QUALITY_INC': 'obs_quality_inc = ["0", "1", "2"];'}, []), - - ({'POINT2GRID_OBS_QUALITY_EXC': '3,4, 5', }, - {'METPLUS_OBS_QUALITY_EXC': 'obs_quality_exc = ["3", "4", "5"];'}, []), - ({'POINT2GRID_GOES_QC_FLAGS': '0,1'}, {}, ['-goes_qc 0,1']), - ({'POINT2GRID_QC_FLAGS': '0,1'}, {}, ['-goes_qc 0,1']), - ({'POINT2GRID_GOES_QC_FLAGS': '0,1', 'POINT2GRID_QC_FLAGS': '2,3'}, {}, ['-goes_qc 0,1']), - + ({'POINT2GRID_PROB_CAT_THRESH': '1'}, ['-prob_cat_thresh 1']), + ({'POINT2GRID_VLD_THRESH': '0.5'}, ['-vld_thresh 0.5']), + ({'POINT2GRID_QC_FLAGS': '0,1'}, ['-qc 0,1']), + ({'POINT2GRID_ADP': '{valid?fmt=%Y%m}.nc'}, ['-adp 201706.nc']), + ({'POINT2GRID_REGRID_TO_GRID': 'G212'}, []), + ({'POINT2GRID_REGRID_TO_GRID': 'lambert 614 428 12.190 -133.459 -95.0 12.19058 6367.47 25.0 N'}, []), + ({'POINT2GRID_INPUT_LEVEL': '(*,*)'}, []), ] ) @pytest.mark.wrapper -def test_point2grid_run(metplus_config, config_overrides, optional_args, - env_var_values): +def test_point2grid_run(metplus_config, config_overrides, optional_args): config = metplus_config set_minimum_config_settings(config) @@ -174,19 +143,13 @@ def test_point2grid_run(metplus_config, config_overrides, optional_args, else: level = '' - config_file = wrapper.c_dict.get('CONFIG_FILE') extra_args = " ".join(optional_args) + " " if optional_args else "" - - missing_env = [item for item in env_var_values - if item not in wrapper.WRAPPER_ENV_VAR_KEYS] - env_var_keys = wrapper.WRAPPER_ENV_VAR_KEYS + missing_env - expected_cmds = [] - for idx in range(0, len(input_files)): + for idx in range(0, 3): expected_cmds.append( f'{app_path} {input_files[idx]} {grids[idx]} {output_files[idx]}' f' -field \'name="{input_name}"; level="{level}";\'' - f' -config {config_file} {extra_args}{verbosity}' + f' {extra_args}{verbosity}' ) all_cmds = wrapper.run_all_times() @@ -195,30 +158,3 @@ def test_point2grid_run(metplus_config, config_overrides, optional_args, for (cmd, env_vars), expected_cmd in zip(all_cmds, expected_cmds): # ensure commands are generated as expected assert cmd == expected_cmd - - # check that environment variables were set properly - # including deprecated env vars (not in wrapper env var keys) - for env_var_key in env_var_keys: - print(f"ENV VAR: {env_var_key}") - match = next((item for item in env_vars if - item.startswith(env_var_key)), None) - assert match is not None - value = match.split('=', 1)[1] - assert env_var_values.get(env_var_key, '') == value - - -@pytest.mark.wrapper -def test_get_config_file(metplus_config): - fake_config_name = '/my/config/file' - - config = metplus_config - default_config_file = os.path.join(config.getdir('PARM_BASE'), - 'met_config', - 'Point2GridConfig_wrapped') - - wrapper = Point2GridWrapper(config) - assert wrapper.c_dict['CONFIG_FILE'] == default_config_file - - config.set('config', 'POINT2GRID_CONFIG_FILE', fake_config_name) - wrapper = Point2GridWrapper(config) - assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name diff --git a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py index 3660a1b37a..bc59a795ad 100755 --- a/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py @@ -671,6 +671,13 @@ def test_met_dictionary_in_var_options(metplus_config): ({'OBS_POINT_STAT_INPUT_TEMPLATE': '{valid?fmt=%Y%m%d%H}/obs_file,{valid?fmt=%Y%m%d%H}/obs_file2', }, {}), ({'OBS_POINT_STAT_INPUT_TEMPLATE': '{valid?fmt=%Y%m%d%H}/obs_file,{valid?fmt=%Y%m%d%H}/obs_file2,{valid?fmt=%Y%m%d%H}/obs_file3', }, {}), + ({'POINT_STAT_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + ({'TIME_OFFSET_WARNING': 2, 'POINT_STAT_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), + # fcst climo_mean ({'POINT_STAT_FCST_CLIMO_MEAN_FILE_NAME': '/some/climo_mean/file.txt', }, {'METPLUS_FCST_CLIMO_MEAN_DICT': 'climo_mean = {file_name = ["/some/climo_mean/file.txt"];}'}), diff --git a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py index 9dc06f9f88..6d0f619ec7 100644 --- a/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py +++ b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py @@ -173,7 +173,15 @@ def set_minimum_config_settings(config): 'MODEL1': '{custom}', 'MODEL_LIST': '{custom}'}, {'METPLUS_MODEL': 'model = ["CUSTOM_MODEL"];'}), - + # 17 time_offset_warning app + ({'STAT_ANALYSIS_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + # 18 time_offset_warning generic + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + # 19 time_offset_warning both + ({'TIME_OFFSET_WARNING': 2, 'STAT_ANALYSIS_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper_d diff --git a/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py index 3f0c5ed75e..88b9ef1eb5 100644 --- a/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_diag/test_tc_diag_wrapper.py @@ -239,6 +239,12 @@ def test_tc_diag_missing_inputs(metplus_config, get_test_data_dir, ({'TC_DIAG_ONE_TIME_PER_FILE_FLAG': 'false', }, {'METPLUS_ONE_TIME_PER_FILE_FLAG': 'one_time_per_file_flag = FALSE;'}), + ({'TC_DIAG_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + ({'TIME_OFFSET_WARNING': 2, 'TC_DIAG_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py index d1d5b24c23..c9beca4fde 100644 --- a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py @@ -339,6 +339,12 @@ def test_tc_gen_missing_inputs(metplus_config, get_test_data_dir, allow_missing, }, {'METPLUS_OPS_HIT_WINDOW_DICT': 'ops_hit_window = {beg = 1;end = 47;}'}), + ({'TC_GEN_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + ({'TIME_OFFSET_WARNING': 2, 'TC_GEN_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper_a diff --git a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index 7f9d560eed..70adbd62d5 100644 --- a/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -607,6 +607,12 @@ def test_tc_pairs_storm_id_lists(metplus_config, get_test_data_dir, config_overr }, {'DIAG_ARG': '-diag TCDIAG /bmlq2014123118.gfso.0104',}), + ('VALID', {'TC_PAIRS_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + ('VALID', {'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + ('VALID', {'TIME_OFFSET_WARNING': 2, 'TC_PAIRS_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py index 8f07f3154b..0f52648e6b 100644 --- a/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py @@ -242,6 +242,12 @@ def test_tc_stat_handle_jobs(metplus_config, config_overrides, expected_dirs, ({'TC_STAT_OUTPUT_TEMPLATE': 'tc_stat.out.nc', }, {}), + ({'TC_STAT_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + ({'TIME_OFFSET_WARNING': 2, 'TC_STAT_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py b/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py index 6721527a64..7a4d3ea6e9 100644 --- a/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py +++ b/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py @@ -122,6 +122,12 @@ def set_minimum_config_settings(config): ({'TC_RMW_RMW_SCALE': '15', }, {'METPLUS_RMW_SCALE': 'rmw_scale = 15.0;'}), + ({'TC_RMW_TIME_OFFSET_WARNING': 3}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 3;'}), + ({'TIME_OFFSET_WARNING': 2}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), + ({'TIME_OFFSET_WARNING': 2, 'TC_RMW_TIME_OFFSET_WARNING': 4}, + {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), ] ) @pytest.mark.wrapper diff --git a/internal/tests/use_cases/all_use_cases.txt b/internal/tests/use_cases/all_use_cases.txt index ef58c7ff60..1522963abe 100644 --- a/internal/tests/use_cases/all_use_cases.txt +++ b/internal/tests/use_cases/all_use_cases.txt @@ -184,7 +184,7 @@ Category: short_range 12::UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection::model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf:: metplotpy_env 13::MODEMultivar_fcstHRRR_obsMRMS_HRRRanl::model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf 14::UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot::model_applications/short_range/UserScript_fcstRRFS_fcstOnly_Reformat_Aggregate_Plot.conf:: metdataio, metcalcpy, metplotpy,mp_analysis_env -15::MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning::model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf + Category: space_weather 0::GridStat_fcstGloTEC_obsGloTEC_vx7:: model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf diff --git a/metplus/parm b/metplus/parm deleted file mode 120000 index 302dacab35..0000000000 --- a/metplus/parm +++ /dev/null @@ -1 +0,0 @@ -../parm \ No newline at end of file diff --git a/metplus/scripts/run_metplus.py b/metplus/scripts/run_metplus.py deleted file mode 120000 index 116d56b438..0000000000 --- a/metplus/scripts/run_metplus.py +++ /dev/null @@ -1 +0,0 @@ -../../ush/run_metplus.py \ No newline at end of file diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index 238b2c10b5..c0d25deea6 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -11,6 +11,7 @@ import os import re +import sys import logging from datetime import datetime, timezone import time @@ -19,7 +20,7 @@ from pathlib import Path import uuid -from metplus.produtil.config import ProdConfig +from produtil.config import ProdConfig from .constants import RUNTIME_CONFS, MISSING_DATA_VALUE from .string_template_substitution import do_string_sub @@ -62,7 +63,7 @@ # set parm base to METPLUS_BASE/parm unless METPLUS_PARM_BASE env var is set PARM_BASE = os.environ.get('METPLUS_PARM_BASE', - os.path.join(METPLUS_BASE, 'metplus', PARM_DIR)) + os.path.join(METPLUS_BASE, PARM_DIR)) # name of directory under PARM_DIR that contains defaults METPLUS_CONFIG_DIR = 'metplus_config' diff --git a/metplus/util/constants.py b/metplus/util/constants.py index 3b5c0f7ff2..3208aa2a26 100644 --- a/metplus/util/constants.py +++ b/metplus/util/constants.py @@ -93,14 +93,6 @@ 'GenEnsProd', ) -# wrappers that support the time_offset_warning global MET config variable -TIME_OFFSET_WARNING_WRAPPERS = ( - 'GridStat', - 'MODE', - 'SeriesAnalysis', - 'WaveletStat', -) - # configuration variables that are specific to a given run # these are copied from [config] to [runtime] at the # end of the run so they will not be read if the final diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index 90ace07686..145468c5a6 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -6,7 +6,7 @@ from logging import Logger import shlex -from metplus.produtil.run import exe, run +from produtil.run import exe, run from .string_manip import get_logfile_info, log_terminal_includes_info from .system_util import get_user_info, write_list_to_file diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index d47efa2617..b9ba20f3e3 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -15,8 +15,7 @@ from abc import ABCMeta from inspect import getframeinfo, stack -from ..util.constants import PYTHON_EMBEDDING_TYPES, COMPRESSION_EXTENSIONS -from ..util.constants import MULTIPLE_INPUT_WRAPPERS, TIME_OFFSET_WARNING_WRAPPERS +from ..util.constants import PYTHON_EMBEDDING_TYPES, COMPRESSION_EXTENSIONS, MULTIPLE_INPUT_WRAPPERS from ..util import getlist, preprocess_file, loop_over_times_and_call from ..util import do_string_sub, ti_calculate, get_seconds_from_string from ..util import get_time_from_file, shift_time_seconds, seconds_to_met_time @@ -104,16 +103,15 @@ def __init__(self, config, instance=None): # add key to list of env vars to set self.env_var_keys.append(self.MET_OVERRIDES_KEY) - if get_wrapper_name(self.app_name) in TIME_OFFSET_WARNING_WRAPPERS: - # add time_offset_warning env var - self.env_var_keys.append('METPLUS_TIME_OFFSET_WARNING') - time_offset_warning = self.get_wrapper_or_generic_config( - 'TIME_OFFSET_WARNING', var_type='int' + # add time_offset_warning env var + self.env_var_keys.append('METPLUS_TIME_OFFSET_WARNING') + time_offset_warning = self.get_wrapper_or_generic_config( + 'TIME_OFFSET_WARNING', var_type='int' + ) + if time_offset_warning != '': + self.env_var_dict['METPLUS_TIME_OFFSET_WARNING'] = ( + f'time_offset_warning = {time_offset_warning};' ) - if time_offset_warning != '': - self.env_var_dict['METPLUS_TIME_OFFSET_WARNING'] = ( - f'time_offset_warning = {time_offset_warning};' - ) # warn if any environment variables set by the wrapper are not # being utilized in the user's config file diff --git a/metplus/wrappers/point2grid_wrapper.py b/metplus/wrappers/point2grid_wrapper.py index 11ed39ce0c..c95a9da3d4 100755 --- a/metplus/wrappers/point2grid_wrapper.py +++ b/metplus/wrappers/point2grid_wrapper.py @@ -14,8 +14,6 @@ from ..util import do_string_sub from ..util import remove_quotes -from ..util.met_config import add_met_config_dict_list - from . import ReformatPointWrapper '''!@namespace Point2GridWrapper @@ -28,15 +26,6 @@ class Point2GridWrapper(ReformatPointWrapper): RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] - WRAPPER_ENV_VAR_KEYS = [ - 'METPLUS_VALID_TIME', - 'METPLUS_OBS_WINDOW_DICT', - 'METPLUS_MESSAGE_TYPE', - 'METPLUS_VAR_NAME_MAP_LIST', - 'METPLUS_OBS_QUALITY_INC', - 'METPLUS_OBS_QUALITY_EXC', - ] - def __init__(self, config, instance=None): self.app_name = "point2grid" self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), @@ -81,12 +70,8 @@ def create_c_dict(self): 'POINT2GRID_INPUT_LEVEL', '') - # support legacy QC_FLAGS and new GOES_QC_FLAGS to set -goes_qc arg - config_name = self.config.get_mp_config_name(['POINT2GRID_GOES_QC_FLAGS', - 'POINT2GRID_QC_FLAGS']) - if config_name: - c_dict['GOES_QC_FLAGS'] = self.config.getraw('config', config_name) - + c_dict['QC_FLAGS'] = self.config.getraw('config', + 'POINT2GRID_QC_FLAGS') c_dict['ADP'] = self.config.getraw('config', 'POINT2GRID_ADP') c_dict['REGRID_METHOD'] = self.config.getstr('config', @@ -109,28 +94,6 @@ def create_c_dict(self): 'POINT2GRID_VLD_THRESH', '') - # get the MET config file path or use default - c_dict['CONFIG_FILE'] = self.get_config_file('Point2GridConfig_wrapped') - - # read config file variables - self.add_met_config(name='valid_time', data_type='string') - self.add_met_config_window('obs_window') - self.add_met_config(name='message_type', data_type='list') - if not add_met_config_dict_list(config=self.config, - app_name=self.app_name, - output_dict=self.env_var_dict, - dict_name='var_name_map', - dict_items={'key': 'string', - 'val': 'string'}): - self.isOK = False - - self.add_met_config(name='obs_quality_inc', data_type='list', - metplus_configs=['POINT2GRID_OBS_QUALITY_INC', - 'POINT2GRID_OBS_QUALITY_INCLUDE', - 'POINT2GRID_OBS_QUALITY']) - self.add_met_config(name='obs_quality_exc', data_type='list', - metplus_configs=['POINT2GRID_OBS_QUALITY_EXC', - 'POINT2GRID_OBS_QUALITY_EXCLUDE']) return c_dict def find_input_files(self, time_info): @@ -168,11 +131,8 @@ def set_command_line_arguments(self, time_info): #Add either the specified level above or the defauilt blank one self.args.append(f"-field 'name=\"{input_field}\"; level=\"{input_level}\";'") - config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) - self.args.append(f'-config {config_file}') - - if self.c_dict.get('GOES_QC_FLAGS', '') != '': - self.args.append(f"-goes_qc {self.c_dict['GOES_QC_FLAGS']}") + if self.c_dict['QC_FLAGS'] != '': + self.args.append(f"-qc {self.c_dict['QC_FLAGS']}") if self.c_dict['ADP']: self.args.append(f"-adp {self.c_dict['ADP']}") diff --git a/metplus/wrappers/series_analysis_wrapper.py b/metplus/wrappers/series_analysis_wrapper.py index 1fb1948a6b..cea1a44b4a 100755 --- a/metplus/wrappers/series_analysis_wrapper.py +++ b/metplus/wrappers/series_analysis_wrapper.py @@ -784,7 +784,9 @@ def build_and_run_series_request(self, time_info, fcst_path, obs_path): add_field_info_to_time_info(time_info, var_info) # get formatted field dictionary to pass into the MET config file - fcst_field, obs_field = self.get_formatted_fields(var_info, time_info) + fcst_field, obs_field = ( + self.get_formatted_fields(var_info, time_info, fcst_path, obs_path) + ) if fcst_field is None: continue @@ -1033,20 +1035,22 @@ def _get_netcdf_min_max(filepath, variable_name): except (FileNotFoundError, KeyError): return None, None - def get_formatted_fields(self, var_info, time_info): + def get_formatted_fields(self, var_info, time_info, fcst_path, obs_path): """! Get forecast and observation field information for var_info and format it so it can be passed into the MET config file @param var_info dictionary containing info to format @param time_info dictionary containing time information + @param fcst_path path to file list file for forecast data + @param obs_path path to file list file for observation data @returns tuple containing strings of the formatted forecast and observation information or (None, None) if something went wrong """ fcst_field_list = ( - self._get_field_list('fcst', var_info, time_info) + self._get_field_list('fcst', var_info, time_info, obs_path) ) obs_field_list = ( - self._get_field_list('obs', var_info, time_info) + self._get_field_list('obs', var_info, time_info, fcst_path) ) if not fcst_field_list or not obs_field_list: @@ -1058,7 +1062,7 @@ def get_formatted_fields(self, var_info, time_info): return fcst_fields, obs_fields - def _get_field_list(self, data_type, var_info, time_info): + def _get_field_list(self, data_type, var_info, time_info, file_list_path): """!Get formatted field information in a list. If no time (init/valid/lead) filename template tags were found in the level value or if the time info contains all init/valid/lead values @@ -1071,8 +1075,10 @@ def _get_field_list(self, data_type, var_info, time_info): @param data_type type of data to process, e.g. fcst or obs @param var_info dictionary containing info to format @param time_info dictionary containing time information + @param file_list_path path to file list file to parse @returns list containing formatted field info to pass to MET config """ + other = 'OBS' if data_type == 'fcst' else 'FCST' # if there are no time tags (init/valid/lead) in the field level # or if init, valid, and lead have values in time_info, # get field info for a single field to pass to the MET config file @@ -1082,10 +1088,15 @@ def _get_field_list(self, data_type, var_info, time_info): field_list = [] - # loop through fcst/obs files to read time info + # handle multiple templates + templates = [] + for template in self.c_dict[f'{other}_INPUT_TEMPLATE'].split(','): + templates.append(os.path.join(self.c_dict[f'{other}_INPUT_DIR'], template.strip())) + + # loop through fcst/obs files to extract time info # for each file apply time info to field info and add to list - for file_dict in self.c_dict['ALL_FILES']: - file_time_info = file_dict['time_info'] + for file_time_info in self._get_times_from_file_list(file_list_path, + templates): field = self._get_field_sub_level(data_type, var_info, file_time_info) if field: field_list.extend(field) diff --git a/parm/met_config/Ascii2NcConfig_wrapped b/parm/met_config/Ascii2NcConfig_wrapped index 4233450615..e86116277b 100644 --- a/parm/met_config/Ascii2NcConfig_wrapped +++ b/parm/met_config/Ascii2NcConfig_wrapped @@ -38,4 +38,5 @@ message_type_map = [ tmp_dir = "${MET_TMP_DIR}"; +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/EnsembleStatConfig_wrapped b/parm/met_config/EnsembleStatConfig_wrapped index 9f508ff4c7..fb505cc09e 100644 --- a/parm/met_config/EnsembleStatConfig_wrapped +++ b/parm/met_config/EnsembleStatConfig_wrapped @@ -242,4 +242,5 @@ ${METPLUS_OUTPUT_PREFIX} tmp_dir = "${MET_TMP_DIR}"; +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/GenEnsProdConfig_wrapped b/parm/met_config/GenEnsProdConfig_wrapped index 5659d9330a..0a67225e2a 100644 --- a/parm/met_config/GenEnsProdConfig_wrapped +++ b/parm/met_config/GenEnsProdConfig_wrapped @@ -118,4 +118,5 @@ ${METPLUS_ENSEMBLE_FLAG_DICT} tmp_dir = "${MET_TMP_DIR}"; +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/GridDiagConfig_wrapped b/parm/met_config/GridDiagConfig_wrapped index 691338acf2..8c5117ddbe 100644 --- a/parm/met_config/GridDiagConfig_wrapped +++ b/parm/met_config/GridDiagConfig_wrapped @@ -41,4 +41,5 @@ ${METPLUS_MASK_DICT} tmp_dir = "${MET_TMP_DIR}"; +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/IODA2NCConfig_wrapped b/parm/met_config/IODA2NCConfig_wrapped index ba1faa1695..7f25fb8cf1 100644 --- a/parm/met_config/IODA2NCConfig_wrapped +++ b/parm/met_config/IODA2NCConfig_wrapped @@ -114,4 +114,5 @@ tmp_dir = "${MET_TMP_DIR}"; //////////////////////////////////////////////////////////////////////////////// +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/MTDConfig_wrapped b/parm/met_config/MTDConfig_wrapped index d521670eac..f010982346 100644 --- a/parm/met_config/MTDConfig_wrapped +++ b/parm/met_config/MTDConfig_wrapped @@ -257,4 +257,5 @@ tmp_dir = "${MET_TMP_DIR}"; //////////////////////////////////////////////////////////////////////////////// +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/Madis2NcConfig_wrapped b/parm/met_config/Madis2NcConfig_wrapped index 4cafab0038..282f6782f1 100644 --- a/parm/met_config/Madis2NcConfig_wrapped +++ b/parm/met_config/Madis2NcConfig_wrapped @@ -25,4 +25,6 @@ ${METPLUS_TIME_SUMMARY_DICT} // //version = "V12.0.0"; +${METPLUS_TIME_OFFSET_WARNING} + ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/PB2NCConfig_wrapped b/parm/met_config/PB2NCConfig_wrapped index 8b9057923b..6174608bb9 100644 --- a/parm/met_config/PB2NCConfig_wrapped +++ b/parm/met_config/PB2NCConfig_wrapped @@ -129,4 +129,5 @@ tmp_dir = "${MET_TMP_DIR}"; //////////////////////////////////////////////////////////////////////////////// +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/PlotPointObsConfig_wrapped b/parm/met_config/PlotPointObsConfig_wrapped index 2194f01fee..8fd113b86d 100644 --- a/parm/met_config/PlotPointObsConfig_wrapped +++ b/parm/met_config/PlotPointObsConfig_wrapped @@ -96,4 +96,5 @@ tmp_dir = "${MET_TMP_DIR}"; //////////////////////////////////////////////////////////////////////////////// +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/Point2GridConfig_wrapped b/parm/met_config/Point2GridConfig_wrapped deleted file mode 100644 index 5ebc51a460..0000000000 --- a/parm/met_config/Point2GridConfig_wrapped +++ /dev/null @@ -1,54 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Point2Grid configuration file. -// -// For additional information, please see the MET User's Guide. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// NetCDF output variable timing information: -// - valid_time in YYYYMMDD[_HH[MMSS]] format -// -//valid_time = -${METPLUS_VALID_TIME} - -// -// Observation time window -// -//obs_window = { -${METPLUS_OBS_WINDOW_DICT} - -//////////////////////////////////////////////////////////////////////////////// - - -// -// Observation message type -// -//message_type = [ -${METPLUS_MESSAGE_TYPE} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Mapping of input variable names to output variables names. -// -//var_name_map = [ -${METPLUS_VAR_NAME_MAP_LIST} - -//////////////////////////////////////////////////////////////////////////////// - -//obs_quality_inc = [ -${METPLUS_OBS_QUALITY_INC} - -//obs_quality_exc = [ -${METPLUS_OBS_QUALITY_EXC} - -//////////////////////////////////////////////////////////////////////////////// - -tmp_dir = "${MET_TMP_DIR}"; -//version = "V12.0.0"; - -//////////////////////////////////////////////////////////////////////////////// - -${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index 0d12bd0a73..7b717c6d2c 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -216,4 +216,5 @@ ${METPLUS_OUTPUT_PREFIX} //////////////////////////////////////////////////////////////////////////////// +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/STATAnalysisConfig_wrapped b/parm/met_config/STATAnalysisConfig_wrapped index d38c8def2d..e3e32deba4 100644 --- a/parm/met_config/STATAnalysisConfig_wrapped +++ b/parm/met_config/STATAnalysisConfig_wrapped @@ -164,4 +164,5 @@ tmp_dir = "${MET_TMP_DIR}"; //version = "V10.0"; +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCDiagConfig_wrapped b/parm/met_config/TCDiagConfig_wrapped index 8dacb81b26..0407e982c5 100644 --- a/parm/met_config/TCDiagConfig_wrapped +++ b/parm/met_config/TCDiagConfig_wrapped @@ -180,4 +180,5 @@ tmp_dir = "${MET_TMP_DIR}"; //////////////////////////////////////////////////////////////////////////////// +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCGenConfig_wrapped b/parm/met_config/TCGenConfig_wrapped index c18f895f8b..7d813aa105 100644 --- a/parm/met_config/TCGenConfig_wrapped +++ b/parm/met_config/TCGenConfig_wrapped @@ -294,4 +294,5 @@ ${METPLUS_NC_PAIRS_GRID} tmp_dir = "${MET_TMP_DIR}"; +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCPairsConfig_wrapped b/parm/met_config/TCPairsConfig_wrapped index 93947c0c18..7b9aa04651 100644 --- a/parm/met_config/TCPairsConfig_wrapped +++ b/parm/met_config/TCPairsConfig_wrapped @@ -168,4 +168,5 @@ ${METPLUS_DIAG_CONVERT_MAP_LIST} tmp_dir = "${MET_TMP_DIR}"; +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCRMWConfig_wrapped b/parm/met_config/TCRMWConfig_wrapped index 9df3ddec8f..00a7ea0ad7 100644 --- a/parm/met_config/TCRMWConfig_wrapped +++ b/parm/met_config/TCRMWConfig_wrapped @@ -89,4 +89,5 @@ ${METPLUS_RMW_SCALE} tmp_dir = "${MET_TMP_DIR}"; +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/met_config/TCStatConfig_wrapped b/parm/met_config/TCStatConfig_wrapped index dc4d5ec780..a6bd4afb23 100644 --- a/parm/met_config/TCStatConfig_wrapped +++ b/parm/met_config/TCStatConfig_wrapped @@ -233,4 +233,5 @@ ${METPLUS_JOBS} tmp_dir = "${MET_TMP_DIR}"; +${METPLUS_TIME_OFFSET_WARNING} ${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf b/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf index b6a0de754e..f1aa331915 100644 --- a/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf +++ b/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf @@ -79,3 +79,5 @@ ASCII2NC_TIME_SUMMARY_VAR_NAMES = ASCII2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 ASCII2NC_TIME_SUMMARY_VALID_FREQ = 0 ASCII2NC_TIME_SUMMARY_VALID_THRESH = 0.0 + +#ASCII2NC_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf index 3ac91314f2..3f911cbf71 100644 --- a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf +++ b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf @@ -246,3 +246,5 @@ ENSEMBLE_STAT_NC_ORANK_FLAG_WEIGHT = FALSE #ENSEMBLE_STAT_CONTROL_ID = #ENSEMBLE_STAT_GRID_WEIGHT_FLAG = + +#ENSEMBLE_STAT_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf index 55ff515571..2d2394d374 100644 --- a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf +++ b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf @@ -165,3 +165,5 @@ GEN_ENS_PROD_ENS_THRESH = 0.8 #GEN_ENS_PROD_ENS_MEMBER_IDS = #GEN_ENS_PROD_CONTROL_ID = + +#GEN_ENS_PROD_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf b/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf index 4104ee6016..23813198fa 100644 --- a/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf +++ b/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf @@ -83,3 +83,5 @@ GRID_DIAG_CONFIG_FILE = {PARM_BASE}/met_config/GridDiagConfig_wrapped #GRID_DIAG_REGRID_CENSOR_VAL = GRID_DIAG_MASK_POLY = MET_BASE/poly/SAO.poly + +#GRID_DIAG_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf b/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf index 0948944a55..a036901873 100644 --- a/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf +++ b/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf @@ -96,3 +96,5 @@ IODA2NC_TIME_SUMMARY_OBS_VAR = "WIND" IODA2NC_TIME_SUMMARY_TYPE = "min", "max", "range", "mean", "stdev", "median", "p80" IODA2NC_TIME_SUMMARY_VLD_FREQ = 0 IODA2NC_TIME_SUMMARY_VLD_THRESH = 0.0 + +#IODA2NC_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf b/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf index 41b8009075..ac4067dd6c 100644 --- a/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf +++ b/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf @@ -83,4 +83,6 @@ MADIS2NC_TYPE = metar #MADIS2NC_TIME_SUMMARY_VLD_FREQ = #MADIS2NC_TIME_SUMMARY_VLD_THRESH = +#MADIS2NC_TIME_OFFSET_WARNING = + #MADIS2NC_MET_CONFIG_OVERRIDES = \ No newline at end of file diff --git a/parm/use_cases/met_tool_wrapper/MTD/MTD.conf b/parm/use_cases/met_tool_wrapper/MTD/MTD.conf index 6f615537e0..30d3c1403c 100644 --- a/parm/use_cases/met_tool_wrapper/MTD/MTD.conf +++ b/parm/use_cases/met_tool_wrapper/MTD/MTD.conf @@ -99,3 +99,5 @@ MTD_REGRID_TO_GRID = OBS MTD_MIN_VOLUME = 2000 MTD_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_NAME}_{CURRENT_FCST_LEVEL} + +#MTD_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf b/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf index 93651289ea..9acd346c18 100644 --- a/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf +++ b/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf @@ -94,3 +94,5 @@ PB2NC_TIME_SUMMARY_VALID_FREQ = 0 PB2NC_TIME_SUMMARY_VALID_THRESH = 0.0 #PB2NC_OBS_BUFR_MAP = + +#PB2NC_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf index 304e1c5253..6289088f02 100644 --- a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf +++ b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf @@ -131,3 +131,5 @@ PLOT_POINT_OBS_POINT_DATA = colorbar_flag = FALSE; } } + +#PLOT_POINT_OBS_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf index 03695fcc67..570ee1cc81 100644 --- a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf +++ b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf @@ -68,24 +68,14 @@ POINT2GRID_REGRID_METHOD = MAX POINT2GRID_INPUT_FIELD =TMP POINT2GRID_INPUT_LEVEL = -#POINT2GRID_GOES_QC_FLAGS = +#POINT2GRID_QC_FLAGS = POINT2GRID_ADP = + POINT2GRID_GAUSSIAN_DX = 81.271 POINT2GRID_GAUSSIAN_RADIUS = 120 POINT2GRID_PROB_CAT_THRESH = POINT2GRID_VLD_THRESH = - -#POINT2GRID_VALID_TIME = -#POINT2GRID_OBS_WINDOW_BEG = -#POINT2GRID_OBS_WINDOW_END = -#POINT2GRID_MESSAGE_TYPE = -#POINT2GRID_VAR_NAME_MAP1_KEY = -#POINT2GRID_VAR_NAME_MAP1_VAL = -#POINT2GRID_OBS_QUALITY_INC = -#POINT2GRID_OBS_QUALITY_EXC = - -#POINT2GRID_MET_CONFIG_OVERRIDES = diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf index 6d793e2bf9..3cfaa3b52e 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf @@ -264,3 +264,5 @@ POINT_STAT_MESSAGE_TYPE = ADPUPA, ADPSFC #POINT_STAT_UGRID_MAX_DISTANCE_KM = #POINT_STAT_UGRID_COORDINATES_FILE = #POINT_STAT_UGRID_CONFIG_FILE = + +#POINT_STAT_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf b/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf index 14eb180192..59378482fe 100644 --- a/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf +++ b/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf @@ -84,3 +84,5 @@ LINE_TYPE_LIST = GROUP_LIST_ITEMS = FCST_INIT_HOUR_LIST LOOP_LIST_ITEMS = FCST_VALID_HOUR_LIST, MODEL_LIST + +#STAT_ANALYSIS_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf index b10919341f..078e9c04ae 100644 --- a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf +++ b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf @@ -117,3 +117,5 @@ TC_DIAG_DOMAIN_INFO1_OVERRIDE_DIAGS = #TC_DIAG_NC_DIAG_FLAG = #TC_DIAG_CIRA_DIAG_FLAG = #TC_DIAG_OUTPUT_BASE_FORMAT = + +#TC_DIAG_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf b/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf index ace36034eb..424c3376bb 100644 --- a/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf +++ b/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf @@ -177,3 +177,5 @@ TC_GEN_DLAND_FILE = MET_BASE/tc_data/dland_global_tenth_degree.nc TC_GEN_BASIN_FILE = MET_BASE/tc_data/basin_global_tenth_degree.nc TC_GEN_NC_PAIRS_GRID = G003 + +#TC_GEN_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf index e5a0bcf1eb..231138ef23 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf @@ -122,3 +122,5 @@ TC_PAIRS_MISSING_VAL = -9999 #TC_PAIRS_DIAG_CONVERT_MAP1_DIAG_SOURCE = #TC_PAIRS_DIAG_CONVERT_MAP1_KEY = #TC_PAIRS_DIAG_CONVERT_MAP1_CONVERT = + +#TC_PAIRS_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf index 60b2e494db..a410e91ffe 100644 --- a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf +++ b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf @@ -95,3 +95,5 @@ TC_RMW_CYCLONE = 14 #TC_RMW_VALID_INCLUDE_LIST = #TC_RMW_VALID_EXCLUDE_LIST = #TC_RMW_VALID_HOUR_LIST = + +#TC_RMW_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf b/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf index ec1705915b..7f4be36833 100644 --- a/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf +++ b/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf @@ -117,3 +117,5 @@ TC_STAT_MATCH_POINTS = false #TC_STAT_EVENT_EQUAL_LEAD = #TC_STAT_OUT_INIT_MASK = #TC_STAT_OUT_VALID_MASK = + +#TC_STAT_TIME_OFFSET_WARNING = diff --git a/parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf b/parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf deleted file mode 100644 index 7bd4c7fb7b..0000000000 --- a/parm/use_cases/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.conf +++ /dev/null @@ -1,132 +0,0 @@ -[config] - -# Documentation for this use-case can be found at: -# https://metplus.readthedocs.io/en/latest/generated/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning.html - -# Processes to run -PROCESS_LIST = MODE - -# Time Info -LOOP_BY = INIT -INIT_TIME_FMT = %Y%m%d%H -INIT_BEG=2024010905 -INIT_END=2024010905 -INIT_INCREMENT = 1H - -LEAD_SEQ = 9,10 - -MODEL = RRFS -OBTYPE = ANALYSIS - -################################## -# Multivariate MODE Configurations -################################## - -# Run MODE to output super objects -MODE_MULTIVAR_LOGIC = #1 && #2 || #3 -MODE_MULTIVAR_INTENSITY_FLAG = FALSE,TRUE,TRUE - -FCST_MODE_INPUT_DIR = {INPUT_BASE}/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning -FCST_MODE_INPUT_TEMPLATE= {init?fmt=%Y%m%d}.{init?fmt=%H}00.F{lead?fmt=%HH}00.nc,{init?fmt=%Y%m%d}.{init?fmt=%H}00.F{lead?fmt=%HH}00.nc,{init?fmt=%Y%m%d}.{init?fmt=%H}00.F{lead?fmt=%HH}00.nc - -OBS_MODE_INPUT_DIR = {INPUT_BASE}/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning -OBS_MODE_INPUT_TEMPLATE = remap_GOES-16.{valid?fmt=%Y%m%d}.{valid?fmt=%H}0000.nc,{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000_remap.nc,remap_GOES-16_lightning_{valid?fmt=%Y%m%d}-{valid?fmt=%H}00_75W.nc - -MODE_OUTPUT_DIR = {OUTPUT_BASE}/model_applications/short_range/MODEMultivar_fcstRRFS_obsGOES_MRMS_BrightnessTemp_Lightning -MODE_OUTPUT_TEMPLATE = f{lead?fmt=%2H} - -FCST_MODE_FILE_WINDOW_BEGIN = 0 -FCST_MODE_FILE_WINDOW_END = 0 - -OBS_MODE_FILE_WINDOW_BEGIN = 0 -OBS_MODE_FILE_WINDOW_END = 0 - -### -# Field Info -### - - -FCST_VAR1_NAME = bt13_topofatmosphere_G16 -FCST_VAR1_LEVELS = (*,*) -FCST_VAR1_OPTIONS = conv_radius = 5; conv_thresh = <=230; merge_thresh = <=230; merge_flag = NONE; file_type = NETCDF_MET; filter_attr_name = ["AREA"]; filter_attr_thresh= [ >=1 ]; censor_thresh = [<=0]; censor_val = [9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA - - -OBS_VAR1_NAME = channel_13_brightness_temperature -OBS_VAR1_LEVELS = L0 -OBS_VAR1_OPTIONS = conv_radius = 0; conv_thresh = <=230; merge_thresh = <=230; merge_flag = NONE; file_type = NETCDF_MET; filter_attr_name = ["AREA"]; filter_attr_thresh= [ >=1 ]; censor_thresh = [<=0]; censor_val = [9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA - - -FCST_VAR2_NAME = REFC_entireatmosphere_consideredasinglelayer -FCST_VAR2_LEVELS = (*,*) -FCST_VAR2_OPTIONS = conv_radius = 5; conv_thresh = >=40; merge_thresh = >=40; merge_flag = NONE; file_type = NETCDF_MET; filter_attr_name = ["AREA"]; filter_attr_thresh= [ >=1 ]; censor_thresh = [< -30]; censor_val = [9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA - -OBS_VAR2_NAME = MergedReflectivityQCComposite -OBS_VAR2_LEVELS = (*,*) -OBS_VAR2_OPTIONS = conv_radius = 5; conv_thresh = >=40; merge_thresh = >=40; merge_flag = NONE; file_type = NETCDF_MET; filter_attr_name = ["AREA"]; filter_attr_thresh= [ >=1 ]; censor_thresh = [< -30]; censor_val = [9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA - - -FCST_VAR3_NAME = LTNG_entireatmosphere -FCST_VAR3_LEVELS = (*,*) -FCST_VAR3_OPTIONS = conv_radius = 3; conv_thresh = >SFP10; merge_thresh = >SFP10; merge_flag = THRESH; file_type = NETCDF_MET; censor_thresh = [<= 0]; censor_val = [-9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA - -OBS_VAR3_NAME = flash_extent_density -OBS_VAR3_LEVELS = (*,*) -OBS_VAR3_OPTIONS = conv_radius = 3; conv_thresh = >SOP10; merge_thresh = >SOP10; merge_flag = THRESH; file_type = NETCDF_MET; censor_thresh = [<= 0]; censor_val = [-9999]; vld_thresh = 0.5; inten_perc_value = 100; inten_perc_thresh = NA - - -MODE_MATCH_FLAG = MERGE_BOTH - -MODE_REGRID_TO_GRID = NONE - -LOG_MODE_VERBOSITY = 4 - -MODE_OUTPUT_PREFIX = {MODEL}_or_{OBTYPE} - -MODE_GRID_RES = 3 - -MODE_INTEREST_FUNCTION_CENTROID_DIST = (( 0.0, 1.0 )( 60.0/grid_res, 1.0 )( 450.0/grid_res, 0.0 )) -MODE_INTEREST_FUNCTION_BOUNDARY_DIST = (( 0.0, 1.0 )( 400.0/grid_res, 0.0 )) -MODE_INTEREST_FUNCTION_CONVEX_HULL_DIST = (( 0.0, 1.0 )( 400.0/grid_res, 0.0 )) -MODE_INTEREST_FUNCTION_ANGLE_DIFF = (( 0.0, 1.0 )( 30.0, 1.0 )( 90.0, 0.0 )) -MODE_INTEREST_FUNCTION_ASPECT_DIFF = (( 0.00, 1.0 )( 0.10, 1.0 )( 0.75, 0.0 )) -MODE_INTEREST_FUNCTION_AREA_RATIO = (( 0.0, 0.0 )( 0.8, 1.0 )( 1.0, 1.0 )) -MODE_INTEREST_FUNCTION_INT_AREA_RATIO = (( 0.00, 0.00 )( 0.10, 0.50 )( 0.25, 1.00 )( 1.00, 1.00 )) -MODE_INTEREST_FUNCTION_CURVATURE_RATIO = (( 0.0, 0.0 )( 0.8, 1.0 )( 1.0, 1.0 )) -MODE_INTEREST_FUNCTION_COMPLEXITY_RATIO = (( 0.0, 0.0 )( 0.8, 1.0 )( 1.0, 1.0 )) -MODE_INTEREST_FUNCTION_INTEN_PERC_RATIO = (( 0.0, 0.0 )( 0.8, 1.0 )( 1.0, 1.0 )) - -MODE_TOTAL_INTEREST_THRESH = 0.65 - -MODE_MASK_GRID = "" -MODE_MASK_GRID_FLAG = NONE -MODE_MASK_POLY = "" -MODE_MASK_POLY_FLAG = NONE - -MODE_MASK_MISSING_FLAG = BOTH -MODE_MATCH_FLAG = MERGE_BOTH - -MODE_MAX_CENTROID_DIST = 600.0/grid_res - -MODE_WEIGHT_CENTROID_DIST = 4.0 -MODE_WEIGHT_BOUNDARY_DIST = 3.0 -MODE_WEIGHT_CONVEX_HULL_DIST = 1.0 -MODE_WEIGHT_ANGLE_DIFF = 1.0 -MODE_WEIGHT_ASPECT_DIFF = 0.0 -MODE_WEIGHT_AREA_RATIO = 4.0 -MODE_WEIGHT_INT_AREA_RATIO = 3.0 -MODE_WEIGHT_CURVATURE_RATIO = 0.0 -MODE_WEIGHT_COMPLEXITY_RATIO = 0.0 -MODE_WEIGHT_INTEN_PERC_RATIO = 0.0 -MODE_WEIGHT_INTEN_PERC_VALUE = 50 - -MODE_NC_PAIRS_FLAG_LATLON = TRUE -MODE_NC_PAIRS_FLAG_RAW = TRUE -MODE_NC_PAIRS_FLAG_OBJECT_RAW = TRUE -MODE_NC_PAIRS_FLAG_OBJECT_ID = TRUE -MODE_NC_PAIRS_FLAG_CLUSTER_ID = TRUE -MODE_NC_PAIRS_FLAG_POLYLINES = TRUE - -MODE_QUILT = False - -MODE_PS_PLOT_FLAG = TRUE -MODE_CT_STATS_FLAG = TRUE diff --git a/metplus/produtil/README_produtil.md b/produtil/README_produtil.md similarity index 100% rename from metplus/produtil/README_produtil.md rename to produtil/README_produtil.md diff --git a/metplus/produtil/__init__.py b/produtil/__init__.py similarity index 100% rename from metplus/produtil/__init__.py rename to produtil/__init__.py diff --git a/metplus/produtil/batchsystem.py b/produtil/batchsystem.py similarity index 100% rename from metplus/produtil/batchsystem.py rename to produtil/batchsystem.py diff --git a/metplus/produtil/cluster.py b/produtil/cluster.py similarity index 100% rename from metplus/produtil/cluster.py rename to produtil/cluster.py diff --git a/metplus/produtil/config.py b/produtil/config.py similarity index 99% rename from metplus/produtil/config.py rename to produtil/config.py index ffffa40621..f020838a10 100644 --- a/metplus/produtil/config.py +++ b/produtil/config.py @@ -11,18 +11,19 @@ # decides what symbols are imported by "from produtil.config import *" __all__=['from_file','confwalker','ProdConfig','ENVIRONMENT','ProdTask'] -import collections,re,os,logging,threading +import collections,re,string,os,logging,threading import os.path,sys import datetime -import metplus.produtil.fileop as fileop +import produtil.fileop, produtil.datastore +import produtil.numerics, produtil.log import configparser from configparser import ConfigParser from io import StringIO -from metplus.produtil.datastore import Datastore,Task +from produtil.datastore import Datastore,Task -from metplus.produtil.numerics import to_datetime, to_datetime_rel, fcst_hr_min +from produtil.numerics import to_datetime from string import Formatter from configparser import NoOptionError,NoSectionError @@ -378,7 +379,7 @@ def get_value(self,key,args,kwargs): v=ap6.strftime(ANL_P6_KEYS[key]) elif '__ftime' in kwargs and '__atime' in kwargs and \ key in TIME_DIFF_KEYS: - (ihours,iminutes)=fcst_hr_min( + (ihours,iminutes)=produtil.numerics.fcst_hr_min( kwargs['__ftime'],kwargs['__atime']) if key=='fahr': v=int(ihours) @@ -701,7 +702,7 @@ def from_args(self,args=None,allow_files=True,allow_options=True, elif not os.path.isfile(path): logger.error(path+': conf file is not a regular file.') sys.exit(2) - elif not fileop.isnonempty(path) and verbose: + elif not produtil.fileop.isnonempty(path) and verbose: logger.warning( path+': conf file is empty. Will continue anyway.') if verbose: logger.info('Conf input: '+repr(path)) @@ -822,7 +823,7 @@ def getdatastore(self): with self: if self._datastore is None: dsfile=self.getstr('config','datastore') - self._datastore=Datastore(dsfile, + self._datastore=produtil.datastore.Datastore(dsfile, logger=self.log('datastore')) return self._datastore @@ -1000,7 +1001,7 @@ def makedirs(self,*args): with self: dirs=[self.getstr('dir',arg) for arg in args] for makeme in dirs: - fileop.makedirs(makeme) + produtil.fileop.makedirs(makeme) def keys(self,sec): """!get options in a section @@ -1106,13 +1107,13 @@ def timestrinterp(self,sec,string,ftime=None,atime=None,**kwargs): @param atime the analysis time or None @param kwargs more variables for string expansion""" if atime is not None: - atime=to_datetime(atime) + atime=produtil.numerics.to_datetime(atime) else: atime=self.cycle if ftime is None: ftime=atime else: - ftime=to_datetime_rel(ftime,atime) + ftime=produtil.numerics.to_datetime_rel(ftime,atime) with self: return self._time_formatter.format(string,__section=sec, __key='__string__',__depth=0,__conf=self._conf,ENV=ENVIRONMENT, @@ -1342,7 +1343,7 @@ def getbool(self,sec,opt,default=None,badtypeok=False,morevars=None,taskvars=Non ######################################################################## -class ProdTask(Task): +class ProdTask(produtil.datastore.Task): """!A subclass of produtil.datastore.Task that provides a variety of convenience functions related to unix conf files and logging.""" diff --git a/metplus/produtil/datastore.py b/produtil/datastore.py similarity index 98% rename from metplus/produtil/datastore.py rename to produtil/datastore.py index 42268fe3f4..cb2a665956 100644 --- a/metplus/produtil/datastore.py +++ b/produtil/datastore.py @@ -6,10 +6,9 @@ Datum, which is the base class of anything that can be stored in the Datastore.""" -import sqlite3, threading, collections, re, contextlib, datetime, logging, os, time -import metplus.produtil.fileop as fileop -from metplus.produtil.locking import LockFile -from metplus.produtil.log import jlogger +import sqlite3, threading, collections, re, contextlib, time, random,\ + traceback, datetime, logging, os, time +import produtil.fileop, produtil.locking, produtil.sigsafety, produtil.log ##@var __all__ # Symbols exported by "from produtil.datastore import *" @@ -176,7 +175,7 @@ def __init__(self,filename,logger=None,locking=True): lockfile=filename+'.lock' if logger is not None: logger.debug('Lockfile is %s for database %s'%(lockfile,filename)) - self._file_lock=LockFile( + self._file_lock=produtil.locking.LockFile( lockfile,logger=logger,max_tries=300,sleep_time=0.1,first_warn=50) self._transtack=collections.defaultdict(list) with self.transaction(): @@ -885,7 +884,7 @@ def undeliver(self,delete=True,logger=None): @param logger a logging.Logger for log messages""" loc=self.location if loc and delete: - fileop.remove_file(filename=loc,logger=logger,info=True) + produtil.fileop.remove_file(filename=loc,logger=logger,info=True) self.available=False def deliver(self,location=None,frominfo=None,keep=True,logger=None, copier=None): @@ -918,7 +917,7 @@ def deliver(self,location=None,frominfo=None,keep=True,logger=None, raise UnknownLocation( '%s: no location known when delivering product. Specify a ' 'location to deliver().'%(self.did)) - fileop.deliver_file(frominfo,loc,keep=keep,logger=logger, + produtil.fileop.deliver_file(frominfo,loc,keep=keep,logger=logger, copier=copier) if setloc: self.set_loc_avail(loc,True) @@ -967,7 +966,7 @@ def check(self,frominfo=None,minsize=None,minage=None,logger=None): minsize=int(self.get('minsize',0)) if minage is None: minage=int(self.get('minage',20)) - if not fileop.check_file(loc,min_size=minsize, + if not produtil.fileop.check_file(loc,min_size=minsize, min_mtime_age=minage): if self.available: self.available=False @@ -1094,7 +1093,7 @@ def jlogfile(self): intended to receive only major errors, and per-job start and completion information. This is equivalent to simply accessing produtil.log.jlogger.""" - return jlogger + return produtil.log.jlogger def postmsg(self,message,*args,**kwargs): """!same as produtil.log.jlogger.info() @@ -1104,7 +1103,7 @@ def postmsg(self,message,*args,**kwargs): @param message the message @param args positional arguments for string replacement @param kwargs keyword arguments for string replacement.""" - jlogger.info(message,*args,**kwargs) + produtil.log.jlogger.info(message,*args,**kwargs) def setstate(self,val): """!Sets the state of this job. diff --git a/metplus/produtil/dbnalert.py b/produtil/dbnalert.py similarity index 98% rename from metplus/produtil/dbnalert.py rename to produtil/dbnalert.py index 62f9830a2a..de8343146e 100644 --- a/metplus/produtil/dbnalert.py +++ b/produtil/dbnalert.py @@ -6,9 +6,10 @@ __all__=["DBNAlert"] import logging, os +import produtil.run -from metplus.produtil.prog import Runner -from metplus.produtil.run import batchexe, alias, run +from produtil.prog import Runner +from produtil.run import checkrun, batchexe, alias, run # Globals: diff --git a/metplus/produtil/externals b/produtil/externals similarity index 100% rename from metplus/produtil/externals rename to produtil/externals diff --git a/metplus/produtil/fileop.py b/produtil/fileop.py similarity index 99% rename from metplus/produtil/fileop.py rename to produtil/fileop.py index 6853f25a95..348daa6006 100644 --- a/metplus/produtil/fileop.py +++ b/produtil/fileop.py @@ -18,7 +18,7 @@ 'netcdfver','touch'] import os,tempfile,filecmp,stat,shutil,errno,random,time,fcntl,math,logging -import metplus.produtil.cluster as cluster +import produtil.cluster, produtil.pipeline module_logger=logging.getLogger('produtil.fileop') @@ -403,9 +403,9 @@ def deliver_file(infile,outfile,keep=True,verify=False,blocksize=1048576, the temp_file_object is an object that can be used to write to the file. The copier should NOT close the temp_file_object. """ if preserve_group is None: - preserve_group = not cluster.group_quotas() + preserve_group = not produtil.cluster.group_quotas() if copy_acl is None: - copy_acl = cluster.use_acl_for_rstdata() + copy_acl = produtil.cluster.use_acl_for_rstdata() if copier is not None: # Cannot simply do a "move" if we are using an external # function to copy. diff --git a/metplus/produtil/locking.py b/produtil/locking.py similarity index 97% rename from metplus/produtil/locking.py rename to produtil/locking.py index 0cd4a7e617..1cf90fd277 100644 --- a/metplus/produtil/locking.py +++ b/produtil/locking.py @@ -14,9 +14,9 @@ ... the file is now unlocked ... @endcode""" -import fcntl, errno, os.path -import metplus.produtil.retry as retry -import metplus.produtil.fileop as fileop +import fcntl, time, errno, os.path +import produtil.retry as retry +import produtil.fileop ##@var __all__ # Symbols exported by "from produtil.locking import *" @@ -110,7 +110,7 @@ def acquire_impl(self): 'the process was exiting.') thedir=os.path.dirname(self._filename) if thedir: - fileop.makedirs(thedir) + produtil.fileop.makedirs(thedir) if self._fd is None: self._fd=open(self._filename,'wb') try: diff --git a/metplus/produtil/log.py b/produtil/log.py similarity index 99% rename from metplus/produtil/log.py rename to produtil/log.py index 0d6487f382..329cab79f2 100644 --- a/metplus/produtil/log.py +++ b/produtil/log.py @@ -12,7 +12,7 @@ 'MasterLogHandler','JLogHandler','set_jlogfile' ] import logging, os, sys, traceback, threading -import metplus.produtil.batchsystem as batchsystem +import produtil.batchsystem ##@var logthread # string for log messages to indicate thread number/name @@ -379,7 +379,7 @@ def configureLogging(jlogfile=None, # Configure log formatting: jobstr=os.environ.get('job',None) if jobstr is None: - jobstr=batchsystem.jobname() + jobstr=produtil.batchsystem.jobname() jobstr=str(jobstr).replace('(','_').replace(')','_').replace('%','_') # Format for jlogfile domain logging to jlogfile: jformat=JLogFormatter( diff --git a/metplus/produtil/mpi_impl/__init__.py b/produtil/mpi_impl/__init__.py similarity index 89% rename from metplus/produtil/mpi_impl/__init__.py rename to produtil/mpi_impl/__init__.py index 10923ff839..b986e7d76c 100644 --- a/metplus/produtil/mpi_impl/__init__.py +++ b/produtil/mpi_impl/__init__.py @@ -125,8 +125,7 @@ # __init__.py on how to modify it to achieve these steps. import logging -import metplus.produtil.fileop -import metplus.produtil.prog +import produtil.fileop ##@var __all__ # An empty list that indicates no symbols are exported by "from @@ -221,8 +220,8 @@ def register_implementations(logger=None): # no_implementation=None is used to detect if # register_implementations was called. global no_implementation - from metplus.produtil.mpi_impl.no_mpi import Implementation - no_implementation=Implementation.detect() + import produtil.mpi_impl.no_mpi + no_implementation=produtil.mpi_impl.no_mpi.Implementation.detect() # Now add each implementation. We need to wrap each around a # try...except so that NCEP Central Operations can delete the @@ -231,64 +230,64 @@ def register_implementations(logger=None): try: # If we have srun, and we're in a pack group... - import metplus.produtil.mpi_impl.srun_pack_groups - add_implementation(metplus.produtil.mpi_impl.srun_pack_groups.Implementation) + import produtil.mpi_impl.srun_pack_groups + add_implementation(produtil.mpi_impl.srun_pack_groups.Implementation) except ImportError: pass try: # This must be after the pack group case. # If we have srun and SLURM resources... - import metplus.produtil.mpi_impl.srun - add_implementation(metplus.produtil.mpi_impl.srun.Implementation) + import produtil.mpi_impl.srun + add_implementation(produtil.mpi_impl.srun.Implementation) except ImportError: pass try: - import metplus.produtil.mpi_impl.inside_aprun - add_implementation(metplus.produtil.mpi_impl.inside_aprun.Implementation) + import produtil.mpi_impl.inside_aprun + add_implementation(produtil.mpi_impl.inside_aprun.Implementation) except ImportError: pass try: - import metplus.produtil.mpi_impl.pbs_cray_intel - add_implementation(metplus.produtil.mpi_impl.pbs_cray_intel.Implementation) + import produtil.mpi_impl.pbs_cray_intel + add_implementation(produtil.mpi_impl.pbs_cray_intel.Implementation) except ImportError: pass try: - import metplus.produtil.mpi_impl.lsf_cray_intel - add_implementation(metplus.produtil.mpi_impl.lsf_cray_intel.Implementation) + import produtil.mpi_impl.lsf_cray_intel + add_implementation(produtil.mpi_impl.lsf_cray_intel.Implementation) except ImportError: pass try: - import metplus.produtil.mpi_impl.impi - add_implementation(metplus.produtil.mpi_impl.impi.Implementation) + import produtil.mpi_impl.impi + add_implementation(produtil.mpi_impl.impi.Implementation) except ImportError: pass try: - import metplus.produtil.mpi_impl.mpirun_lsf - add_implementation(metplus.produtil.mpi_impl.mpirun_lsf.Implementation) + import produtil.mpi_impl.mpirun_lsf + add_implementation(produtil.mpi_impl.mpirun_lsf.Implementation) except ImportError: pass try: - import metplus.produtil.mpi_impl.mpiexec_mpt - add_implementation(metplus.produtil.mpi_impl.mpiexec_mpt.Implementation) + import produtil.mpi_impl.mpiexec_mpt + add_implementation(produtil.mpi_impl.mpiexec_mpt.Implementation) except ImportError: pass try: - import metplus.produtil.mpi_impl.mpiexec - add_implementation(metplus.produtil.mpi_impl.mpiexec.Implementation) + import produtil.mpi_impl.mpiexec + add_implementation(produtil.mpi_impl.mpiexec.Implementation) except ImportError: pass try: - import metplus.produtil.mpi_impl.srun_shell - add_implementation(metplus.produtil.mpi_impl.srun_shell.Implementation) + import produtil.mpi_impl.srun_shell + add_implementation(produtil.mpi_impl.srun_shell.Implementation) except ImportError: pass @@ -316,7 +315,7 @@ def get_mpi(mpi_name=NO_NAME,force=False,logger=None,**kwargs): @raise NotImplementedError if the MPI implementation is unknown, or if the implementation is unavailble on this machine, and force=False""" - import metplus.produtil.mpi_impl.no_mpi as no_mpi + if logger is None: logger=logging.getLogger('mpi_impl') @@ -352,8 +351,8 @@ def get_mpi(mpi_name=NO_NAME,force=False,logger=None,**kwargs): result=detect( force=force,logger=logger,**kwargs) except (Exception, - metplus.produtil.fileop.FileOpError, - metplus.produtil.prog.ExitStatusException): + produtil.fileop.FileOpError, + produtil.prog.ExitStatusException): # Ignore exceptions related to an inability to detect the # MPI implementation. We assume the issue has already # been logged, and we move on to the next implementation's diff --git a/metplus/produtil/mpi_impl/mpi_impl_base.py b/produtil/mpi_impl/mpi_impl_base.py similarity index 98% rename from metplus/produtil/mpi_impl/mpi_impl_base.py rename to produtil/mpi_impl/mpi_impl_base.py index 6f1bb4411b..572bfc63db 100644 --- a/metplus/produtil/mpi_impl/mpi_impl_base.py +++ b/produtil/mpi_impl/mpi_impl_base.py @@ -12,8 +12,9 @@ import tempfile,stat,os, logging, io, re -from metplus.produtil.pipeline import Pipeline -from metplus.produtil.prog import shbackslash, Runner, ImmutableRunner +import produtil.prog +import produtil.pipeline +from produtil.prog import shbackslash module_logger=logging.getLogger('produtil.mpi_impl') @@ -138,8 +139,8 @@ def find_mpiserial(self,mpiserial_path,force): def runsync(self,logger=None): """!Runs the "sync" command as an exe().""" if logger is None: logger=self.logger - sync=Runner(['/bin/sync']) - Pipeline(sync,capture=True,logger=logger) + sync=produtil.prog.Runner(['/bin/sync']) + produtil.pipeline.Pipeline(sync,capture=True,logger=logger) def openmp(self,arg,threads): """!Does nothing. This implementation does not support OpenMP. @@ -167,7 +168,7 @@ def make_bigexe(self,exe,**kwargs): @returns an empty list @param exe The executable to run on compute nodes. @param kwargs Ignored.""" - return ImmutableRunner([str(exe)],**kwargs) + return produtil.prog.ImmutableRunner([str(exe)],**kwargs) class CMDFGen(object): diff --git a/metplus/produtil/mpi_impl/no_mpi.py b/produtil/mpi_impl/no_mpi.py similarity index 88% rename from metplus/produtil/mpi_impl/no_mpi.py rename to produtil/mpi_impl/no_mpi.py index bb20f9a2b3..71dfc1bfd9 100644 --- a/metplus/produtil/mpi_impl/no_mpi.py +++ b/produtil/mpi_impl/no_mpi.py @@ -6,8 +6,9 @@ # produtil.run.mpiserial functions, providing the implementation # needed to run when MPI is unavailable. -import logging -from .mpi_impl_base import ImplementationBase +import os, logging +import produtil.prog,produtil.pipeline +from .mpi_impl_base import MPIDisabled,ImplementationBase module_logger=logging.getLogger('lsf_cray_intel') class Implementation(ImplementationBase): diff --git a/metplus/produtil/mpi_impl/srun.py b/produtil/mpi_impl/srun.py similarity index 90% rename from metplus/produtil/mpi_impl/srun.py rename to produtil/mpi_impl/srun.py index d4518ed0fa..c3574ab1ee 100644 --- a/metplus/produtil/mpi_impl/srun.py +++ b/produtil/mpi_impl/srun.py @@ -6,14 +6,12 @@ # commands. import os, logging, re -from metplus.produtil.fileop import find_exe -from metplus.produtil.prog import Runner, ImmutableRunner -import metplus.produtil.mpiprog as mpiprog -import metplus.produtil.pipeline as pipeline +import produtil.fileop,produtil.prog,produtil.mpiprog,produtil.pipeline from .mpi_impl_base import MPIMixed,CMDFGen,ImplementationBase, \ MPIThreadsMixed,MPILocalOptsMixed,MPITooManyRanks -from metplus.produtil.mpiprog import MIXED_VALUES +from produtil.pipeline import NoMoreProcesses +from produtil.mpiprog import MIXED_VALUES class Implementation(ImplementationBase): """Adds SLURM srun support to produtil.run @@ -39,12 +37,12 @@ def detect(srun_path=None,mpiserial_path=None,logger=None,force=False,silent=Fal if force: srun_path='srun' else: - srun_path=find_exe('srun',raise_missing=True) + srun_path=produtil.fileop.find_exe('srun',raise_missing=True) if scontrol_path is None: if force: scontrol_path='scontrol' else: - scontrol_path=find_exe('scontrol',raise_missing=True) + scontrol_path=produtil.fileop.find_exe('scontrol',raise_missing=True) if 'SLURM_NODELIST' not in os.environ and not force: return None return Implementation(srun_path,scontrol_path,mpiserial_path,logger,silent,force) @@ -60,8 +58,8 @@ def __init__(self,srun_path,scontrol_path,mpiserial_path,logger,silent,force): def runsync(self,logger=None): """!Runs the "sync" command as an exe().""" if logger is None: logger=self.logger - sync=Runner(['/bin/sync']) - pipeline.Pipeline(sync,capture=True,logger=logger) + sync=produtil.prog.Runner(['/bin/sync']) + produtil.pipeline.Pipeline(sync,capture=True,logger=logger) def openmp(self,arg,threads): """!Adds OpenMP support to the provided object @@ -88,7 +86,7 @@ def make_bigexe(self,exe,**kwargs): @returns an empty list @param exe The executable to run on compute nodes. @param kwargs Ignored.""" - return ImmutableRunner([str(exe)],**kwargs) + return produtil.prog.ImmutableRunner([str(exe)],**kwargs) def mpirunner(self,arg,allranks=False,**kwargs): """!Turns a produtil.mpiprog.MPIRanksBase tree into a produtil.prog.Runner @@ -106,10 +104,10 @@ def mpirunner(self,arg,allranks=False,**kwargs): def _get_available_nodes(self): available_nodes=list() nodeset=set() - scontrol=Runner([ + scontrol=produtil.prog.Runner([ self.scontrol_path,'show','hostnames', os.environ['SLURM_NODELIST']]) - p=pipeline.Pipeline( + p=produtil.pipeline.Pipeline( scontrol,capture=True,logger=self.logger) nodelist=p.to_string() for line in nodelist.splitlines(): @@ -123,7 +121,7 @@ def _get_available_nodes(self): def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False,**kwargs): """!This is the underlying implementation of mpirunner and should not be called directly.""" - assert(isinstance(arg,mpiprog.MPIRanksBase)) + assert(isinstance(arg,produtil.mpiprog.MPIRanksBase)) (serial,parallel)=arg.check_serial() if serial and parallel: raise MPIMixed('Cannot mix serial and parallel MPI ranks in the ' @@ -145,7 +143,7 @@ def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False, srun_args.append('--distribution=block:block') arglist=[ str(a) for a in arg.to_arglist( pre=srun_args,before=[],between=[])] - return Runner(arglist) + return produtil.prog.Runner(arglist) elif allranks: raise MPIAllRanksError( "When using allranks=True, you must provide an mpi program " @@ -153,9 +151,9 @@ def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False, "all ranks).") elif serial: srun_args.append('--distribution=block:block') - arg=mpiprog.collapse(arg) + arg=produtil.mpiprog.collapse(arg) lines=[str(a) for a in arg.to_arglist(to_shell=True,expand=True)] - return Runner( + return produtil.prog.Runner( [self.srun_path,'--ntasks','%s'%(arg.nranks()),self.mpiserial_path], prerun=CMDFGen('serialcmdf',lines,silent=self.silent,**kwargs)) else: @@ -206,4 +204,4 @@ def mpirunner_impl(self,arg,allranks=False,rewrite_nodefile=True,label_io=False, silent=self.silent,filename_option='--nodelist', next_prerun=prerun,**kwargs) - return Runner(srun_args,prerun=prerun) + return produtil.prog.Runner(srun_args,prerun=prerun) diff --git a/metplus/produtil/mpiprog.py b/produtil/mpiprog.py similarity index 99% rename from metplus/produtil/mpiprog.py rename to produtil/mpiprog.py index ad7609f445..e7adf6e1f4 100644 --- a/metplus/produtil/mpiprog.py +++ b/produtil/mpiprog.py @@ -44,9 +44,12 @@ # Ensure nothing is loaded by "from produtil.mpiprog import *" __all__=[] +import sys + import io import logging -from metplus.produtil.prog import ProgSyntaxError, shbackslash, Runner, ImmutableRunner +import produtil.prog +from produtil.prog import ProgSyntaxError, shbackslash class MPIProgSyntaxError(ProgSyntaxError): """!Base class of syntax errors in MPI program specifications""" @@ -828,7 +831,7 @@ def __init__(self,arg,logger=None): self._localopts=list(arg._localopts) self._turbomode=arg.turbomode self._ranks_per_node=arg.ranks_per_node - elif isinstance(arg,Runner): + elif isinstance(arg,produtil.prog.Runner): if arg.isplainexe(): self._args=[x for x in arg.args()] else: @@ -868,7 +871,7 @@ def delthreads(self): def to_shell(self): """!Return a POSIX sh representation of this MPI rank, if possible.""" - return ' '.join([shbackslash(x) for x in self._args]) + return ' '.join([produtil.prog.shbackslash(x) for x in self._args]) def __getitem__(self,args): """!Adds arguments to this MPI rank's program.""" c=self.copy() @@ -992,8 +995,8 @@ def __init__(self,runner,logger=None): self._ranks_per_node=0 def make_runners_immutable(self): """!Creates a version of self with a produtil.prog.ImmutableRunner child.""" - if not isinstance(self._runner,ImmutableRunner): - return MPISerial(ImmutableRunner(self._runner),self._logger) + if not isinstance(self._runner,produtil.prog.ImmutableRunner): + return MPISerial(produtil.prog.ImmutableRunner(self._runner),self._logger) else: return self def copy(self): diff --git a/metplus/produtil/numerics.py b/produtil/numerics.py similarity index 100% rename from metplus/produtil/numerics.py rename to produtil/numerics.py diff --git a/metplus/produtil/pipeline.py b/produtil/pipeline.py similarity index 100% rename from metplus/produtil/pipeline.py rename to produtil/pipeline.py diff --git a/metplus/produtil/prog.py b/produtil/prog.py similarity index 99% rename from metplus/produtil/prog.py rename to produtil/prog.py index 441cc6bd35..f6f2c6fdfc 100644 --- a/metplus/produtil/prog.py +++ b/produtil/prog.py @@ -37,9 +37,11 @@ directly, except for type checking (ie.: to see if your argument is a Runner before passing it to produtil.run.checkrun)..""" -import re, os +import produtil.sigsafety +import io,select,io,re,time,fcntl,os,logging,signal -from metplus.produtil.pipeline import ERR2OUT +import produtil.mpi_impl +from produtil.pipeline import launch, manage, PIPE, ERR2OUT ERR2OUT_FCT_STR = '.err2out()' diff --git a/metplus/produtil/retry.py b/produtil/retry.py similarity index 100% rename from metplus/produtil/retry.py rename to produtil/retry.py diff --git a/metplus/produtil/run.py b/produtil/run.py similarity index 97% rename from metplus/produtil/run.py rename to produtil/run.py index bdd305ea29..a7e469b890 100644 --- a/metplus/produtil/run.py +++ b/produtil/run.py @@ -155,14 +155,15 @@ """ import time, logging -import metplus.produtil.mpi_impl as mpi_impl -import metplus.produtil.prog as prog -import metplus.produtil.mpiprog as mpiprog -import metplus.produtil.pipeline as pipeline +import produtil.mpi_impl +import produtil.sigsafety +import produtil.prog as prog +import produtil.mpiprog as mpiprog +import produtil.pipeline as pipeline # These two were moved to produtil.prog to avoid a cyclic import. # They still need to be available from produtil.run: -from metplus.produtil.prog import InvalidRunArgument,ExitStatusException +from produtil.prog import InvalidRunArgument,ExitStatusException ##@var __all__ # List of symbols exported by "from produtil.run import *" @@ -179,7 +180,7 @@ # The cached return value from detect_mpi() _detected_mpi=None -def make_mpi(mpi_name=mpi_impl.NO_NAME,**kwargs): +def make_mpi(mpi_name=produtil.mpi_impl.NO_NAME,**kwargs): """!Creates an MPI implementation object for the specified MPI implementation. @@ -207,7 +208,7 @@ def make_mpi(mpi_name=mpi_impl.NO_NAME,**kwargs): # situation does not need any initialization of the produtil.run # module: if mpi_name is None: - return mpi_impl.get_mpi(None,**kwargs) + return produtil.mpi_impl.get_mpi(None,**kwargs) # For anything other than "None," we have to ensure the # produtil.run is initialized to something before running @@ -215,7 +216,7 @@ def make_mpi(mpi_name=mpi_impl.NO_NAME,**kwargs): detect_mpi() # Next, return the requested implementation: - return mpi_impl.get_mpi(mpi_name,**kwargs) + return produtil.mpi_impl.get_mpi(mpi_name,**kwargs) def detect_mpi(): """!Called by functions inside produtil.run to automatically @@ -252,10 +253,10 @@ def detect_mpi(): # First, set the implementation to None so that the mpi_impl # subclasses can use produtil.run. This will only allow serial, # non-OpenMP programs: - _detected_mpi=mpi_impl.get_mpi(None) + _detected_mpi=produtil.mpi_impl.get_mpi(None) - # Next, ask mpi_impl to detect the MPI implementation: - detected=mpi_impl.get_mpi() + # Next, ask produtil.mpi_impl to detect the MPI implementation: + detected=produtil.mpi_impl.get_mpi() # If detection succeeds, override the selected MPI implementation: if detected: diff --git a/metplus/produtil/setup.py b/produtil/setup.py similarity index 85% rename from metplus/produtil/setup.py rename to produtil/setup.py index b97bdc8840..9a58e6a8b0 100644 --- a/metplus/produtil/setup.py +++ b/produtil/setup.py @@ -7,12 +7,9 @@ # Lists symbols exported by "from produtil.setup import *" __all__=['setup'] -import threading -from metplus.produtil.sigsafety import install_handlers -from metplus.produtil.log import configureLogging -from metplus.produtil.dbnalert import init_module -from metplus.produtil.cluster import set_cluster, where -from metplus.produtil.batchsystem import set_default_name +import logging, threading +import produtil.sigsafety, produtil.log, produtil.dbnalert, produtil.cluster +import produtil.batchsystem def setup(ignore_hup=False,dbnalert_logger=None,jobname=None,cluster=None, send_dbn=None,thread_logger=False,thread_stack=2**24,**kwargs): @@ -64,19 +61,19 @@ def setup(ignore_hup=False,dbnalert_logger=None,jobname=None,cluster=None, # Set the default jobname. This is usually used for manually-run # scripts to ensure they have a "jobname" in the logging system: if jobname is not None: - set_default_name(jobname) + produtil.batchsystem.set_default_name(jobname) # Configure logging next so that the install_handlers will be able # to log. - configureLogging(thread_logger=thread_logger,**kwargs) + produtil.log.configureLogging(thread_logger=thread_logger,**kwargs) # Install signal handlers, and let the caller configure SIGHUP settings: - install_handlers(ignore_hup=ignore_hup) + produtil.sigsafety.install_handlers(ignore_hup=ignore_hup) # Set up dbnalert: - init_module(logger=dbnalert_logger,jobname=jobname, + produtil.dbnalert.init_module(logger=dbnalert_logger,jobname=jobname, send_dbn=send_dbn) # Set up cluster: if cluster is not None: - set_cluster(cluster) + produtil.cluster.set_cluster(cluster) else: - where() # guess cluster + produtil.cluster.where() # guess cluster diff --git a/metplus/produtil/sigsafety.py b/produtil/sigsafety.py similarity index 97% rename from metplus/produtil/sigsafety.py rename to produtil/sigsafety.py index 23699d766d..301e488031 100644 --- a/metplus/produtil/sigsafety.py +++ b/produtil/sigsafety.py @@ -40,8 +40,7 @@ One can call install_handlers directly, though it is recommended to call produtil.setup.setup instead.""" -from metplus.produtil.locking import disable_locking -from metplus.produtil.pipeline import kill_all +import produtil.locking, produtil.pipeline import signal ##@var defaultsigs @@ -136,7 +135,7 @@ def hup_handler(signum,frame): caught_signal=signum caught_class=HangupSignal - disable_locking() + produtil.locking.disable_locking() raise HangupSignal(signum) def term_handler(signum,frame): @@ -146,8 +145,8 @@ def term_handler(signum,frame): caught_signal=signum caught_class=FatalSignal - disable_locking() # forbid file locks - kill_all() # kill all subprocesses + produtil.locking.disable_locking() # forbid file locks + produtil.pipeline.kill_all() # kill all subprocesses uninstall_handlers() raise FatalSignal(signum) diff --git a/pyproject.toml b/pyproject.toml index 30e5629b73..b5a3c468d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,47 +1,6 @@ [build-system] requires = [ - "setuptools", - "wheel", - "setuptools-git-versioning>=2.0,<3", + "setuptools>=42", + "wheel" ] -build-backend = "setuptools.build_meta" - -[project] -name = "metplus" -dynamic = ["version", "dependencies"] -description = "METplus Wrappers" -authors = [ - {name = "METplus"}, -] -requires-python = ">=3.10.4" -readme = "README.md" -license = {text = "MIT"} -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", -] - -[project.urls] -Homepage = "https://github.com/dtcenter/METplus" - -[project.scripts] -run_metplus = "metplus.scripts.run_metplus:cli_main" - -[tool.setuptools.packages] -find = {include = ["metplus*"], exclude = ["parm"]} - -[tool.setuptools.package-data] -metplus = [ - "VERSION", "RELEASE_DATE", "PYTHON_VERSION", "PYTHON_VERSION_MIN", "parm/**" -] - -[tool.setuptools.exclude-package-data] -metplus = ["README", "__pycache__", "*~"] - -[tool.setuptools.dynamic] -dependencies = {file = ["requirements.txt"]} - -[tool.setuptools-git-versioning] -enabled = true -version_file = "metplus/VERSION" +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2c05e903ae..169d9d94cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -certifi>=2024.7.4 -python-dateutil>=2.8.2 -six>=1.16.0 +certifi==2024.7.4 +python-dateutil==2.8.2 +six==1.16.0 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..2114a77efe --- /dev/null +++ b/setup.py @@ -0,0 +1,54 @@ +from setuptools import setup, find_packages +from distutils.util import convert_path +import os + +with open("README.md", "r") as fh: + long_description = fh.read() + +with open("metplus/VERSION", "r") as fh: + version = fh.read().strip() + +with open("metplus/PYTHON_VERSION_MIN", "r") as fh: + python_version_req = fh.read().strip() + +# get list of additional files needed to add to package +data_files = [] +# add version and release date files +data_files.append(('metplus', + ['metplus/VERSION', + 'metplus/RELEASE_DATE', + 'metplus/PYTHON_VERSION', + 'metplus/PYTHON_VERSION_MIN', + ])) + +for root, _, files in os.walk('parm'): + parm_files = [] + for filename in files: + filepath = os.path.join(root, filename) + # skip README, tilda, and pycache files + if ('__pycache__' in filepath or + 'README' in filepath or + filepath.endswith('~')): + continue + parm_files.append(filepath) + if parm_files: + data_files.append((root, parm_files)) + +setup( + name="metplus", + version=version, + author="METplus", + description="METplus Wrappers", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/dtcenter/METplus", + packages=find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + python_requires=f'>={python_version_req}', + data_files=data_files, + zip_safe=False, +) diff --git a/ush/run_metplus.py b/ush/run_metplus.py index 6f329867ac..eb374b4427 100755 --- a/ush/run_metplus.py +++ b/ush/run_metplus.py @@ -25,7 +25,7 @@ # add metplus directory to path so the wrappers and utilities can be found sys.path.insert(0, abspath(join(dirname(realpath(__file__)), pardir))) -from metplus.produtil.setup import setup as produtil_setup +import produtil.setup from metplus.util import pre_run_setup, run_metplus, post_run_cleanup from metplus import __version__ as metplus_version @@ -110,15 +110,12 @@ def get_config_inputs_from_command_line(): return config_inputs -def cli_main(): +if __name__ == "__main__": try: - produtil_setup(send_dbn=False, jobname='run-METplus') + produtil.setup.setup(send_dbn=False, jobname='run-METplus') if not main(): sys.exit(1) except Exception as exc: print(traceback.format_exc()) print('ERROR: run_metplus failed: %s' % exc) sys.exit(2) - -if __name__ == "__main__": - cli_main()