diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 32c5c26f12..ac5c9e2399 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -123,6 +123,7 @@ jobs: - "s2s:1-3" - "space_weather:0-1" - "tc_and_extra_tc:0-2" + - "met_tool_wrapper:54:NEW" steps: - uses: actions/download-artifact@v2 with: @@ -138,6 +139,12 @@ jobs: echo run_this_case=$run_this_case >> $GITHUB_ENV echo Ends with NEW: ${{ endsWith(matrix.categories, ':NEW') }} echo Run all: ${{ env.run_all_use_cases }} + - name: Create directories for database + run: | + mkdir -p $RUNNER_WORKSPACE/mysql + mkdir -p $RUNNER_WORKSPACE/output/metviewer + chmod a+w $RUNNER_WORKSPACE/mysql + chmod a+w $RUNNER_WORKSPACE/output/metviewer - uses: actions/checkout@v2 if: ${{ env.run_this_case == 'true' }} - uses: ./ci/actions/run_tests diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index f34d3040e8..931b2ee0ec 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -46,6 +46,16 @@ if [ "$INPUT_CATEGORIES" == "pytests" ]; then exit $? fi +# get METviewer if used in any use cases +all_requirements=`./ci/jobs/get_requirements.py ${CATEGORIES} ${SUBSETLIST}` +echo All requirements: $all_requirements +NETWORK_ARG="" +if [[ "$all_requirements" =~ .*"metviewer".* ]]; then + echo "Setting up METviewer" + ${GITHUB_WORKSPACE}/ci/jobs/python_requirements/get_metviewer.sh + NETWORK_ARG=--network="container:mysql_mv" +fi + # install Pillow library needed for diff testing # this will be replaced with better image diffing package used by METplotpy pip_command="pip3 install Pillow; yum -y install poppler-utils; pip3 install pdf2image" @@ -86,6 +96,9 @@ fi echo VOLUMES_FROM: $VOLUMES_FROM +echo docker ps: +docker ps -a + echo "Run Docker container: $DOCKERHUBTAG" -echo docker run -e GITHUB_WORKSPACE -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" -docker run -e GITHUB_WORKSPACE -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" +echo docker run -e GITHUB_WORKSPACE $NETWORK_ARG -v $RUNNER_WORKSPACE/output/mysql:/var/lib/mysql -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" +docker run -e GITHUB_WORKSPACE $NETWORK_ARG -v $RUNNER_WORKSPACE/output/mysql:/var/lib/mysql -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" diff --git a/ci/jobs/get_requirements.py b/ci/jobs/get_requirements.py new file mode 100755 index 0000000000..4b13b85603 --- /dev/null +++ b/ci/jobs/get_requirements.py @@ -0,0 +1,37 @@ +#! /usr/bin/env python3 + +# Used in GitHub Actions (in ci/actions/run_tests/entrypoint.sh) +# to obtain list of requirements from use case group + +import os +import sys + +import get_use_case_commands + +# add internal_tests/use_cases directory to path so the test suite can be found +USE_CASES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) +sys.path.insert(0, USE_CASES_DIR) + +from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite + +def main(): + all_requirements = set() + categories, subset_list, compare = ( + get_use_case_commands.handle_command_line_args() + ) + + test_suite = METplusUseCaseSuite() + test_suite.add_use_case_groups(categories, subset_list) + + for group_name, use_cases_by_req in test_suite.category_groups.items(): + for use_case_by_req in use_cases_by_req: + for requirement in use_case_by_req.requirements: + all_requirements.add(requirement) + + return list(all_requirements) + +if __name__ == '__main__': + all_requirements = main() + print(','.join(all_requirements)) diff --git a/ci/jobs/get_use_case_commands.py b/ci/jobs/get_use_case_commands.py index 597e8e80a2..761d88a060 100755 --- a/ci/jobs/get_use_case_commands.py +++ b/ci/jobs/get_use_case_commands.py @@ -19,6 +19,11 @@ def handle_requirements(requirements, work_dir): requirement_args = [] for requirement in requirements: + # don't obtain METviewer here because it has to be set up outside of + # docker container that runs the use cases + if requirement.lower() == 'metviewer': + continue + # check if get_{requirement} script exists and use it if it does script_path = os.path.join(work_dir, 'ci', diff --git a/ci/jobs/python_requirements/get_metdatadb.sh b/ci/jobs/python_requirements/get_metdatadb.sh new file mode 100755 index 0000000000..6aad83ee26 --- /dev/null +++ b/ci/jobs/python_requirements/get_metdatadb.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +pip3 install lxml +pip3 install PyMySQL + +basedir=$(dirname "$0") +work_dir=$basedir/../../.. + +# run manage externals to obtain METdatadb +${work_dir}/manage_externals/checkout_externals -e ${work_dir}/ci/parm/Externals_metdatadb.cfg diff --git a/ci/jobs/python_requirements/get_metviewer.sh b/ci/jobs/python_requirements/get_metviewer.sh new file mode 100755 index 0000000000..2dc87018e9 --- /dev/null +++ b/ci/jobs/python_requirements/get_metviewer.sh @@ -0,0 +1,33 @@ +#! /bin/bash + +# set environment variables needed by METviewer docker-compose.yml +export METVIEWER_DATA=$RUNNER_WORKSPACE +export MYSQL_DIR=$RUNNER_WORKSPACE/mysql +export METVIEWER_DIR=$RUNNER_WORKSPACE/output/metviewer +export METVIEWER_DOCKER_IMAGE=dtcenter/metviewer + +# install docker-compose +apk add docker-compose + +# download docker-compose.yml file from METviewer develop branch +wget https://raw.githubusercontent.com/dtcenter/METviewer/develop/docker/docker-compose.yml + +# Run docker-compose to create the containers +docker-compose up -d + +# sleep for a few seconds to ensure database has fully started +sleep 20 + +# print list of currently running containers to +# verify mysql and metviewer are running +docker ps -a + +# commands to run inside METviewer container +cmd="mysql -hmysql_mv -uroot -pmvuser -e\"create database mv_metplus_test;\";" +cmd+="mysql -hmysql_mv -uroot -pmvuser mv_metplus_test < /METviewer/sql/mv_mysql.sql" +cmd+=";mysql -hmysql_mv -uroot -pmvuser -e\"show databases;\"" + +# execute commands inside metviewer container to create database +echo Executing commands inside metviewer_1 container to create database +echo docker exec metviewer_1 /bin/bash -c "$cmd" +docker exec metviewer_1 /bin/bash -c "$cmd" diff --git a/ci/jobs/set_job_controls.sh b/ci/jobs/set_job_controls.sh index cea137273a..ab64c382b3 100755 --- a/ci/jobs/set_job_controls.sh +++ b/ci/jobs/set_job_controls.sh @@ -48,6 +48,10 @@ else run_use_cases=false fi + if grep -q "ci-skip-unit-tests" <<< "$commit_msg"; then + run_unit_tests=false + fi + if grep -q "ci-only-docs" <<< "$commit_msg"; then run_docs=true run_get_image=false diff --git a/ci/parm/Externals_metdatadb.cfg b/ci/parm/Externals_metdatadb.cfg new file mode 100644 index 0000000000..888d20303c --- /dev/null +++ b/ci/parm/Externals_metdatadb.cfg @@ -0,0 +1,9 @@ +[METdatadb] +local_path = ../METdatadb +protocol = git +required = True +repo_url = https://github.com/dtcenter/METdatadb +branch = develop + +[externals_description] +schema_version = 1.0.0 diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 755140d2d9..30ee719644 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -5985,6 +5985,129 @@ METplus Configuration Glossary | *Used by:* SeriesAnalysis + MET_DB_LOAD_RUNTIME_FREQ + Frequency to run Grid-Diag. See :ref:`Runtime_Freq` for more information. + + | *Used by:* GridDiag + + MET_DATA_DB_DIR + Set this the location of the dtcenter/METdatadb repository. + + | *Used by:* METdbLoad + + MET_DB_LOAD_XML_FILE + Template XML file that is used to load data into METviewer using the + met_db_load.py script. Values from the METplus configuration file are + substituted into this file before passing it to the script. The default + value can be used to run unless the template doesn't fit the needs of the + use case. + + | *Used by:* METdbLoad + + MET_DB_LOAD_REMOVE_TMP_XML + If set to False, then the temporary XML file with substituted values will + not be removed after the use case finishes. This is used for debugging + purposes only. The temporary XML file may contain sensitive information + like database credentials so it is recommended to remove the temporary + file after each run. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_HOST + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_DATABASE + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_USER + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_PASSWORD + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_VERBOSE + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_INSERT_SIZE + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_MODE_HEADER_DB_CHECK + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_DROP_INDEXES + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_APPLY_INDEXES + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_GROUP + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_LOAD_STAT + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_LOAD_MODE + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_LOAD_MTD + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_MV_LOAD_MPR + Set the value in the + METdbLoad XML template file. + + | *Used by:* METdbLoad + + MET_DB_LOAD_INPUT_TEMPLATE + Path to a directory containing .stat or .tcst file that will be loaded + into METviewer. This can be a single directory or a list of directories. + The paths can include filename template tags that correspond to each + run time. The wrapper will traverse through each sub directory under the + directories listed here and add any directory that contains any files that + end with .stat or .tcst to the XML file that is passed into the + met_db_load.py script. + + | *Used by:* METdbLoad + CYCLONE_PLOTTER_ADD_WATERMARK If set to True, add a watermark with the current time to the image generated by CyclonePlotter. diff --git a/docs/Users_Guide/quicksearch.rst b/docs/Users_Guide/quicksearch.rst index b73a0d01e1..78bb07aa08 100644 --- a/docs/Users_Guide/quicksearch.rst +++ b/docs/Users_Guide/quicksearch.rst @@ -65,6 +65,7 @@ Use Cases by METplus Feature: | `Looping by Month or Year `_ | `List Expansion (using begin_end_incr syntax) `_ | `Masking for Regions of Interest `_ +| `METdbLoad `_ | `MET_PYTHON_EXE Environment Variable `_ | `Multiple Conf File Use `_ | `Observation Time Summary `_ diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index cafed478f6..d1e03eb611 100755 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -1546,6 +1546,218 @@ configuration file: | :term:`EVENT_EQUALIZATION` | +.. _met_db_load_wrapper: + +METdbLoad +--------- + +Description +~~~~~~~~~~~ + +Used to call the met_db_load.py script from dtcenter/METdatadb to load MET +output into a METviewer database. + +Configuration +~~~~~~~~~~~~~ + +| :term:`MET_DB_LOAD_RUNTIME_FREQ` +| :term:`MET_DATA_DB_DIR` +| :term:`MET_DB_LOAD_XML_FILE` +| :term:`MET_DB_LOAD_REMOVE_TMP_XML` +| :term:`MET_DB_LOAD_MV_HOST` +| :term:`MET_DB_LOAD_MV_DATABASE` +| :term:`MET_DB_LOAD_MV_USER` +| :term:`MET_DB_LOAD_MV_PASSWORD` +| :term:`MET_DB_LOAD_MV_VERBOSE` +| :term:`MET_DB_LOAD_MV_INSERT_SIZE` +| :term:`MET_DB_LOAD_MV_MODE_HEADER_DB_CHECK` +| :term:`MET_DB_LOAD_MV_DROP_INDEXES` +| :term:`MET_DB_LOAD_MV_APPLY_INDEXES` +| :term:`MET_DB_LOAD_MV_GROUP` +| :term:`MET_DB_LOAD_MV_LOAD_STAT` +| :term:`MET_DB_LOAD_MV_LOAD_MODE` +| :term:`MET_DB_LOAD_MV_LOAD_MTD` +| :term:`MET_DB_LOAD_MV_LOAD_MPR` +| :term:`MET_DB_LOAD_INPUT_TEMPLATE` + +.. _met_db_load-xml-conf: + +XML Configuration +~~~~~~~~~~~~~~~~~ + +Below is the XML template configuration file used for this wrapper. The wrapper +substitutes values from the METplus configuration file into this configuration +file. While it may appear that environment variables are used in the XML +template file, they are not actually environment variables. The wrapper +searches for these strings and substitutes the values as appropriate. + +.. literalinclude:: ../../parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoadConfig.xml + +**${METPLUS_MV_HOST}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_HOST` + - + +**${METPLUS_MV_DATABASE}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_DATABASE` + - + +**${METPLUS_MV_USER}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_USER` + - + +**${METPLUS_MV_PASSWORD}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_PASSWORD` + - + +**${METPLUS_MV_VERBOSE}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_VERBOSE` + - + +**${METPLUS_MV_INSERT_SIZE}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_INSERT_SIZE` + - + +**${METPLUS_MV_MODE_HEADER_DB_CHECK}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_MODE_HEADER_DB_CHECK` + - + +**${METPLUS_MV_DROP_INDEXES}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_DROP_INDEXES` + - + +**${METPLUS_MV_APPLY_INDEXES}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_APPLY_INDEXES` + - + +**${METPLUS_MV_GROUP}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_GROUP` + - + +**${METPLUS_MV_LOAD_STAT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_LOAD_STAT` + - + +**${METPLUS_MV_LOAD_MODE}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_LOAD_MODE` + - + +**${METPLUS_MV_LOAD_MTD}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_LOAD_MTD` + - + +**${METPLUS_MV_LOAD_MPR}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_MV_LOAD_MPR` + - + +**${METPLUS_INPUT_PATHS}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - XML Config File + * - :term:`MET_DB_LOAD_INPUT_TEMPLATE` + - + .. _mode_wrapper: MODE diff --git a/docs/use_cases/met_tool_wrapper/METdbLoad/METdbLoad.py b/docs/use_cases/met_tool_wrapper/METdbLoad/METdbLoad.py new file mode 100644 index 0000000000..17e885d43e --- /dev/null +++ b/docs/use_cases/met_tool_wrapper/METdbLoad/METdbLoad.py @@ -0,0 +1,116 @@ +""" +METdbLoad: Basic Use Case +========================= + +met_tool_wrapper/METdbLoad/METdbLoad.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# +# Load MET data into a database using the met_db_load.py script +# found in dtcenter/METdatadb + +############################################################################## +# Datasets +# -------- +# +# | **Input:** Various MET .stat and .tcst files +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to see the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. +# | + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes the METplus METdbLoad wrapper to search for +# files ending with .stat or .tcst, substitute values into an XML load +# configuration file, and call met_db_load.py to load MET data into a +# database. + +############################################################################## +# METplus Workflow +# ---------------- +# +# METdbLoad is the only tool called in this example. It does not loop over +# multiple run times: +# + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line +# with the -c option, i.e. -c parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoad.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoad.conf + +############################################################################## +# XML Configuration +# ----------------- +# +# METplus substitutes values in the template XML configuration file based on +# user settings in the METplus configuration file. While the XML template may +# appear to reference environment variables, this is not actually the case. +# These strings are used as a reference for the wrapper to substitute values. +# +# .. note:: +# See the :ref:`METdbLoad XML Configuration` +# section of the User's Guide for more information on the values +# substituted in the file below: +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoadConfig.xml + +############################################################################## +# Running METplus +# --------------- +# +# This use case can be run two ways: +# +# 1) Passing in METdbLoad.conf followed by a user-specific system configuration file:: +# +# run_metplus.py -c /path/to/METplus/parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoad.conf -c /path/to/user_system.conf +# +# 2) Modifying the configurations in parm/metplus_config and then passing in METdbLoad.conf:: +# +# run_metplus.py -c /path/to/METplus/parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoad.conf +# +# The former method is recommended. Whether you add them to a user-specific configuration file or modify the metplus_config files, the following variables must be set correctly: +# +# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs). This is not required to run METplus, but it is required to run the examples in parm/use_cases +# * **OUTPUT_BASE** - Path to directory where METplus output will be written. This must be in a location where you have write permissions +# * **MET_INSTALL_DIR** - Path to location where MET is installed locally +# +# Example User Configuration File:: +# +# [dir] +# INPUT_BASE = /path/to/sample/input/data +# OUTPUT_BASE = /path/to/output/dir +# MET_INSTALL_DIR = /path/to/met-X.Y +# +# **NOTE:** All of these items must be found under the [dir] section. +# + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# + + +############################################################################## +# Keywords +# -------- +# +# sphinx_gallery_thumbnail_path = '_static/met_tool_wrapper-METdbLoad.png' +# +# .. note:: `METdbLoadUseCase `_ diff --git a/docs/use_cases/met_tool_wrapper/METdbLoad/README.rst b/docs/use_cases/met_tool_wrapper/METdbLoad/README.rst new file mode 100644 index 0000000000..97d799df69 --- /dev/null +++ b/docs/use_cases/met_tool_wrapper/METdbLoad/README.rst @@ -0,0 +1,2 @@ +METdbLoad +--------- diff --git a/internal_tests/use_cases/all_use_cases.txt b/internal_tests/use_cases/all_use_cases.txt index 30e66e9e9f..435c11acf5 100644 --- a/internal_tests/use_cases/all_use_cases.txt +++ b/internal_tests/use_cases/all_use_cases.txt @@ -53,6 +53,7 @@ Category: met_tool_wrapper 51::met_tool_wrapper/UserScript/UserScript_run_once_per_init.conf 52::met_tool_wrapper/UserScript/UserScript_run_once_per_lead.conf 53::met_tool_wrapper/UserScript/UserScript_run_once_per_valid.conf +54::METdbLoad::met_tool_wrapper/METdbLoad/METdbLoad.conf::metdatadb,metviewer Category: air_quality_and_comp diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index 67ad9dbeef..4c15036c83 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -15,6 +15,7 @@ 'griddiag': 'GridDiag', 'gridstat': 'GridStat', 'makeplots': 'MakePlots', + 'metdbload': 'METDbLoad', 'mode': 'MODE', 'mtd': 'MTD', 'modetimedomain': 'MTD', diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index 92d0b7ea66..997fd83e94 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -2732,7 +2732,7 @@ def expand_int_string_to_list(int_string): if hasPlus: subset_list.append('+') - print(f"{int_string} converted to {subset_list}") + return subset_list def subset_list(full_list, subset_definition): @@ -2831,6 +2831,13 @@ def netcdf_has_var(file_path, name, level): except (AttributeError, OSError, ImportError): return False +def generate_tmp_filename(): + import random + import string + random_string = ''.join(random.choice(string.ascii_letters) + for i in range(10)) + return f"metplus_tmp_{random_string}" + def format_level(level): """! Format level string to prevent NetCDF level values from creating filenames and field names with bad characters. Replaces '*' with 'all' diff --git a/metplus/wrappers/met_db_load_wrapper.py b/metplus/wrappers/met_db_load_wrapper.py new file mode 100755 index 0000000000..74e0c909e8 --- /dev/null +++ b/metplus/wrappers/met_db_load_wrapper.py @@ -0,0 +1,256 @@ +""" +Program Name: met_db_load_wrapper.py +Contact(s): George McCabe +Abstract: Parent class for wrappers that process groups of times +History Log: Initial version +Usage: +Parameters: None +Input Files: +Output Files: +Condition codes: 0 for success, 1 for failure +""" + +import os +from datetime import datetime + +from ..util import met_util as util +from ..util import time_util +from . import RuntimeFreqWrapper +from ..util import do_string_sub, getlist + +'''!@namespace METDbLoadWrapper +@brief Parent class for wrappers that run over a grouping of times +@endcode +''' + +class METDbLoadWrapper(RuntimeFreqWrapper): + """! Config variable names - All names are prepended with MET_DB_LOAD_MV_ + and all c_dict values are prepended with MV_. + The name is the key and string specifying the type is the value. + """ + CONFIG_NAMES = {'HOST': 'string', + 'DATABASE': 'string', + 'USER': 'string', + 'PASSWORD': 'string', + 'VERBOSE': 'bool', + 'INSERT_SIZE': 'int', + 'MODE_HEADER_DB_CHECK': 'bool', + 'DROP_INDEXES': 'bool', + 'APPLY_INDEXES': 'bool', + 'GROUP': 'string', + 'LOAD_STAT': 'bool', + 'LOAD_MODE': 'bool', + 'LOAD_MTD': 'bool', + 'LOAD_MPR': 'bool', + } + + def __init__(self, config, instance=None, config_overrides={}): + met_data_db_dir = config.getdir('MET_DATA_DB_DIR') + self.app_path = os.path.join(met_data_db_dir, + 'METdbLoad', + 'ush', + 'met_db_load') + self.app_name = os.path.basename(self.app_path) + super().__init__(config, + instance=instance, + config_overrides=config_overrides) + + def create_c_dict(self): + c_dict = super().create_c_dict() + + c_dict['XML_TEMPLATE'] = ( + self.config.getraw('config', + 'MET_DB_LOAD_XML_FILE') + ) + if not c_dict['XML_TEMPLATE']: + self.log_error("Must supply an XML file with " + "MET_DB_LOAD_XML_FILE") + + c_dict['INPUT_TEMPLATE'] = ( + self.config.getraw('config', + 'MET_DB_LOAD_INPUT_TEMPLATE') + ) + if not c_dict['INPUT_TEMPLATE']: + self.log_error("Must supply an input template with " + "MET_DB_LOAD_INPUT_TEMPLATE") + + c_dict['REMOVE_TMP_XML'] = ( + self.config.getbool('config', + 'MET_DB_LOAD_REMOVE_TMP_XML', + True) + ) + + # read config variables + for name, type in self.CONFIG_NAMES.items(): + if type == 'int': + get_fct = self.config.getint + elif type == 'bool': + get_fct = self.config.getbool + else: + get_fct = self.config.getraw + value = get_fct('config', + f'MET_DB_LOAD_MV_{name}', + '') + if value == '': + self.log_error(f"Must set MET_DB_LOAD_MV_{name}") + c_dict[f'MV_{name}'] = value + + c_dict['IS_MET_CMD'] = False + c_dict['LOG_THE_OUTPUT'] = True + + return c_dict + + def get_command(self): + """! Builds the command to run the MET application + @rtype string + @return Returns a MET command with arguments that you can run + """ + return f"python3 {self.app_path}.py {self.c_dict.get('XML_TMP_FILE')}" + + def run_at_time_once(self, time_info): + """! Process runtime and build command to run + + @param time_info dictionary containing time information + @returns True if command was run successfully, False otherwise + """ + success = True + + # if custom is already set in time info, run for only that item + # if not, loop over the CUSTOM_LOOP_LIST and process once for each + if 'custom' in time_info: + custom_loop_list = [time_info['custom']] + else: + custom_loop_list = self.c_dict['CUSTOM_LOOP_LIST'] + + for custom_string in custom_loop_list: + if custom_string: + self.logger.info(f"Processing custom string: {custom_string}") + + time_info['custom'] = custom_string + # if lead and either init or valid are set, compute other string sub + if time_info.get('lead') != '*': + if (time_info.get('init') != '*' + or time_info.get('valid') != '*'): + time_info = time_util.ti_calculate(time_info) + + self.set_environment_variables(time_info) + + if not self.replace_values_in_xml(time_info): + return + + # run command + if not self.build(): + success = False + + # remove tmp file + if self.c_dict.get('REMOVE_TMP_XML', True): + xml_file = self.c_dict.get('XML_TMP_FILE') + if xml_file and os.path.exists(xml_file): + self.logger.debug(f"Removing tmp file: {xml_file}") + os.remove(xml_file) + + return success + + def get_all_files(self, custom=None): + """! Don't get list of all files for METdataDB wrapper + + @returns True to report that no failures occurred + """ + return True + + def get_stat_directories(self, input_paths): + """! Traverse through files under input path and find all directories + that contain .stat or .tcst files. + + @param input_path top level directory to search + @returns list of unique directories that contain stat files + """ + stat_dirs = set() + for input_path in getlist(input_paths): + self.logger.debug("Finding directories with stat files " + f"under {input_path}") + for root, _, files in os.walk(input_path): + for filename in files: + if (not filename.endswith('.stat') and + not filename.endswith('.tcst')): + continue + filepath = os.path.join(root, filename) + stat_dir = os.path.dirname(filepath) + stat_dirs.add(stat_dir) + + stat_dirs = list(stat_dirs) + for stat_dir in stat_dirs: + self.logger.info(f"Adding stat file directory: {stat_dir}") + + return stat_dirs + + def format_stat_dirs(self, stat_dirs): + """! Format list of stat directories to substitute into XML file. + tags wil be added around each value. + + @param stat_dirs list of directories that contain stat files + @returns string of formatted values + """ + formatted_stat_dirs = [] + for stat_dir in stat_dirs: + formatted_stat_dirs.append(f'{stat_dir}') + + output_string = '\n '.join(formatted_stat_dirs) + return output_string + + def populate_sub_dict(self, time_info): + sub_dict = {} + + # substitute values from time dictionary + input_paths = ( + do_string_sub(self.c_dict['INPUT_TEMPLATE'], + **time_info) + ) + stat_dirs = self.get_stat_directories(input_paths) + formatted_stat_dirs = self.format_stat_dirs(stat_dirs) + sub_dict['METPLUS_INPUT_PATHS'] = formatted_stat_dirs + + for name, type in self.CONFIG_NAMES.items(): + value = str(self.c_dict.get(f'MV_{name}')) + if type == 'bool': + value = value.lower() + + value = do_string_sub(value, + **time_info) + + sub_dict[f'METPLUS_MV_{name}'] = value + + return sub_dict + + def replace_values_in_xml(self, time_info): + self.c_dict['XML_TMP_FILE'] = None + + xml_template = self.c_dict.get('XML_TEMPLATE') + if not xml_template: + return False + + # set up dictionary of text to substitute in XML file + sub_dict = self.populate_sub_dict(time_info) + + # open XML template file and replace any values encountered + with open(xml_template, 'r') as file_handle: + input_lines = file_handle.read().splitlines() + + output_lines = [] + for input_line in input_lines: + output_line = input_line + for replace_string, value in sub_dict.items(): + output_line = output_line.replace(f"${{{replace_string}}}", + value) + output_lines.append(output_line) + + # write tmp file with XML content with substituted values + out_filename = util.generate_tmp_filename() + out_path = os.path.join(self.config.getdir('TMP_DIR'), + out_filename) + with open(out_path, 'w') as file_handle: + for line in output_lines: + file_handle.write(f'{line}\n') + + self.c_dict['XML_TMP_FILE'] = out_path + return True diff --git a/parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoad.conf b/parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoad.conf new file mode 100644 index 0000000000..7aa355faea --- /dev/null +++ b/parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoad.conf @@ -0,0 +1,44 @@ +[config] + +# METdbLoad example + +PROCESS_LIST = METDbLoad + +LOOP_BY = VALID + +VALID_TIME_FMT = %Y%m%d%H +VALID_BEG = 2005080712 +VALID_END = 2005080800 +VALID_INCREMENT = 12H + +LOOP_ORDER = processes + +MET_DB_LOAD_RUNTIME_FREQ = RUN_ONCE + +MET_DATA_DB_DIR = {METPLUS_BASE}/../METdatadb + +MET_DB_LOAD_XML_FILE = {PARM_BASE}/use_cases/met_tool_wrapper/METdbLoad/METdbLoadConfig.xml + +# If true, remove temporary XML with values substituted from XML_FILE +# Set to false for debugging purposes +MET_DB_LOAD_REMOVE_TMP_XML = True + +# connection info +MET_DB_LOAD_MV_HOST = localhost:3306 +MET_DB_LOAD_MV_DATABASE = mv_metplus_test +MET_DB_LOAD_MV_USER = root +MET_DB_LOAD_MV_PASSWORD = mvuser + +# data info +MET_DB_LOAD_MV_VERBOSE = false +MET_DB_LOAD_MV_INSERT_SIZE = 1 +MET_DB_LOAD_MV_MODE_HEADER_DB_CHECK = false +MET_DB_LOAD_MV_DROP_INDEXES = false +MET_DB_LOAD_MV_APPLY_INDEXES = true +MET_DB_LOAD_MV_GROUP = METplus Input Test +MET_DB_LOAD_MV_LOAD_STAT = true +MET_DB_LOAD_MV_LOAD_MODE = false +MET_DB_LOAD_MV_LOAD_MTD = false +MET_DB_LOAD_MV_LOAD_MPR = false + +MET_DB_LOAD_INPUT_TEMPLATE = {INPUT_BASE}/met_test/out/grid_stat diff --git a/parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoadConfig.xml b/parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoadConfig.xml new file mode 100644 index 0000000000..2bc657baf2 --- /dev/null +++ b/parm/use_cases/met_tool_wrapper/METdbLoad/METdbLoadConfig.xml @@ -0,0 +1,26 @@ + + + ${METPLUS_MV_HOST} + ${METPLUS_MV_DATABASE} + ${METPLUS_MV_USER} + ${METPLUS_MV_PASSWORD} + + + ${METPLUS_MV_VERBOSE} + ${METPLUS_MV_INSERT_SIZE} + ${METPLUS_MV_MODE_HEADER_DB_CHECK} + ${METPLUS_MV_DROP_INDEXES} + ${METPLUS_MV_APPLY_INDEXES} + ${METPLUS_MV_GROUP} + ${METPLUS_MV_LOAD_STAT} + ${METPLUS_MV_LOAD_MODE} + ${METPLUS_MV_LOAD_MTD} + ${METPLUS_MV_LOAD_MPR} + + {dirs} + + + ${METPLUS_INPUT_PATHS} + + + \ No newline at end of file