Feature 629 metdbload

.github/workflows/main.yml

@@ -123,6 +123,7 @@ jobs:
           - "s2s:1-3"
           - "space_weather:0-1"
           - "tc_and_extra_tc:0-2"
+          - "met_tool_wrapper:54:NEW"
     steps:
       - uses: actions/download-artifact@v2
         with:
@@ -138,6 +139,12 @@ jobs:
           echo run_this_case=$run_this_case >> $GITHUB_ENV
           echo Ends with NEW: ${{ endsWith(matrix.categories, ':NEW') }}
           echo Run all: ${{ env.run_all_use_cases }}
+      - name: Create directories for database
+        run: |
+          mkdir -p $RUNNER_WORKSPACE/mysql
+          mkdir -p $RUNNER_WORKSPACE/output/metviewer
+          chmod a+w $RUNNER_WORKSPACE/mysql
+          chmod a+w $RUNNER_WORKSPACE/output/metviewer
       - uses: actions/checkout@v2
         if: ${{ env.run_this_case == 'true' }}
       - uses: ./ci/actions/run_tests

ci/actions/run_tests/entrypoint.sh

@@ -46,6 +46,16 @@ if [ "$INPUT_CATEGORIES" == "pytests" ]; then
   exit $?
 fi
 
+# get METviewer if used in any use cases
+all_requirements=`./ci/jobs/get_requirements.py ${CATEGORIES} ${SUBSETLIST}`
+echo All requirements: $all_requirements
+NETWORK_ARG=""
+if [[ "$all_requirements" =~ .*"metviewer".* ]]; then
+  echo "Setting up METviewer"
+  ${GITHUB_WORKSPACE}/ci/jobs/python_requirements/get_metviewer.sh
+  NETWORK_ARG=--network="container:mysql_mv"
+fi
+
 # install Pillow library needed for diff testing
 # this will be replaced with better image diffing package used by METplotpy
 pip_command="pip3 install Pillow; yum -y install poppler-utils; pip3 install pdf2image"
@@ -86,6 +96,9 @@ fi
 
 echo VOLUMES_FROM: $VOLUMES_FROM
 
+echo docker ps:
+docker ps -a
+
 echo "Run Docker container: $DOCKERHUBTAG"
-echo docker run -e GITHUB_WORKSPACE -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}"
-docker run -e GITHUB_WORKSPACE -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}"
+echo docker run -e GITHUB_WORKSPACE $NETWORK_ARG -v $RUNNER_WORKSPACE/output/mysql:/var/lib/mysql -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}"
+docker run -e GITHUB_WORKSPACE $NETWORK_ARG -v $RUNNER_WORKSPACE/output/mysql:/var/lib/mysql -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}"

ci/jobs/get_requirements.py

@@ -0,0 +1,37 @@
+#! /usr/bin/env python3
+
+# Used in GitHub Actions (in ci/actions/run_tests/entrypoint.sh)
+# to obtain list of requirements from use case group
+
+import os
+import sys
+
+import get_use_case_commands
+
+# add internal_tests/use_cases directory to path so the test suite can be found
+USE_CASES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                             os.pardir,
+                                             os.pardir))
+sys.path.insert(0, USE_CASES_DIR)
+
+from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite
+
+def main():
+    all_requirements = set()
+    categories, subset_list, compare = (
+        get_use_case_commands.handle_command_line_args()
+    )
+
+    test_suite = METplusUseCaseSuite()
+    test_suite.add_use_case_groups(categories, subset_list)
+
+    for group_name, use_cases_by_req in test_suite.category_groups.items():
+        for use_case_by_req in use_cases_by_req:
+            for requirement in use_case_by_req.requirements:
+                all_requirements.add(requirement)
+
+    return list(all_requirements)
+
+if __name__ == '__main__':
+    all_requirements = main()
+    print(','.join(all_requirements))

ci/jobs/get_use_case_commands.py

@@ -19,6 +19,11 @@
 def handle_requirements(requirements, work_dir):
     requirement_args = []
     for requirement in requirements:
+        # don't obtain METviewer here because it has to be set up outside of
+        # docker container that runs the use cases
+        if requirement.lower() == 'metviewer':
+            continue
+
         # check if get_{requirement} script exists and use it if it does
         script_path = os.path.join(work_dir,
                                    'ci',

ci/jobs/python_requirements/get_metdatadb.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+pip3 install lxml
+pip3 install PyMySQL
+
+basedir=$(dirname "$0")
+work_dir=$basedir/../../..
+
+# run manage externals to obtain METdatadb
+${work_dir}/manage_externals/checkout_externals -e ${work_dir}/ci/parm/Externals_metdatadb.cfg

ci/jobs/python_requirements/get_metviewer.sh

@@ -0,0 +1,33 @@
+#! /bin/bash
+
+# set environment variables needed by METviewer docker-compose.yml
+export METVIEWER_DATA=$RUNNER_WORKSPACE
+export MYSQL_DIR=$RUNNER_WORKSPACE/mysql
+export METVIEWER_DIR=$RUNNER_WORKSPACE/output/metviewer
+export METVIEWER_DOCKER_IMAGE=dtcenter/metviewer
+
+# install docker-compose
+apk add docker-compose
+
+# download docker-compose.yml file from METviewer develop branch
+wget https://raw.githubusercontent.com/dtcenter/METviewer/develop/docker/docker-compose.yml
+
+# Run docker-compose to create the containers
+docker-compose up -d
+
+# sleep for a few seconds to ensure database has fully started
+sleep 20
+
+# print list of currently running containers to
+# verify mysql and metviewer are running
+docker ps -a
+
+# commands to run inside METviewer container
+cmd="mysql -hmysql_mv -uroot -pmvuser -e\"create database mv_metplus_test;\";"
+cmd+="mysql -hmysql_mv -uroot -pmvuser mv_metplus_test < /METviewer/sql/mv_mysql.sql"
+cmd+=";mysql -hmysql_mv -uroot -pmvuser -e\"show databases;\""
+
+# execute commands inside metviewer container to create database
+echo Executing commands inside metviewer_1 container to create database
+echo docker exec metviewer_1 /bin/bash -c "$cmd"
+docker exec metviewer_1 /bin/bash -c "$cmd"

ci/jobs/set_job_controls.sh

@@ -48,6 +48,10 @@ else
     run_use_cases=false
   fi
 
+  if grep -q "ci-skip-unit-tests" <<< "$commit_msg"; then
+    run_unit_tests=false
+  fi
+
   if grep -q "ci-only-docs" <<< "$commit_msg"; then
     run_docs=true
     run_get_image=false

ci/parm/Externals_metdatadb.cfg

@@ -0,0 +1,9 @@
+[METdatadb]
+local_path = ../METdatadb
+protocol = git
+required = True
+repo_url = https://github.com/dtcenter/METdatadb
+branch = develop
+
+[externals_description]
+schema_version = 1.0.0

docs/Users_Guide/glossary.rst

@@ -5985,6 +5985,129 @@ METplus Configuration Glossary
 
      | *Used by:*  SeriesAnalysis
 
+   MET_DB_LOAD_RUNTIME_FREQ
+     Frequency to run Grid-Diag. See :ref:`Runtime_Freq` for more information.
+
+     | *Used by:*  GridDiag
+
+   MET_DATA_DB_DIR
+     Set this the location of the dtcenter/METdatadb repository.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_XML_FILE
+     Template XML file that is used to load data into METviewer using the
+     met_db_load.py script. Values from the METplus configuration file are
+     substituted into this file before passing it to the script. The default
+     value can be used to run unless the template doesn't fit the needs of the
+     use case.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_REMOVE_TMP_XML
+     If set to False, then the temporary XML file with substituted values will
+     not be removed after the use case finishes. This is used for debugging
+     purposes only. The temporary XML file may contain sensitive information
+     like database credentials so it is recommended to remove the temporary
+     file after each run.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_HOST
+     Set the <load_spec><connection><host> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_DATABASE
+     Set the <load_spec><connection><database> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_USER
+     Set the <load_spec><connection><user> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_PASSWORD
+     Set the <load_spec><connection><password> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_VERBOSE
+     Set the <load_spec><verbose> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_INSERT_SIZE
+     Set the <load_spec><insert_size> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_MODE_HEADER_DB_CHECK
+     Set the <load_spec><mode_header_db_check> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_DROP_INDEXES
+     Set the <load_spec><drop_indexes> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_APPLY_INDEXES
+     Set the <load_spec><apply_indexes> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_GROUP
+     Set the <load_spec><group> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_LOAD_STAT
+     Set the <load_spec><load_stat> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_LOAD_MODE
+     Set the <load_spec><load_mode> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_LOAD_MTD
+     Set the <load_spec><load_mtd> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_MV_LOAD_MPR
+     Set the <load_spec><load_mpr> value in the
+     METdbLoad XML template file.
+
+     | *Used by:*  METdbLoad
+
+   MET_DB_LOAD_INPUT_TEMPLATE
+     Path to a directory containing .stat or .tcst file that will be loaded
+     into METviewer. This can be a single directory or a list of directories.
+     The paths can include filename template tags that correspond to each
+     run time. The wrapper will traverse through each sub directory under the
+     directories listed here and add any directory that contains any files that
+     end with .stat or .tcst to the XML file that is passed into the
+     met_db_load.py script.
+
+     | *Used by:*  METdbLoad
+
    CYCLONE_PLOTTER_ADD_WATERMARK
      If set to True, add a watermark with the current time to the image generated by
      CyclonePlotter.

docs/Users_Guide/quicksearch.rst

@@ -65,6 +65,7 @@ Use Cases by METplus Feature:
 | `Looping by Month or Year  <https://metplus.readthedocs.io/en/develop/search.html?q=LoopByMonthFeatureUseCase&check_keywords=yes&area=default>`_
 | `List Expansion (using begin_end_incr syntax)  <https://metplus.readthedocs.io/en/develop/search.html?q=ListExpansionFeatureUseCase&check_keywords=yes&area=default>`_
 | `Masking for Regions of Interest <https://metplus.readthedocs.io/en/develop/search.html?q=MaskingFeatureUseCase&check_keywords=yes&area=default>`_
+| `METdbLoad <https://metplus.readthedocs.io/en/develop/search.html?q=METdbLoadUseCase&check_keywords=yes&area=default>`_
 | `MET_PYTHON_EXE Environment Variable  <https://metplus.readthedocs.io/en/develop/search.html?q=MET_PYTHON_EXEUseCase&check_keywords=yes&area=default>`_
 | `Multiple Conf File Use <https://metplus.readthedocs.io/en/develop/search.html?q=MultiConfUseCase&check_keywords=yes&area=default>`_
 | `Observation Time Summary <https://metplus.readthedocs.io/en/develop/search.html?q=ObsTimeSummaryUseCase&check_keywords=yes&area=default>`_