Update develop-ref after #2050 (#2053)

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,7 +1,7 @@
 ---
 name: Bug report
 about: Fix something that's not working
-title: ''
+title: 'Bugfix: '
 labels: 'alert: NEED ACCOUNT KEY, alert: NEED MORE DEFINITION, alert: NEED PROJECT ASSIGNMENT, type: bug'
 assignees: ''
 

.github/ISSUE_TEMPLATE/new_use_case.md

@@ -1,7 +1,7 @@
 ---
 name: New use case
 about: Add a new use case
-title: 'New Use Case:'
+title: 'New Use Case: '
 labels: 'alert: NEED ACCOUNT KEY, alert: NEED MORE DEFINITION, alert: NEED PROJECT ASSIGNMENT, type: new use case'
 assignees: ''
 
@@ -56,7 +56,7 @@ Consider breaking the new feature down into sub-issues.
 
 ## Define Related Issue(s) ##
 Consider the impact to the other METplus components.
-- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdatadb](https://github.com/dtcenter/METdatadb/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose)
+- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose)
 
 ## New Use Case Checklist ##
 See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details.

.github/actions/run_tests/Dockerfile.run

@@ -6,5 +6,6 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env
 ARG METPLUS_IMG_TAG=develop
 FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG}
 
-COPY --from=env /usr/local/envs /usr/local/envs/
-COPY --from=env /usr/local/bin/conda /usr/local/bin/conda
+RUN mkdir -p /usr/local/conda/envs && mkdir -p /usr/local/conda/bin
+COPY --from=env /usr/local/conda/envs /usr/local/conda/envs/
+COPY --from=env /usr/local/conda/bin/conda /usr/local/conda/bin/conda

.github/actions/run_tests/Dockerfile.run_cartopy

@@ -6,7 +6,8 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env
 ARG METPLUS_IMG_TAG=develop
 FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG}
 
-COPY --from=env /usr/local/envs /usr/local/envs/
-COPY --from=env /usr/local/bin/conda /usr/local/bin/conda
+RUN mkdir -p /usr/local/conda/envs && mkdir -p /usr/local/conda/bin
+COPY --from=env /usr/local/conda/envs /usr/local/conda/envs/
+COPY --from=env /usr/local/conda/bin/conda /usr/local/conda/bin/conda
 
 COPY --from=env /root/.local/share/cartopy /root/.local/share/cartopy/

.github/actions/run_tests/Dockerfile.run_gempak

@@ -6,6 +6,8 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env
 ARG METPLUS_IMG_TAG=develop
 FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG}
 
-COPY --from=env /usr/lib/jvm/jre /usr/lib/jvm/jre/
-COPY --from=env /usr/share/javazi-1.8/tzdb.dat /usr/share/javazi-1.8/
+COPY --from=env /usr/lib/jvm/adoptopenjdk-8-hotspot-amd64 /usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/
 COPY --from=env /data/input/GempakToCF.jar /data/input/GempakToCF.jar
+
+# add java executable to path
+RUN echo "export PATH=/usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/bin:\$PATH;" >> /root/.bashrc

.github/actions/run_tests/Dockerfile.run_gfdl

@@ -6,6 +6,7 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env
 ARG METPLUS_IMG_TAG=develop
 FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG}
 
+COPY --from=env /usr/local/lib/libhdf5* /usr/local/lib/
 COPY --from=env /usr/local/lib/libnetcdf.so.19* /usr/local/lib/
 COPY --from=env /usr/local/lib/libnetcdff* /usr/local/lib/
 COPY --from=env /standalone_gfdl-vortextracker_v3.9a_gcc/trk_exec/* /usr/local/bin/

.github/actions/run_tests/entrypoint.sh

@@ -34,7 +34,7 @@ fi
 
 # running unit tests (pytests)
 if [[ "$INPUT_CATEGORIES" == pytests* ]]; then
-  export METPLUS_ENV_TAG="pytest"
+  export METPLUS_ENV_TAG="pytest.v5.1"
   export METPLUS_IMG_TAG=${branch_name}
   echo METPLUS_ENV_TAG=${METPLUS_ENV_TAG}
   echo METPLUS_IMG_TAG=${METPLUS_IMG_TAG}
@@ -61,7 +61,7 @@ if [[ "$INPUT_CATEGORIES" == pytests* ]]; then
   for x in `cat $PYTESTS_GROUPS_FILEPATH`; do
     marker="${x//_or_/ or }"
     marker="${marker//not_/not }"
-    command+="/usr/local/envs/pytest/bin/pytest -vv --cov=../../../metplus -m \"$marker\""
+    command+="/usr/local/conda/envs/${METPLUS_ENV_TAG}/bin/pytest -vv --cov=../../../metplus -m \"$marker\""
     command+=";if [ \$? != 0 ]; then status=1; fi;"
   done
   command+="if [ \$status != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi"

.github/jobs/bash_functions.sh

@@ -1,16 +1,20 @@
 #! /bin/bash
 
 # utility function to run command get log the time it took to run
+# ::group:: and ::endgroup:: create collapsible log groups in GitHub Actions
 function time_command {
   local start_seconds=$SECONDS
-  echo "RUNNING: $*"
+  echo "::group::RUNNING: $*"
   "$@"
   local error=$?
 
   local duration=$(( SECONDS - start_seconds ))
   echo "TIMING: Command took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS): '$*'"
+  echo "::endgroup::"
+
   if [ ${error} -ne 0 ]; then
     echo "ERROR: '$*' exited with status = ${error}"
   fi
+
   return $error
 }

.github/jobs/copy_error_logs.py

@@ -10,12 +10,11 @@
 
 def main(output_data_dir, error_logs_dir):
     """! Copy log output to error log directory if any use case failed """
-    for use_case_dir in os.listdir(output_data_dir):
-        log_dir = os.path.join(output_data_dir,
-                               use_case_dir,
-                               'logs')
-        if not os.path.isdir(log_dir):
+    for log_dir, _, log_files in os.walk(output_data_dir):
+        # skip non logs directories
+        if not log_dir.endswith('/logs'):
             continue
+        use_case_dir = log_dir.replace(f'{output_data_dir}/', '')[0:-5]
 
         # check if there are errors in the metplus.log file and
         # only copy directory if there are any errors
@@ -28,9 +27,7 @@ def main(output_data_dir, error_logs_dir):
         if not found_errors:
             continue
 
-        output_dir = os.path.join(error_logs_dir,
-                                  use_case_dir)
-        log_files = os.listdir(log_dir)
+        output_dir = os.path.join(error_logs_dir, use_case_dir)
         for log_file in log_files:
             log_path = os.path.join(log_dir, log_file)
             output_path = os.path.join(output_dir, log_file)

.github/jobs/create_dirs_for_database.sh

@@ -1,6 +1,6 @@
 #! /bin/bash
 
-# Create directories used for use case that uses METviewer and METdatadb
+# Create directories used for use case that uses METviewer and METdataio
 # Open up write permissions for these directories so that files can
 # be written by METviewer Docker container.
 # Called by .github/workflows/testing.yml

.github/jobs/docker_utils.py

@@ -1,5 +1,8 @@
 import os
 import re
+import subprocess
+import shlex
+import time
 
 # Utilities used by various CI jobs. Functionality includes:
 #  - Check if Docker data volumes need to be updated.
@@ -13,7 +16,8 @@
 DOCKERHUB_METPLUS_DATA_DEV = 'dtcenter/metplus-data-dev'
 
 # extension to add to conda environments
-VERSION_EXT = '.v5'
+VERSION_EXT = '.v5.1'
+
 
 def get_data_repo(branch_name):
     """! Branch names that start with main_v or contain only
@@ -26,10 +30,12 @@ def get_data_repo(branch_name):
         return DOCKERHUB_METPLUS_DATA
     return DOCKERHUB_METPLUS_DATA_DEV
 
+
 def get_dockerhub_url(branch_name):
     data_repo = get_data_repo(branch_name)
     return f'https://hub.docker.com/v2/repositories/{data_repo}/tags'
 
+
 def docker_get_volumes_last_updated(current_branch):
     import requests
     dockerhub_url = get_dockerhub_url(current_branch)
@@ -60,22 +66,79 @@ def docker_get_volumes_last_updated(current_branch):
 
     return volumes_last_updated
 
+
 def get_branch_name():
     # get branch name from env var BRANCH_NAME
     branch_name = os.environ.get('BRANCH_NAME')
     if branch_name:
-        return branch_name
+        return branch_name.replace('/', '_')
 
     # if BRANCH_NAME not set, use GITHUB env vars
     github_event_name = os.environ.get('GITHUB_EVENT_NAME')
     if not github_event_name:
         return None
 
     if github_event_name == 'pull_request':
-        return os.environ.get('GITHUB_HEAD_REF')
+        branch_name = os.environ.get('GITHUB_HEAD_REF')
+        if branch_name:
+            branch_name = branch_name.replace('/', '_')
+        return branch_name
 
     github_ref = os.environ.get('GITHUB_REF')
     if github_ref is None:
         return None
 
-    return github_ref.replace('refs/heads/', '')
+    return github_ref.replace('refs/heads/', '').replace('/', '_')
+
+
+def run_commands(commands):
+    """!Run a list of commands via subprocess. Print the command and the length
+    of time it took to run. Includes ::group:: and ::endgroup:: syntax which
+    creates log groups in GitHub Actions log output.
+
+    @param commands list of commands to run or a single command string
+    @returns True if all commands ran successfully, False if any commands fail
+    """
+    # handle a single command string or list of command strings
+    if isinstance(commands, str):
+        command_list = [commands]
+    else:
+        command_list = commands
+
+    is_ok = True
+    for command in command_list:
+        error_message = None
+        print(f"::group::RUNNING {command}")
+        start_time = time.time()
+        try:
+            process = subprocess.Popen(shlex.split(command),
+                                       shell=False,
+                                       encoding='utf-8',
+                                       stdout=subprocess.PIPE,
+                                       stderr=subprocess.STDOUT)
+            # Poll process.stdout to show stdout live
+            while True:
+                output = process.stdout.readline()
+                if process.poll() is not None:
+                    break
+                if output:
+                    print(output.strip())
+            rc = process.poll()
+            if rc:
+                raise subprocess.CalledProcessError(rc, command)
+
+        except subprocess.CalledProcessError as err:
+            error_message = f"ERROR: Command failed -- {err}"
+            is_ok = False
+
+        end_time = time.time()
+        print("TIMING: Command took "
+              f"{time.strftime('%M:%S', time.gmtime(end_time - start_time))}"
+              f" (MM:SS): '{command}')")
+
+        print("::endgroup::")
+
+        if error_message:
+            print(error_message)
+
+    return is_ok

.github/jobs/get_data_volumes.py

@@ -12,6 +12,8 @@
 
 from docker_utils import docker_get_volumes_last_updated, get_branch_name
 from docker_utils import get_data_repo, DOCKERHUB_METPLUS_DATA_DEV
+from docker_utils import run_commands
+
 
 def main(args):
     # get METplus version
@@ -94,9 +96,7 @@ def main(args):
         print(f"CREATING DATA VOLUME FROM: {full_volume_name}")
         cmd = (f'docker create --name {model_app_name} '
                f'{full_volume_name}')
-        ret = subprocess.run(shlex.split(cmd))
-
-        if ret.returncode:
+        if not run_commands(cmd):
             continue
 
         # add name to volumes from list to pass to docker build

.github/jobs/get_metviewer.sh

@@ -21,7 +21,7 @@ sleep 20
 # print list of currently running containers to
 # verify mysql and metviewer are running
 docker ps -a
-# TODO: when METviewer Dockerfile changes METdatadb to METdataio, update path to sql file accordingly
+
 # commands to run inside METviewer container
 cmd="mysql -hmysql_mv -uroot -pmvuser -e\"create database mv_metplus_test;\";"
 cmd+="mysql -hmysql_mv -uroot -pmvuser mv_metplus_test < /METviewer-python/METdataio/METdbLoad/sql/mv_mysql.sql"