feature #1566 Use Python 3.8.6 for automated tests

.github/actions/run_tests/Dockerfile.run

@@ -1,4 +1,4 @@
-ARG METPLUS_ENV_TAG=metplus_base
+ARG METPLUS_ENV_TAG=metplus_base.v5
 ARG METPLUS_IMG_TAG=develop
 
 FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env

.github/actions/run_tests/Dockerfile.run_cartopy

@@ -1,4 +1,4 @@
-ARG METPLUS_ENV_TAG=cycloneplotter
+ARG METPLUS_ENV_TAG=cycloneplotter.v5
 ARG METPLUS_IMG_TAG=develop
 
 FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env

.github/jobs/docker_utils.py

@@ -12,6 +12,9 @@
 # repository used for storing input data for development branches
 DOCKERHUB_METPLUS_DATA_DEV = 'dtcenter/metplus-data-dev'
 
+# extension to add to conda environments
+VERSION_EXT = '.v5'
+
 def get_data_repo(branch_name):
     """! Branch names that start with main_v or contain only
        digits and dots with out without a prefix 'v' will return

.github/jobs/get_requirements.py

@@ -8,11 +8,11 @@
 
 import get_use_case_commands
 
-# add internal_tests/use_cases directory to path so the test suite can be found
-USE_CASES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                             os.pardir,
-                                             os.pardir))
-sys.path.insert(0, USE_CASES_DIR)
+# add METplus directory to path so the test suite can be found
+METPLUS_TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                               os.pardir,
+                                               os.pardir))
+sys.path.insert(0, METPLUS_TOP_DIR)
 
 from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite
 

.github/jobs/get_use_case_commands.py

@@ -8,17 +8,22 @@
 import os
 
 # add METplus directory to sys path so the test suite can be found
-USE_CASES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                             os.pardir,
-                                             os.pardir))
-sys.path.insert(0, USE_CASES_DIR)
+METPLUS_TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                               os.pardir,
+                                               os.pardir))
+sys.path.insert(0, METPLUS_TOP_DIR)
 
 from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite
 from metplus.util.met_util import expand_int_string_to_list
+from docker_utils import VERSION_EXT
 
-METPLUS_BASE_ENV = 'metplus_base'
+
+# path to METplus install location in Docker
 METPLUS_DOCKER_LOC = '/metplus/METplus'
 
+# name of conda environment used for cases that don't need special env
+METPLUS_BASE_ENV = 'metplus_base'
+
 # keywords in requirements list that trigger obtaining METcalcpy and METplotpy
 PLOTCALC_KEYWORDS = [
     'metplotpy',
@@ -27,6 +32,12 @@
     'weatherregime',
 ]
 
+# Docker envs that do not use Python so they do not need print conda list
+NOT_PYTHON_ENVS = [
+    'gfdl-tracker',
+    'gempak',
+]
+
 def handle_automation_env(host_name, reqs, work_dir):
     # if no env is specified, use metplus base environment
     conda_env = METPLUS_BASE_ENV
@@ -44,12 +55,15 @@ def handle_automation_env(host_name, reqs, work_dir):
             return '', 'user_env_vars.MET_PYTHON_EXE=python3'
         return '', ''
 
+    # add version extension to conda environment name
+    conda_env_w_ext = f'{conda_env}{VERSION_EXT}'
+
     # start building commands to run before run_metplus.py in Docker
     setup_env = 'source /etc/bashrc;'
 
     # add conda bin to beginning of PATH
     python_dir = os.path.join('/usr', 'local', 'envs',
-                              conda_env, 'bin')
+                              conda_env_w_ext, 'bin')
     python_path = os.path.join(python_dir, 'python3')
     setup_env += f' export PATH={python_dir}:$PATH;'
 
@@ -92,12 +106,12 @@ def handle_automation_env(host_name, reqs, work_dir):
         setup_env += f'export PYTHONPATH={METPLUS_DOCKER_LOC}:$PYTHONPATH;'
 
     # list packages in python environment that will be used
-    if conda_env != 'gempak':
+    if conda_env not in NOT_PYTHON_ENVS:
         setup_env += (
-            f'echo Using environment: dtcenter/metplus-envs:{conda_env};'
-            f'echo cat /usr/local/envs/{conda_env}/environments.yml;'
+            f'echo Using environment: dtcenter/metplus-envs:{conda_env_w_ext};'
+            f'echo cat /usr/local/envs/{conda_env_w_ext}/environments.yml;'
             f'echo ----------------------------------------;'
-            f'cat /usr/local/envs/{conda_env}/environments.yml;'
+            f'cat /usr/local/envs/{conda_env_w_ext}/environments.yml;'
             'echo ----------------------------------------;'
         )
 
@@ -108,7 +122,7 @@ def main(categories, subset_list, work_dir=None,
     all_commands = []
 
     if work_dir is None:
-        work_dir = USE_CASES_DIR
+        work_dir = METPLUS_TOP_DIR
 
     test_suite = METplusUseCaseSuite()
     test_suite.add_use_case_groups(categories, subset_list)

.github/jobs/setup_and_run_diff.py

@@ -5,6 +5,8 @@
 import subprocess
 import shlex
 
+from docker_utils import VERSION_EXT
+
 ci_dir = os.path.join(os.environ.get('GITHUB_WORKSPACE'), '.github')
 sys.path.insert(0, ci_dir)
 
@@ -46,12 +48,12 @@
 mount_args = ' '.join(volume_mounts)
 
 # command to run inside Docker
-cmd = ('/usr/local/envs/diff/bin/python3 '
+cmd = (f'/usr/local/envs/diff{VERSION_EXT}/bin/python3 '
        f'{GITHUB_WORKSPACE}/{CI_JOBS_DIR}/run_diff_docker.py')
 
 # run inside diff env: mount METplus code and output dir, volumes from output volumes
 docker_cmd = (f'docker run -e GITHUB_WORKSPACE {VOLUMES_FROM} '
-              f'{mount_args} dtcenter/metplus-envs:diff '
+              f'{mount_args} dtcenter/metplus-envs:diff{VERSION_EXT} '
               f'bash -c "{cmd}"')
 print(f'RUNNING: {docker_cmd}')
 try:

.github/jobs/setup_and_run_use_cases.py

@@ -16,7 +16,7 @@
 
 import get_use_case_commands
 import get_data_volumes
-from docker_utils import get_branch_name
+from docker_utils import get_branch_name, VERSION_EXT
 
 runner_workspace = os.environ.get('RUNNER_WORKSPACE')
 github_workspace = os.environ.get('GITHUB_WORKSPACE')
@@ -70,6 +70,8 @@ def main():
         else:
             env_tag = 'metplus_base'
 
+        env_tag = f'{env_tag}{VERSION_EXT}'
+
         # get Dockerfile to use
         dockerfile_name = 'Dockerfile.run'
         if 'gempak' in str(requirements).lower():

internal_tests/use_cases/all_use_cases.txt

@@ -139,6 +139,7 @@ Category: s2s_mid_lat
 2::UserScript_obsERA_obsOnly_WeatherRegime:: model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_WeatherRegime.conf:: weatherregime_env,cartopy,metplus
 3:: UserScript_fcstGFS_obsERA_WeatherRegime:: model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_WeatherRegime.conf:: weatherregime_env,cartopy,metplus
 
+
 Category: s2s_mjo
 0:: UserScript_obsERA_obsOnly_PhaseDiagram:: model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.conf:: spacetime_env
 1:: UserScript_fcstGFS_obsERA_OMI:: model_applications/s2s_mjo/UserScript_fcstGFS_obsERA_OMI.conf:: spacetime_env, metdataio

scripts/docker/docker_env/Dockerfile

@@ -1,13 +1,14 @@
 # Dockerfile to create conda environments used for use case tests
 
-ARG BASE_ENV=metplus_base
+ARG BASE_ENV=metplus_base.v5
 FROM dtcenter/metplus-envs:${BASE_ENV}
 
 ARG ENV_NAME
 WORKDIR /scripts
 COPY scripts/${ENV_NAME}_env.sh .
 
-ARG BASE_ENV=metplus_base
-RUN ./${ENV_NAME}_env.sh ${BASE_ENV}
+ARG BASE_ENV=metplus_base.v5
+RUN conda update -y -n base -c defaults conda \
+  && ./${ENV_NAME}_env.sh ${BASE_ENV}
 
-RUN conda list --name ${ENV_NAME} > /usr/local/envs/${ENV_NAME}/environments.yml
+RUN conda list --name ${ENV_NAME}.v5 > /usr/local/envs/${ENV_NAME}.v5/environments.yml

scripts/docker/docker_env/Dockerfile.metplus_base

@@ -9,4 +9,4 @@ COPY scripts/metplus_base_env.sh .
 RUN conda update -y -n base -c defaults conda \
  && ./metplus_base_env.sh
 
-RUN conda list --name metplus_base > /usr/local/envs/metplus_base/environments.yml
+RUN conda list --name metplus_base.v5 > /usr/local/envs/metplus_base.v5/environments.yml

scripts/docker/docker_env/Dockerfile.py_embed_base

@@ -10,4 +10,4 @@ COPY scripts/py_embed_base_env.sh .
 RUN conda update -y -n base -c defaults conda \
  && ./py_embed_base_env.sh
 
-RUN conda list --name py_embed_base > /usr/local/envs/py_embed_base/environments.yml
+RUN conda list --name py_embed_base.v5 > /usr/local/envs/py_embed_base.v5/environments.yml