From 57c2a8cb415c3188093cc11dcd4acd36407a0c7f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:52:44 -0600 Subject: [PATCH 01/18] set default input version to the latest official release, which is the 2nd highest coordinated release version number in the version lookup table. This assumes that the highest version is what is being developed towards --- metplus/component_versions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/metplus/component_versions.py b/metplus/component_versions.py index 518bcc88f..97275e519 100755 --- a/metplus/component_versions.py +++ b/metplus/component_versions.py @@ -34,6 +34,10 @@ }, } +# assumes the 2nd highest version in the table is the latest +# the highest version in the table is in development +LATEST_OFFICIAL_RELEASE = sorted(VERSION_LOOKUP.keys(), reverse=True)[1] + DEFAULT_OUTPUT_FORMAT = "v{X}.{Y}.{Z}{N}" def get_component_version(input_component, input_version, output_component, @@ -99,9 +103,9 @@ def main(): help='Name of METplus component to use to find version,' ' default is METplus.') parser.add_argument('-v', '--input_version', - default=next(iter(VERSION_LOOKUP)), + default=LATEST_OFFICIAL_RELEASE, help='version of input_component to search,' - ' default is upcoming version') + ' default is latest official release') parser.add_argument('-o', '--output_component', required=True, help='name of METplus component to obtain version') parser.add_argument('-f', '--output_format', From f12e3b92785142dc86cc13eb6698508934e038d8 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 5 Nov 2024 12:48:23 -0700 Subject: [PATCH 02/18] per #2562, update Release Guide to include instructions for updating the component versions lookup table in the METplus repository --- docs/Release_Guide/met_bugfix.rst | 1 + docs/Release_Guide/metcalcpy_bugfix.rst | 1 + docs/Release_Guide/metdataio_bugfix.rst | 1 + docs/Release_Guide/metexpress_official.rst | 1 + docs/Release_Guide/metplotpy_bugfix.rst | 1 + docs/Release_Guide/metplus_bugfix.rst | 1 + docs/Release_Guide/metplus_official.rst | 1 + docs/Release_Guide/metviewer_bugfix.rst | 1 + .../metplus/add_next_version_to_lookup.rst | 38 +++++++++++++++ .../release_steps/update_version_lookup.rst | 46 +++++++++++++++++++ 10 files changed, 92 insertions(+) create mode 100644 docs/Release_Guide/release_steps/metplus/add_next_version_to_lookup.rst create mode 100644 docs/Release_Guide/release_steps/update_version_lookup.rst diff --git a/docs/Release_Guide/met_bugfix.rst b/docs/Release_Guide/met_bugfix.rst index 6861bc8b7..d70e91427 100644 --- a/docs/Release_Guide/met_bugfix.rst +++ b/docs/Release_Guide/met_bugfix.rst @@ -15,4 +15,5 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/met/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_bugfix.rst diff --git a/docs/Release_Guide/metcalcpy_bugfix.rst b/docs/Release_Guide/metcalcpy_bugfix.rst index b224fdb20..62d6ade67 100644 --- a/docs/Release_Guide/metcalcpy_bugfix.rst +++ b/docs/Release_Guide/metcalcpy_bugfix.rst @@ -15,4 +15,5 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_official.rst diff --git a/docs/Release_Guide/metdataio_bugfix.rst b/docs/Release_Guide/metdataio_bugfix.rst index 096e9c260..992e44b48 100644 --- a/docs/Release_Guide/metdataio_bugfix.rst +++ b/docs/Release_Guide/metdataio_bugfix.rst @@ -18,4 +18,5 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/common/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_bugfix.rst diff --git a/docs/Release_Guide/metexpress_official.rst b/docs/Release_Guide/metexpress_official.rst index 256c143a6..20cfcc152 100644 --- a/docs/Release_Guide/metexpress_official.rst +++ b/docs/Release_Guide/metexpress_official.rst @@ -18,5 +18,6 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/metexpress/pull_changes_and_merge_to_dev.rst .. include:: release_steps/update_docs_official.rst .. include:: release_steps/metexpress/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/metexpress/announce_release.rst .. include:: release_steps/set_beta_deletion_reminder_official.rst diff --git a/docs/Release_Guide/metplotpy_bugfix.rst b/docs/Release_Guide/metplotpy_bugfix.rst index ec177ee9b..f9609dcde 100644 --- a/docs/Release_Guide/metplotpy_bugfix.rst +++ b/docs/Release_Guide/metplotpy_bugfix.rst @@ -14,5 +14,6 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_official.rst diff --git a/docs/Release_Guide/metplus_bugfix.rst b/docs/Release_Guide/metplus_bugfix.rst index f4635dae4..ecc08da00 100644 --- a/docs/Release_Guide/metplus_bugfix.rst +++ b/docs/Release_Guide/metplus_bugfix.rst @@ -15,4 +15,5 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/metplus/create_release_extra.rst .. include:: release_steps/metplus/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_official.rst diff --git a/docs/Release_Guide/metplus_official.rst b/docs/Release_Guide/metplus_official.rst index 4b996155e..70ab24a19 100644 --- a/docs/Release_Guide/metplus_official.rst +++ b/docs/Release_Guide/metplus_official.rst @@ -23,5 +23,6 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/finalize_release_on_github_official.rst .. include:: release_steps/metplus/update_version_on_develop.rst .. include:: release_steps/update_docs_official.rst +.. include:: release_steps/metplus/add_next_version_to_lookup.rst .. include:: release_steps/metplus/update_web_server_data.rst .. include:: release_steps/set_beta_deletion_reminder_official.rst diff --git a/docs/Release_Guide/metviewer_bugfix.rst b/docs/Release_Guide/metviewer_bugfix.rst index e8f5a35c1..39a1705a0 100644 --- a/docs/Release_Guide/metviewer_bugfix.rst +++ b/docs/Release_Guide/metviewer_bugfix.rst @@ -15,4 +15,5 @@ Create a new vX.Y.Z bugfix release from the main_vX.Y branch. .. include:: release_steps/create_release_on_github.rst .. include:: release_steps/create_release_extra.rst .. include:: release_steps/update_dtc_website.rst +.. include:: release_steps/update_version_lookup.rst .. include:: release_steps/finalize_release_on_github_official.rst diff --git a/docs/Release_Guide/release_steps/metplus/add_next_version_to_lookup.rst b/docs/Release_Guide/release_steps/metplus/add_next_version_to_lookup.rst new file mode 100644 index 000000000..006d56c45 --- /dev/null +++ b/docs/Release_Guide/release_steps/metplus/add_next_version_to_lookup.rst @@ -0,0 +1,38 @@ +Add Next Version to Lookup Table +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In the develop branch, modify the **metplus/component_versions.py** file to +add an entry for the next coordinated release. + +If the X.0 release was just created, add an entry for the X.1 release. + +If the X.1 release was just created, add an entry for the X+1.0 release. + +Set the appropriate X.Y.0 versions for each component. +Maybe sure to set the Z number to 0. + +Set the version for metexpress to None (not a string). + +For example, if the coordinated 6.0 release was just created, add:: + + '6.1': { + 'metplus': '6.1.0', + 'met': '12.1.0', + 'metplotpy': '3.1.0', + 'metcalcpy': '3.1.0', + 'metdataio': '3.1.0', + 'metviewer': '6.1.0', + 'metexpress': None, + }, + +For example, if the coordinated 6.1 release was just created, add:: + + '7.0': { + 'metplus': '7.0.0', + 'met': '13.0.0', + 'metplotpy': '4.0.0', + 'metcalcpy': '4.0.0', + 'metdataio': '4.0.0', + 'metviewer': '7.0.0', + 'metexpress': None, + }, diff --git a/docs/Release_Guide/release_steps/update_version_lookup.rst b/docs/Release_Guide/release_steps/update_version_lookup.rst new file mode 100644 index 000000000..e4561972e --- /dev/null +++ b/docs/Release_Guide/release_steps/update_version_lookup.rst @@ -0,0 +1,46 @@ +Update Version Lookup Table +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Modify the version lookup table in the METplus repository to include the +correct version. + +.. dropdown:: Instructions + + * Clone the METplus repository. + + Using SSH: + + .. parsed-literal:: + + git clone git@github.com:dtcenter/METplus + + Using HTTP: + + .. parsed-literal:: + + git clone https://github.com/dtcenter/METplus + + * Enter the METplus repository directory: + + .. parsed-literal:: + + cd METplus + + * Checkout the develop branch + + .. parsed-literal:: + + git checkout develop + + * Create a branch off of develop to update. + Include the name of the repository and version in the name. + + .. parsed-literal:: + + git checkout -b update_version_|projectRepo|_vX.Y.Z + + * Open **metplus/component_versions.py** and increment the version for the + appropriate |projectRepo| entry. + + * Commit change, push to GitHub, and create a pull request to merge change + into the **develop** branch. From 122142085605802db6b52f76c5bc8854e9db75ff Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:09:00 -0700 Subject: [PATCH 03/18] Per #2562 and #2597, replace automated test logic that used manage_externals to use component version lookup to get versions of METplotpy/METcalcpy/METdataio --- .github/jobs/get_use_case_commands.py | 37 +++++++++++++-------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 721201988..1347c05bf 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -17,6 +17,7 @@ from metplus.util.string_manip import expand_int_string_to_list from docker_utils import VERSION_EXT from metplus import get_metplus_version +from metplus.component_versions import get_component_version # path to METplus install location in Docker METPLUS_DOCKER_LOC = '/metplus/METplus' @@ -76,35 +77,33 @@ def handle_automation_env(host_name, reqs, work_dir): else: py_embed_arg = '' - # get METplus version to determine Externals file to use - # to get METplotpy/METcalcpy/METdataio + # get METplus version to determine which version of + # METplotpy/METcalcpy/METdataio to use # If stable release, get main branch, otherwise get develop - is_stable_release = len(get_metplus_version().split('-')) == 1 - externals_ext = '_stable.cfg' if is_stable_release else '.cfg' + metplus_version = get_metplus_version() # if any metplotpy/metcalcpy keywords are in requirements list, # add command to obtain and install METplotpy and METcalcpy + components = [] if any([item for item in PLOTCALC_KEYWORDS if item in str(reqs).lower()]): - ce_file = os.path.join(work_dir, '.github', 'parm', - f'Externals_metplotcalcpy{externals_ext}') - setup_env.extend(( - f'cd {METPLUS_DOCKER_LOC}', - f'{work_dir}/manage_externals/checkout_externals -e {ce_file}', - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METplotpy', - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METcalcpy', - 'cd -', - )) + components.extend(('METplotpy', 'METcalcpy')) # if metdataio is in requirements list, add command to obtain METdataio if 'metdataio' in str(reqs).lower(): - ce_file = os.path.join(work_dir, '.github', 'parm', - f'Externals_metdataio{externals_ext}') + components.append('METdataio') + + setup_env.append(f'cd {METPLUS_DOCKER_LOC}') + for component in components: + version = get_component_version(input_component='METplus', + input_version=metplus_version, + output_component=component, + output_format='main_v{X}.{Y}', + get_dev=False) setup_env.extend(( - f'cd {METPLUS_DOCKER_LOC}', - f'{work_dir}/manage_externals/checkout_externals -e {ce_file}', - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METdataio', - 'cd -', + f'git clone https://github.com/dtcenter/{component} --branch {version}', + f'{python_path} -m pip install {component}', )) + setup_env.append('cd -') # if metplus is in requirements list, # add top of METplus repo to PYTHONPATH so metplus can be imported From 5439d17a5e9ed08344e8935afdad96f7c070f527 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:09:12 -0700 Subject: [PATCH 04/18] clean up SonarQube complaints --- .github/jobs/get_use_case_commands.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 1347c05bf..c16056839 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -40,7 +40,7 @@ ] -def handle_automation_env(host_name, reqs, work_dir): +def handle_automation_env(host_name, reqs): # if no env is specified, use metplus base environment conda_env = METPLUS_BASE_ENV @@ -117,7 +117,7 @@ def handle_automation_env(host_name, reqs, work_dir): setup_env.extend(( f'echo Using environment: dtcenter/metplus-envs:{conda_env_w_ext}', f'echo cat /usr/local/conda/envs/{conda_env_w_ext}/environments.yml', - f'echo ----------------------------------------', + 'echo ----------------------------------------', f'cat /usr/local/conda/envs/{conda_env_w_ext}/environments.yml', 'echo ----------------------------------------', )) @@ -153,8 +153,7 @@ def main(categories, subset_list, work_dir=None, for use_case_by_requirement in use_cases_by_req: reqs = use_case_by_requirement.requirements - setup_env, py_embed_arg = handle_automation_env(host_name, reqs, - work_dir) + setup_env, py_embed_arg = handle_automation_env(host_name, reqs) # use status variable to track if any use cases failed use_case_cmds = [] From 83afcde83ff933726b3dc8fa9bce391dcd84cca9 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:10:54 -0700 Subject: [PATCH 05/18] turn on some use cases to test changes --- .github/parm/use_case_groups.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 8d12b81db..a06c34825 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -7,7 +7,7 @@ { "category": "met_tool_wrapper", "index_list": "30-58", - "run": false + "run": true }, { "category": "met_tool_wrapper", @@ -112,8 +112,8 @@ }, { "category": "medium_range", - "index_list": "3-5,10", - "run": false + "index_list": "3-5,10", + "run": true }, { "category": "medium_range", From a7cd29c58e61b104ab878e2626459d37db09c126 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:29:02 -0700 Subject: [PATCH 06/18] fix clone --- .github/jobs/get_use_case_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index c16056839..7a245ed45 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -100,7 +100,8 @@ def handle_automation_env(host_name, reqs): output_format='main_v{X}.{Y}', get_dev=False) setup_env.extend(( - f'git clone https://github.com/dtcenter/{component} --branch {version}', + 'git --version', + f'git clone --single-branch --branch {version} https://github.com/dtcenter/{component}', f'{python_path} -m pip install {component}', )) setup_env.append('cd -') From dd0f5b5513c8b9e6fa1e8c6032669a3bf1eb6f4d Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 5 Nov 2024 14:30:37 -0700 Subject: [PATCH 07/18] do not get python dependencies for METplotpy/METcalcpy/METdataio installs --- .github/jobs/get_use_case_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 7a245ed45..515ded156 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -102,7 +102,7 @@ def handle_automation_env(host_name, reqs): setup_env.extend(( 'git --version', f'git clone --single-branch --branch {version} https://github.com/dtcenter/{component}', - f'{python_path} -m pip install {component}', + f'{python_path} -m pip install --no-deps {component}', )) setup_env.append('cd -') From ea2ea6b819640bfe53b5435753ba5fcc061cd82d Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:51:25 -0700 Subject: [PATCH 08/18] use full path to package to import --- .github/jobs/get_use_case_commands.py | 112 +++++++++++++++----------- 1 file changed, 63 insertions(+), 49 deletions(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 515ded156..18f603187 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -102,7 +102,7 @@ def handle_automation_env(host_name, reqs): setup_env.extend(( 'git --version', f'git clone --single-branch --branch {version} https://github.com/dtcenter/{component}', - f'{python_path} -m pip install --no-deps {component}', + f'{python_path} -m pip install --no-deps {METPLUS_DOCKER_LOC}/{component}', )) setup_env.append('cd -') @@ -140,6 +140,24 @@ def main(categories, subset_list, work_dir=None, test_suite = METplusUseCaseSuite() test_suite.add_use_case_groups(categories, subset_list) + for group_name, use_cases_by_req in test_suite.category_groups.items(): + for use_case_by_requirement in use_cases_by_req: + reqs = use_case_by_requirement.requirements + + setup_env, py_embed_arg = handle_automation_env(host_name, reqs) + + use_case_cmds = _get_use_case_cmds(host_name, use_case_by_requirement, work_dir, group_name, py_embed_arg) + + # add commands to set up environment before use case commands + all_commands.append((setup_env, use_case_cmds, reqs)) + + return all_commands + + +def _get_use_case_cmds(host_name, use_case_by_requirement, work_dir, group_name, py_embed_arg): + # use status variable to track if any use cases failed + use_case_cmds = [] + output_top_dir = os.environ.get('METPLUS_TEST_OUTPUT_BASE', '/data/output') # use METPLUS_TEST_SETTINGS_CONF if set @@ -150,57 +168,53 @@ def main(categories, subset_list, work_dir=None, 'parm', 'test_settings.conf') - for group_name, use_cases_by_req in test_suite.category_groups.items(): - for use_case_by_requirement in use_cases_by_req: - reqs = use_case_by_requirement.requirements + if host_name != 'docker': + use_case_cmds.append('status=0') + for use_case in use_case_by_requirement.use_cases: + # add parm/use_cases path to config args if they are conf files + config_args = _get_config_args(use_case.config_args, work_dir, host_name) + + output_base = os.path.join(output_top_dir, + group_name.split('-')[0], + use_case.name) + use_case_cmd = (f"run_metplus.py" + f" {' '.join(config_args)}" + f" {py_embed_arg}{test_settings_conf}" + f" config.OUTPUT_BASE={output_base}") + use_case_cmds.append(use_case_cmd) + # check exit code from use case command and + # set status to non-zero value on error + if host_name != 'docker': + use_case_cmds.append("if [ $? != 0 ]; then status=1; fi") + + # if any use cases failed, force non-zero exit code with false + if host_name != 'docker': + use_case_cmds.append("if [ $status != 0 ]; then false; fi") - setup_env, py_embed_arg = handle_automation_env(host_name, reqs) + return use_case_cmds - # use status variable to track if any use cases failed - use_case_cmds = [] - if host_name != 'docker': - use_case_cmds.append('status=0') - for use_case in use_case_by_requirement.use_cases: - # add parm/use_cases path to config args if they are conf files - config_args = [] - ci_overrides = None - for config_arg in use_case.config_args: - if config_arg.endswith('.conf'): - config_arg = os.path.join(work_dir, 'parm', - 'use_cases', config_arg) - - # look for CI overrides conf file - override_path = os.path.join(config_arg[0:-5], - 'ci_overrides.conf') - if os.path.exists(override_path): - ci_overrides = override_path - - config_args.append(config_arg) - - # add CI overrides config file if running in docker - if ci_overrides and host_name == 'docker': - config_args.append(ci_overrides) - - output_base = os.path.join(output_top_dir, - group_name.split('-')[0], - use_case.name) - use_case_cmd = (f"run_metplus.py" - f" {' '.join(config_args)}" - f" {py_embed_arg}{test_settings_conf}" - f" config.OUTPUT_BASE={output_base}") - use_case_cmds.append(use_case_cmd) - # check exit code from use case command and - # set status to non-zero value on error - if host_name != 'docker': - use_case_cmds.append("if [ $? != 0 ]; then status=1; fi") - - # if any use cases failed, force non-zero exit code with false - if host_name != 'docker': - use_case_cmds.append("if [ $status != 0 ]; then false; fi") - # add commands to set up environment before use case commands - all_commands.append((setup_env, use_case_cmds, reqs)) - return all_commands +def _get_config_args(input_config_args, work_dir, host_name): + config_args = [] + ci_overrides = None + for config_arg in input_config_args: + if config_arg.endswith('.conf'): + config_arg = os.path.join(work_dir, 'parm', + 'use_cases', config_arg) + + # look for CI overrides conf file + override_path = os.path.join(config_arg[0:-5], + 'ci_overrides.conf') + if os.path.exists(override_path): + ci_overrides = override_path + + config_args.append(config_arg) + + # add CI overrides config file if running in docker + if ci_overrides and host_name == 'docker': + config_args.append(ci_overrides) + + return config_args def handle_command_line_args(): From 67ddee609120669206b6e1a62338c3d7198c9041 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:03:18 -0700 Subject: [PATCH 09/18] clone METplus Analysis repos on the same level as METplus to match previous location using manage_externals, remove manage_externals config files that are no longer used by the automated tests --- .github/jobs/get_use_case_commands.py | 4 ++-- .github/parm/Externals_metdataio.cfg | 9 --------- .github/parm/Externals_metdataio_stable.cfg | 9 --------- .github/parm/Externals_metplotcalcpy.cfg | 16 ---------------- .github/parm/Externals_metplotcalcpy_stable.cfg | 16 ---------------- 5 files changed, 2 insertions(+), 52 deletions(-) delete mode 100644 .github/parm/Externals_metdataio.cfg delete mode 100644 .github/parm/Externals_metdataio_stable.cfg delete mode 100644 .github/parm/Externals_metplotcalcpy.cfg delete mode 100644 .github/parm/Externals_metplotcalcpy_stable.cfg diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 18f603187..f8e1152b5 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -92,7 +92,7 @@ def handle_automation_env(host_name, reqs): if 'metdataio' in str(reqs).lower(): components.append('METdataio') - setup_env.append(f'cd {METPLUS_DOCKER_LOC}') + setup_env.append(f'cd {METPLUS_DOCKER_LOC}/..') for component in components: version = get_component_version(input_component='METplus', input_version=metplus_version, @@ -102,7 +102,7 @@ def handle_automation_env(host_name, reqs): setup_env.extend(( 'git --version', f'git clone --single-branch --branch {version} https://github.com/dtcenter/{component}', - f'{python_path} -m pip install --no-deps {METPLUS_DOCKER_LOC}/{component}', + f'{python_path} -m pip install --no-deps {METPLUS_DOCKER_LOC}/../{component}', )) setup_env.append('cd -') diff --git a/.github/parm/Externals_metdataio.cfg b/.github/parm/Externals_metdataio.cfg deleted file mode 100644 index b7f7544f8..000000000 --- a/.github/parm/Externals_metdataio.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[METdataio] -local_path = ../METdataio -protocol = git -required = True -repo_url = https://github.com/dtcenter/METdataio -branch = develop - -[externals_description] -schema_version = 1.0.0 diff --git a/.github/parm/Externals_metdataio_stable.cfg b/.github/parm/Externals_metdataio_stable.cfg deleted file mode 100644 index af4540233..000000000 --- a/.github/parm/Externals_metdataio_stable.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[METdataio] -local_path = ../METdataio -protocol = git -required = True -repo_url = https://github.com/dtcenter/METdataio -branch = main_v2.1 - -[externals_description] -schema_version = 1.0.0 diff --git a/.github/parm/Externals_metplotcalcpy.cfg b/.github/parm/Externals_metplotcalcpy.cfg deleted file mode 100644 index fdf138355..000000000 --- a/.github/parm/Externals_metplotcalcpy.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[METcalcpy] -local_path = ../METcalcpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METcalcpy -branch = develop - -[METplotpy] -local_path = ../METplotpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METplotpy -branch = develop - -[externals_description] -schema_version = 1.0.0 diff --git a/.github/parm/Externals_metplotcalcpy_stable.cfg b/.github/parm/Externals_metplotcalcpy_stable.cfg deleted file mode 100644 index 5aed1cb69..000000000 --- a/.github/parm/Externals_metplotcalcpy_stable.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[METcalcpy] -local_path = ../METcalcpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METcalcpy -branch = main_v2.1 - -[METplotpy] -local_path = ../METplotpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METplotpy -branch = main_v2.1 - -[externals_description] -schema_version = 1.0.0 From aa1f031711dc34c0433d72ca6781d6d3f6f247aa Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:21:35 -0700 Subject: [PATCH 10/18] clean up linter complaints --- .github/jobs/get_use_case_commands.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index f8e1152b5..1c69828aa 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -61,8 +61,9 @@ def handle_automation_env(host_name, reqs): conda_env_w_ext = f'{conda_env}{VERSION_EXT}' # start building commands to run before run_metplus.py in Docker - setup_env = [] - setup_env.append(_add_to_bashrc('# BELOW WAS ADDED BY TEST SCRIPT')) + setup_env = [ + _add_to_bashrc('# BELOW WAS ADDED BY TEST SCRIPT') + ] # add conda bin to beginning of PATH python_dir = os.path.join('/usr', 'local', 'conda', 'envs', @@ -245,9 +246,9 @@ def handle_command_line_args(): if __name__ == '__main__': - categories, subset_list, _ = handle_command_line_args() - all_commands = main(categories, subset_list) - for setup_commands, use_case_commands, requirements in all_commands: + input_categories, input_subset_list, _ = handle_command_line_args() + commands = main(input_categories, input_subset_list) + for setup_commands, use_case_commands, requirements in commands: print(f"REQUIREMENTS: {','.join(requirements)}") if setup_commands: command_format = ';\\\n'.join(setup_commands.split(';')) From b5d140928609e9fa5e84b2d9a3da06adade7213c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 6 Nov 2024 09:21:46 -0700 Subject: [PATCH 11/18] turn off use cases after confirming that they run successfully --- .github/parm/use_case_groups.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index a06c34825..31f9bc4aa 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -7,7 +7,7 @@ { "category": "met_tool_wrapper", "index_list": "30-58", - "run": true + "run": false }, { "category": "met_tool_wrapper", @@ -113,7 +113,7 @@ { "category": "medium_range", "index_list": "3-5,10", - "run": true + "run": false }, { "category": "medium_range", From e00c62eb4f24bc447e7fcec179ae47fe79f20402 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:33:27 -0700 Subject: [PATCH 12/18] clean up usage statement formatting --- metplus/component_versions.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/metplus/component_versions.py b/metplus/component_versions.py index 97275e519..be8d2c406 100755 --- a/metplus/component_versions.py +++ b/metplus/component_versions.py @@ -100,12 +100,12 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('-i', '--input_component', default='metplus', - help='Name of METplus component to use to find version,' - ' default is METplus.') + help='Name of METplus component to use to find version.' + ' Default is METplus.') parser.add_argument('-v', '--input_version', default=LATEST_OFFICIAL_RELEASE, - help='version of input_component to search,' - ' default is latest official release') + help='version of input_component to search.' + ' Default is latest official release') parser.add_argument('-o', '--output_component', required=True, help='name of METplus component to obtain version') parser.add_argument('-f', '--output_format', @@ -114,7 +114,8 @@ def main(): '{X}, {Y}, and {Z} will be replaced with x, y, and' ' z version numbers from X.Y.Z. {N} will be ' 'replaced with development version if found in the' - 'input version, e.g. "-beta3" or "-rc1"') + ' input version, e.g. "-beta3" or "-rc1".' + ' Default is v{X}.{Y}.{Z}{N}') parser.add_argument('--get_dev_version', action=argparse.BooleanOptionalAction, default=True, help='If True, get corresponding -beta or -rc version. ' From debc3e9996d2197c6a19f904f73419741cf955b4 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:34:01 -0700 Subject: [PATCH 13/18] remove info about updating manage_externals config files used in automated tests that have been removed --- .../metplus/update_manage_externals.rst | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst b/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst index adc564a51..d063602d3 100644 --- a/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst +++ b/docs/Release_Guide/release_steps/metplus/update_manage_externals.rst @@ -43,18 +43,3 @@ Update build_components/Externals_stable.cfg Ensure the *tag* for each component is correct. It should match the format **vX.Y.Z** where X.Y.Z is the version of that component. For example, MET should be **v11.0.0** for METplus 5.0.0. - - -Update .github/parm/Externals_metdataio_stable.cfg -"""""""""""""""""""""""""""""""""""""""""""""""""" - -Ensure the *branch* value is correct. It should match the format -**main_vX.Y** where X.Y is the version of that component. -For example, METdataio should be **main_v2.0** for METplus 5.0.0. - -Update .github/parm/Externals_metplotcalcpy_stable.cfg -"""""""""""""""""""""""""""""""""""""""""""""""""""""" - -Ensure the *branch* for each component is correct. It should match the format -**main_vX.Y** where X.Y is the version of that component. -For example, METplotpy and METcalcpy should be **main_v2.0** for METplus 5.0.0. From b9ce42732fe8c84831c3be6b2c81d31979f1c62c Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:35:52 -0700 Subject: [PATCH 14/18] added description of component versions script to Contributor's Guide and added stubs for other utility descriptions --- docs/Contributors_Guide/index.rst | 1 + docs/Contributors_Guide/utilities.rst | 189 ++++++++++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 docs/Contributors_Guide/utilities.rst diff --git a/docs/Contributors_Guide/index.rst b/docs/Contributors_Guide/index.rst index 2dabc2a16..fb321c1f7 100644 --- a/docs/Contributors_Guide/index.rst +++ b/docs/Contributors_Guide/index.rst @@ -7,6 +7,7 @@ Contributor's Guide :numbered: coding_standards + utilities basic_components create_wrapper conda_env diff --git a/docs/Contributors_Guide/utilities.rst b/docs/Contributors_Guide/utilities.rst new file mode 100644 index 000000000..5950f4ff0 --- /dev/null +++ b/docs/Contributors_Guide/utilities.rst @@ -0,0 +1,189 @@ +.. _cg_util: + +********* +Utilities +********* + +.. _cg_util_version: + +Component Versions Script +========================= + +**metplus/component_versions.py** + +This script is used to query a METplus coordinated release component version +number lookup table to determine a corresponding version number for another +METplus component. + +This script can be called directly from a script or the command line. +It returns the version of the requested (output) METplus component. +This functionality can also be imported in a Python script. +See below for examples. + +Usage Statement +--------------- +:: + + usage: component_versions.py [-h] [-i INPUT_COMPONENT] [-v INPUT_VERSION] -o + OUTPUT_COMPONENT [-f OUTPUT_FORMAT] + [--get_dev_version | --no-get_dev_version] + + options: + -h, --help show this help message and exit + -i INPUT_COMPONENT, --input_component INPUT_COMPONENT + Name of METplus component to use to find version. Default + is METplus. + -v INPUT_VERSION, --input_version INPUT_VERSION + version of input_component to search. Default is latest + official release + -o OUTPUT_COMPONENT, --output_component OUTPUT_COMPONENT + name of METplus component to obtain version + -f OUTPUT_FORMAT, --output_format OUTPUT_FORMAT + format to use to output version number.{X}, {Y}, and {Z} + will be replaced with x, y, and z version numbers from + X.Y.Z. {N} will be replaced with development version if + found in the input version, e.g. "-beta3" or "-rc1" + Default is v{X}.{Y}.{Z}{N} + --get_dev_version, --no-get_dev_version + If True, get corresponding -beta or -rc version. If + False, return develop if development version. (default: + True) + +Examples +-------- + +These examples do not include the full path to the script. + +**Get MET vX.Y.Z version from coordinated release**:: + + component_versions.py -v 5.1 -o MET + +v11.1.1 + +**Get MET vX.Y.Z version from METplus release**:: + + component_versions.py -v 5.1.0 -o MET + +v11.1.1 + +**Get MET vX.Y.Z development version from beta release**:: + + component_versions.py -v 6.0.0-beta3 -o MET + +v12.0.0-beta3 + +**Get MET vX.Y.Z development version from -dev version (always returns develop)**:: + + component_versions.py -v 6.0.0-beta3-dev -o MET + +develop + +**Get MET vX.Y.Z development version from beta version (returns develop because 'get dev version' is set to False)**:: + + component_versions.py -v 6.0.0-beta3 -o MET --no-get_dev_version + +develop + +**Get MET main_vX.Y version from METplus release (format specified)**:: + + component_versions.py -v 5.1.0 -o MET -f main_v{X}.{Y} + +main_v11.1 + +**Get METplotpy main_vX.Y version from METviewer release**:: + + component_versions.py -i METviewer -v 5.1.0 -o METplotpy -f main_v{X}.{Y} + +main_v2.1 + +**Get METplotpy main_vX.Y version from METviewer main_vX.Y branch**:: + + component_versions.py -i METviewer -v main_v5.1 -o METplotpy -f main_v{X}.{Y} + +main_v2.1 + +**Using Python Function**:: + + >>> from metplus.component_versions import get_component_version + >>> version = get_component_version(input_component='METplus', + input_version='6.0.0', + output_component='MET', + output_format='main_v{X}.{Y}', + get_dev=False) + >>> print(version) + +main_v12.0 + +METplus Utils +============= + +These files are found under **metplus/util**. + +Utility scripts used by the METplus Wrappers. + +**MORE INFO COMING SOON** + +config_metplus.py +----------------- + +config_util.py +-------------- + +config_validate.py +------------------ + +constants.py +------------ + +diff_util.py +------------ + +field_util.py +------------- + +met_config.py +------------- + +metplus_check.py +---------------- + +run_util.py +----------- + +string_manip.py +--------------- + +string_template_substitution.py +------------------------------- + +system_util.py +-------------- + +time_looping.py +--------------- + +time_util.py +------------ + +wrapper_init.py +--------------- + +Internal Development Tools +========================== + +These utilities scripts can be found in **internal/scripts/dev_tools**. +They were written to assist with common development tasks. + +Add MET Config Helper +--------------------- + +**internal/scripts/dev_tools/add_met_config_helper.py** + +**MORE INFO COMING SOON** + +Generate Release Notes +---------------------- + +**internal/scripts/dev_tools/generate_release_notes.py** + +**MORE INFO COMING SOON** From fc6c9688b0e0437bed8dab3cc66417313cc821ec Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:57:02 -0700 Subject: [PATCH 15/18] change format of example results and add text to describe what is being demonstrated from each example --- docs/Contributors_Guide/utilities.rst | 84 ++++++++++++++++++--------- 1 file changed, 58 insertions(+), 26 deletions(-) diff --git a/docs/Contributors_Guide/utilities.rst b/docs/Contributors_Guide/utilities.rst index 5950f4ff0..1219dce31 100644 --- a/docs/Contributors_Guide/utilities.rst +++ b/docs/Contributors_Guide/utilities.rst @@ -54,55 +54,87 @@ Examples These examples do not include the full path to the script. -**Get MET vX.Y.Z version from coordinated release**:: +Get MET vX.Y.Z version from METplus release +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: - component_versions.py -v 5.1 -o MET + $ component_versions.py -v 5.1.0 -o MET + v11.1.1 -v11.1.1 +If the input component is not specified, it will use the METplus version. +The default output format is v{X}.{Y}.{Z}{N}. -**Get MET vX.Y.Z version from METplus release**:: +Get MET vX.Y.Z version from coordinated release +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: - component_versions.py -v 5.1.0 -o MET + $ component_versions.py -v 5.1 -o MET + v11.1.1 -v11.1.1 +The coordinated release version matches the METplus X.Y version, +so the coordinated release version can also be used as the input version. -**Get MET vX.Y.Z development version from beta release**:: +Get MET vX.Y.Z development version from beta release +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: - component_versions.py -v 6.0.0-beta3 -o MET + $ component_versions.py -v 6.0.0-beta3 -o MET + v12.0.0-beta3 -v12.0.0-beta3 +If a beta release version is provided as the input, the output will include +the same beta version. -**Get MET vX.Y.Z development version from -dev version (always returns develop)**:: +Get MET vX.Y.Z development version from -dev version +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: - component_versions.py -v 6.0.0-beta3-dev -o MET + $ component_versions.py -v 6.0.0-beta3-dev -o MET + develop -develop +If the input version includes -dev, the result will always be *develop*. -**Get MET vX.Y.Z development version from beta version (returns develop because 'get dev version' is set to False)**:: +Get MET vX.Y.Z development version from beta version +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - component_versions.py -v 6.0.0-beta3 -o MET --no-get_dev_version + $ component_versions.py -v 6.0.0-beta3 -o MET --no-get_dev_version + develop -develop +If the *--no-get_dev_version* argument is provided, +an input version that includes -betaN or -rcN will return *develop* -**Get MET main_vX.Y version from METplus release (format specified)**:: +Get MET main_vX.Y version from METplus release +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: - component_versions.py -v 5.1.0 -o MET -f main_v{X}.{Y} + $ component_versions.py -v 5.1.0 -o MET -f main_v{X}.{Y} + main_v11.1 -main_v11.1 +The output format can be specified using the *-f* argument. +{X}, {Y}, {Z}, and {N} will be substituted with values based on the input. -**Get METplotpy main_vX.Y version from METviewer release**:: +Get METplotpy main_vX.Y version from METviewer release +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: - component_versions.py -i METviewer -v 5.1.0 -o METplotpy -f main_v{X}.{Y} + $ component_versions.py -i METviewer -v 5.1.0 -o METplotpy -f main_v{X}.{Y} + main_v2.1 -main_v2.1 +The *-i* argument can be used to specify the input component that corresponds +to the input version number. -**Get METplotpy main_vX.Y version from METviewer main_vX.Y branch**:: +Get METplotpy main_vX.Y version from METviewer main_vX.Y branch +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: - component_versions.py -i METviewer -v main_v5.1 -o METplotpy -f main_v{X}.{Y} + $ component_versions.py -i METviewer -v main_v5.1 -o METplotpy -f main_v{X}.{Y} + main_v2.1 -main_v2.1 +The input version number can be provided in different formats, +including **main_vX.Y** and **vX.Y.Z**. -**Using Python Function**:: +Using Python Function +^^^^^^^^^^^^^^^^^^^^^ +:: >>> from metplus.component_versions import get_component_version >>> version = get_component_version(input_component='METplus', @@ -111,8 +143,8 @@ main_v2.1 output_format='main_v{X}.{Y}', get_dev=False) >>> print(version) + main_v12.0 -main_v12.0 METplus Utils ============= From cff594cfc9fb678b81153c959c4c4cbb7acf9717 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 6 Nov 2024 12:05:21 -0700 Subject: [PATCH 16/18] fix formatting --- docs/Contributors_Guide/utilities.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/Contributors_Guide/utilities.rst b/docs/Contributors_Guide/utilities.rst index 1219dce31..ffca73ce3 100644 --- a/docs/Contributors_Guide/utilities.rst +++ b/docs/Contributors_Guide/utilities.rst @@ -95,6 +95,7 @@ If the input version includes -dev, the result will always be *develop*. Get MET vX.Y.Z development version from beta version ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: $ component_versions.py -v 6.0.0-beta3 -o MET --no-get_dev_version develop From efeee53d469fa2f58ed261c9b522906fa557e6e4 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 6 Nov 2024 12:19:18 -0700 Subject: [PATCH 17/18] fixed bad variable replacement --- docs/Release_Guide/release_steps/update_version_lookup.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Release_Guide/release_steps/update_version_lookup.rst b/docs/Release_Guide/release_steps/update_version_lookup.rst index e4561972e..d465b1ee2 100644 --- a/docs/Release_Guide/release_steps/update_version_lookup.rst +++ b/docs/Release_Guide/release_steps/update_version_lookup.rst @@ -35,9 +35,9 @@ correct version. * Create a branch off of develop to update. Include the name of the repository and version in the name. - .. parsed-literal:: + .. parsed-literal:: - git checkout -b update_version_|projectRepo|_vX.Y.Z + git checkout -b update_version_vX.Y.Z_repo * Open **metplus/component_versions.py** and increment the version for the appropriate |projectRepo| entry. From 138289e235d1039606de5c7d7cc72c4f18d7d84d Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Wed, 6 Nov 2024 16:54:04 -0700 Subject: [PATCH 18/18] Update docs/Contributors_Guide/utilities.rst --- docs/Contributors_Guide/utilities.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Contributors_Guide/utilities.rst b/docs/Contributors_Guide/utilities.rst index ffca73ce3..9d576142a 100644 --- a/docs/Contributors_Guide/utilities.rst +++ b/docs/Contributors_Guide/utilities.rst @@ -101,7 +101,7 @@ Get MET vX.Y.Z development version from beta version develop If the *--no-get_dev_version* argument is provided, -an input version that includes -betaN or -rcN will return *develop* +an input version that includes -betaN or -rcN will return *develop*. Get MET main_vX.Y version from METplus release ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^