From ce166edcc77c032e3a305361eba4ebd81f0a9a0f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:26:55 -0600 Subject: [PATCH 1/3] fix SonarQube complaints --- .../test_string_template_substitution.py | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py b/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py index dfffe82f22..79ea282ac9 100644 --- a/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py +++ b/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py @@ -354,7 +354,7 @@ def test_filename_does_not_match_template(): template = "{init?fmt=%Y%m%d%H}_dog_A{lead?fmt=%HH}h" filepath = "1987020103_cat_A03h" out = parse_template(template, filepath) - assert out == None + assert out is None @pytest.mark.util @@ -362,7 +362,7 @@ def test_filename_does_not_match_template_end(): template = "{init?fmt=%Y%m%d%H}_dog_A{lead?fmt=%HH}h" filepath = "1987020103_dog_A03d" out = parse_template(template, filepath) - assert out == None + assert out is None @pytest.mark.util @@ -379,7 +379,7 @@ def test_get_tags(): # value is the formatted output string, like 01 # ttype is the unit to check, i.e. 'H', 'M', 'S', 'd', 's' @pytest.mark.parametrize( - 'format, key, value, ttype', [ + 'fmt, key, value, ttype', [ ('H', 1, '01', 'H'), ('1H', 1, '1', 'H'), ('2H', 1, '01', 'H'), @@ -418,15 +418,15 @@ def test_get_tags(): ] ) @pytest.mark.util -def test_format_one_time_item(format, key ,value, ttype): - assert format_one_time_item(format, key, ttype) == value +def test_format_one_time_item(fmt, key ,value, ttype): + assert format_one_time_item(fmt, key, ttype) == value # format is the time format to use, like, %M or %H%M # seconds is the integer number of seconds of the offset to use, i.e. 3601 # value is the formatted output string, like 010001 @pytest.mark.parametrize( - 'format, seconds, value', [ + 'fmt, seconds, value', [ ('%H', 1, '00'), ('%M', 1, '00'), ('%S', 1, '01'), @@ -448,9 +448,9 @@ def test_format_one_time_item(format, key ,value, ttype): ] ) @pytest.mark.util -def test_format_hms(format, seconds, value): +def test_format_hms(fmt, seconds, value): # format should be something like %M or %H%M - assert format_hms(format, seconds == value) + assert format_hms(fmt, seconds == value) @pytest.mark.util @@ -539,7 +539,6 @@ def test_populate_match_dict(template, filepath, expected_match_dict, expected_v elif match_dict is None: # if expected is not None, fail if actual is None assert False - return num_keys = len(match_dict.keys()) expected_num_keys = len(expected_match_dict.keys()) @@ -562,8 +561,6 @@ def test_populate_match_dict(template, filepath, expected_match_dict, expected_v print(f"Incorrect valid shift. Actual {valid_shift}, Expected: {expected_valid_shift}") assert False - assert True - except TypeError: assert expected_match_dict is None and expected_valid_shift is None @@ -591,8 +588,6 @@ def test_get_fmt_info(fmt, filepath, identifier, expected_fmt_len, expected_matc print(f"Expected Dictionary: {expected_match_dict}") assert False - assert True - @pytest.mark.parametrize( 'templ, expected_filename', [ From 140647b839bf43f796056396b57a566e5f70e118 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:42:00 -0600 Subject: [PATCH 2/3] Per #2705, set correct time information in field info by using the time info used to find the input files (self.c_dict['ALL_FILES']) instead of relying on parsing the time info from the other input files (fcst vs. obs) --- metplus/wrappers/series_analysis_wrapper.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/metplus/wrappers/series_analysis_wrapper.py b/metplus/wrappers/series_analysis_wrapper.py index cea1a44b4a..0aa1e09da6 100755 --- a/metplus/wrappers/series_analysis_wrapper.py +++ b/metplus/wrappers/series_analysis_wrapper.py @@ -1047,10 +1047,10 @@ def get_formatted_fields(self, var_info, time_info, fcst_path, obs_path): observation information or (None, None) if something went wrong """ fcst_field_list = ( - self._get_field_list('fcst', var_info, time_info, obs_path) + self._get_field_list('fcst', var_info, time_info) ) obs_field_list = ( - self._get_field_list('obs', var_info, time_info, fcst_path) + self._get_field_list('obs', var_info, time_info) ) if not fcst_field_list or not obs_field_list: @@ -1062,7 +1062,7 @@ def get_formatted_fields(self, var_info, time_info, fcst_path, obs_path): return fcst_fields, obs_fields - def _get_field_list(self, data_type, var_info, time_info, file_list_path): + def _get_field_list(self, data_type, var_info, time_info): """!Get formatted field information in a list. If no time (init/valid/lead) filename template tags were found in the level value or if the time info contains all init/valid/lead values @@ -1075,7 +1075,6 @@ def _get_field_list(self, data_type, var_info, time_info, file_list_path): @param data_type type of data to process, e.g. fcst or obs @param var_info dictionary containing info to format @param time_info dictionary containing time information - @param file_list_path path to file list file to parse @returns list containing formatted field info to pass to MET config """ other = 'OBS' if data_type == 'fcst' else 'FCST' @@ -1095,8 +1094,8 @@ def _get_field_list(self, data_type, var_info, time_info, file_list_path): # loop through fcst/obs files to extract time info # for each file apply time info to field info and add to list - for file_time_info in self._get_times_from_file_list(file_list_path, - templates): + for file_dict in self.c_dict['ALL_FILES']: + file_time_info = file_dict['time_info'] field = self._get_field_sub_level(data_type, var_info, file_time_info) if field: field_list.extend(field) From ff5dfc622000105e56298c4f0a6b302c5cef454f Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Mon, 30 Sep 2024 11:39:18 -0600 Subject: [PATCH 3/3] clean up unused code after bugfix for #2705 --- metplus/wrappers/series_analysis_wrapper.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/metplus/wrappers/series_analysis_wrapper.py b/metplus/wrappers/series_analysis_wrapper.py index 0aa1e09da6..1fb1948a6b 100755 --- a/metplus/wrappers/series_analysis_wrapper.py +++ b/metplus/wrappers/series_analysis_wrapper.py @@ -784,9 +784,7 @@ def build_and_run_series_request(self, time_info, fcst_path, obs_path): add_field_info_to_time_info(time_info, var_info) # get formatted field dictionary to pass into the MET config file - fcst_field, obs_field = ( - self.get_formatted_fields(var_info, time_info, fcst_path, obs_path) - ) + fcst_field, obs_field = self.get_formatted_fields(var_info, time_info) if fcst_field is None: continue @@ -1035,14 +1033,12 @@ def _get_netcdf_min_max(filepath, variable_name): except (FileNotFoundError, KeyError): return None, None - def get_formatted_fields(self, var_info, time_info, fcst_path, obs_path): + def get_formatted_fields(self, var_info, time_info): """! Get forecast and observation field information for var_info and format it so it can be passed into the MET config file @param var_info dictionary containing info to format @param time_info dictionary containing time information - @param fcst_path path to file list file for forecast data - @param obs_path path to file list file for observation data @returns tuple containing strings of the formatted forecast and observation information or (None, None) if something went wrong """ @@ -1077,7 +1073,6 @@ def _get_field_list(self, data_type, var_info, time_info): @param time_info dictionary containing time information @returns list containing formatted field info to pass to MET config """ - other = 'OBS' if data_type == 'fcst' else 'FCST' # if there are no time tags (init/valid/lead) in the field level # or if init, valid, and lead have values in time_info, # get field info for a single field to pass to the MET config file @@ -1087,12 +1082,7 @@ def _get_field_list(self, data_type, var_info, time_info): field_list = [] - # handle multiple templates - templates = [] - for template in self.c_dict[f'{other}_INPUT_TEMPLATE'].split(','): - templates.append(os.path.join(self.c_dict[f'{other}_INPUT_DIR'], template.strip())) - - # loop through fcst/obs files to extract time info + # loop through fcst/obs files to read time info # for each file apply time info to field info and add to list for file_dict in self.c_dict['ALL_FILES']: file_time_info = file_dict['time_info']