bugfix 1555 develop clear args properly (#1562)

internal_tests/pytests/gen_ens_prod/test_gen_ens_prod_wrapper.py

@@ -34,7 +34,7 @@ def set_minimum_config_settings(config):
     config.set('config', 'INIT_BEG', run_times[0])
     config.set('config', 'INIT_END', run_times[-1])
     config.set('config', 'INIT_INCREMENT', '6H')
-    config.set('config', 'LEAD_SEQ', '24H')
+    config.set('config', 'LEAD_SEQ', '24H, 48H')
     config.set('config', 'LOOP_ORDER', 'times')
     config.set('config', 'GEN_ENS_PROD_CONFIG_FILE',
                '{PARM_BASE}/met_config/GenEnsProdConfig_wrapped')
@@ -385,19 +385,32 @@ def test_gen_ens_prod_single_field(metplus_config, config_overrides,
                                  'file_lists')
     config_file = wrapper.c_dict.get('CONFIG_FILE')
     out_dir = wrapper.c_dict.get('OUTPUT_DIR')
-    expected_cmds = [(f"{app_path} {verbosity} -ens "
-                      f"{file_list_dir}/20091231120000_24_gen_ens_prod.txt "
-                      "-out "
-                      f"{out_dir}/gen_ens_prod_20100101_120000V_ens.nc "
-                      f"-config {config_file} -ctrl "
-                      f"{input_dir}/2009123112/arw-tom-gep3/d01_2009123112_02400.grib"),
-                     (f"{app_path} {verbosity} -ens "
-                      f"{file_list_dir}/20091231180000_24_gen_ens_prod.txt "
-                      "-out "
-                      f"{out_dir}/gen_ens_prod_20100101_180000V_ens.nc "
-                      f"-config {config_file} -ctrl "
-                      f"{input_dir}/2009123118/arw-tom-gep3/d01_2009123118_02400.grib"),
-                     ]
+    expected_cmds = [
+        (f"{app_path} {verbosity} -ens "
+         f"{file_list_dir}/20091231120000_24_gen_ens_prod.txt "
+         "-out "
+         f"{out_dir}/gen_ens_prod_20100101_120000V_ens.nc "
+         f"-config {config_file} -ctrl "
+         f"{input_dir}/2009123112/arw-tom-gep3/d01_2009123112_02400.grib"),
+        (f"{app_path} {verbosity} -ens "
+         f"{file_list_dir}/20091231120000_48_gen_ens_prod.txt "
+         "-out "
+         f"{out_dir}/gen_ens_prod_20100102_120000V_ens.nc "
+         f"-config {config_file} -ctrl "
+         f"{input_dir}/2009123112/arw-tom-gep3/d01_2009123112_04800.grib"),
+        (f"{app_path} {verbosity} -ens "
+         f"{file_list_dir}/20091231180000_24_gen_ens_prod.txt "
+         "-out "
+         f"{out_dir}/gen_ens_prod_20100101_180000V_ens.nc "
+         f"-config {config_file} -ctrl "
+         f"{input_dir}/2009123118/arw-tom-gep3/d01_2009123118_02400.grib"),
+        (f"{app_path} {verbosity} -ens "
+         f"{file_list_dir}/20091231180000_48_gen_ens_prod.txt "
+         "-out "
+         f"{out_dir}/gen_ens_prod_20100102_180000V_ens.nc "
+         f"-config {config_file} -ctrl "
+         f"{input_dir}/2009123118/arw-tom-gep3/d01_2009123118_04800.grib"),
+    ]
 
     all_cmds = wrapper.run_all_times()
     print(f"ALL COMMANDS: {all_cmds}")
@@ -472,7 +485,7 @@ def test_gen_ens_prod_fill_missing(metplus_config, config_overrides,
         os.remove(file_list_file)
 
     all_cmds = wrapper.run_all_times()
-    assert len(all_cmds) == 1
+    assert len(all_cmds) == 2
 
     with open(file_list_file, 'r') as file_handle:
         actual_num_files = len(file_handle.read().splitlines()) - 1

internal_tests/pytests/tc_gen/test_tc_gen_wrapper.py

@@ -329,11 +329,14 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values):
     config.set('config', 'TC_GEN_SHAPE_INPUT_DIR', shape_dir)
     config.set('config', 'TC_GEN_SHAPE_INPUT_TEMPLATE', shape_template)
     config.set('config', 'TC_GEN_OUTPUT_DIR', '{OUTPUT_BASE}/TCGen/output')
-    config.set('config', 'TC_GEN_OUTPUT_TEMPLATE', 'tc_gen_{init?fmt=%Y}')
+    config.set('config', 'TC_GEN_OUTPUT_TEMPLATE',
+               'tc_gen_{custom}_{init?fmt=%Y}')
 
     config.set('config', 'TC_GEN_CONFIG_FILE',
                '{PARM_BASE}/met_config/TCGenConfig_wrapped')
 
+    config.set('config', 'TC_GEN_CUSTOM_LOOP_LIST', 'a, b')
+
     # set config variable overrides
     for key, value in config_overrides.items():
         config.set('config', key, value)
@@ -368,7 +371,13 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values):
          f"-edeck {edeck_path} "
          f"-shape {shape_path} "
          f"-track {track_path} "
-         f"-config {config_file} -out {out_dir}/tc_gen_2016"),
+         f"-config {config_file} -out {out_dir}/tc_gen_a_2016"),
+        (f"{app_path} {verbosity} "
+         f"-genesis {genesis_path} "
+         f"-edeck {edeck_path} "
+         f"-shape {shape_path} "
+         f"-track {track_path} "
+         f"-config {config_file} -out {out_dir}/tc_gen_b_2016"),
     ]
 
     all_cmds = wrapper.run_all_times()

internal_tests/pytests/user_script/test_user_script.py

@@ -321,6 +321,30 @@ def set_run_type_info(config, run_type):
           'echo init_20141026093015_valid_20141101093015_lead_144.nc',
           'echo init_20141025093015_valid_20141101093015_lead_168.nc',
           ]),
+        # run once custom loop list
+        ({'USER_SCRIPT_RUNTIME_FREQ': 'RUN_ONCE',
+          'USER_SCRIPT_COMMAND': 'echo {custom}',
+          'USER_SCRIPT_CUSTOM_LOOP_LIST': 'a,b'},
+         None,
+         ['echo a', 'echo b']),
+        # run per valid custom loop list
+        ({'USER_SCRIPT_RUNTIME_FREQ': 'RUN_ONCE_PER_INIT_OR_VALID',
+          'USER_SCRIPT_COMMAND': 'echo {custom}',
+          'USER_SCRIPT_CUSTOM_LOOP_LIST': 'a,b'},
+         ['VALID'],
+         ['echo a'] * 3 + ['echo b'] * 3),
+        # run per init custom loop list
+        ({'USER_SCRIPT_RUNTIME_FREQ': 'RUN_ONCE_PER_INIT_OR_VALID',
+          'USER_SCRIPT_COMMAND': 'echo {custom}',
+          'USER_SCRIPT_CUSTOM_LOOP_LIST': 'a,b'},
+         ['INIT'],
+         ['echo a'] * 3 + ['echo b'] * 3),
+        # run all init/lead sequence - simple
+        ({'USER_SCRIPT_RUNTIME_FREQ': 'RUN_ONCE_FOR_EACH',
+          'USER_SCRIPT_COMMAND': 'echo {custom}',
+          'USER_SCRIPT_CUSTOM_LOOP_LIST': 'a,b'},
+         ['LEAD_SEQ', 'INIT'],
+         ['echo a'] * 12 + ['echo b'] * 12),
     ]
 )
 def test_run_user_script_all_times(metplus_config, input_configs,
@@ -340,7 +364,7 @@ def test_run_user_script_all_times(metplus_config, input_configs,
     all_commands = wrapper.run_all_times()
 
     if not all_commands:
-        assert(False)
+        assert False
 
     clock_time = datetime.strptime(config.getstr('config', 'CLOCK_TIME'),
                                    '%Y%m%d%H%M%S')
@@ -349,4 +373,4 @@ def test_run_user_script_all_times(metplus_config, input_configs,
         expected_cmd = sub_clock_time(expected_cmd, clock_time)
         print(f"  ACTUAL:{actual_cmd}")
         print(f"EXPECTED:{expected_cmd}")
-        assert(actual_cmd == expected_cmd)
+        assert actual_cmd == expected_cmd

metplus/util/met_util.py

@@ -388,11 +388,12 @@ def is_loop_by_init(config):
 
     return None
 
-def loop_over_times_and_call(config, processes):
+def loop_over_times_and_call(config, processes, custom=None):
     """! Loop over all run times and call wrappers listed in config
 
     @param config METplusConfig object
     @param processes list of CommandBuilder subclass objects (Wrappers) to call
+    @param custom (optional) custom loop string value
     @returns list of tuples with all commands run and the environment variables
     that were set for each
     """
@@ -410,7 +411,8 @@ def loop_over_times_and_call(config, processes):
 
             log_runtime_banner(config, time_input, process)
             add_to_time_input(time_input,
-                              instance=process.instance)
+                              instance=process.instance,
+                              custom=custom)
 
             process.clear()
             process.run_at_time(time_input)

metplus/wrappers/command_builder.py

@@ -1373,10 +1373,13 @@ def run_at_time(self, input_dict):
                        'wrapper. Cannot run with LOOP_ORDER = times')
         return None
 
-    def run_all_times(self):
-        """!Loop over time range specified in conf file and
-        call METplus wrapper for each time"""
-        return util.loop_over_times_and_call(self.config, self)
+    def run_all_times(self, custom=None):
+        """! Loop over time range specified in conf file and
+        call METplus wrapper for each time
+
+        @param custom (optional) custom loop string value
+        """
+        return util.loop_over_times_and_call(self.config, self, custom=custom)
 
     @staticmethod
     def format_met_config_dict(c_dict, name, keys=None):

metplus/wrappers/compare_gridded_wrapper.py

@@ -169,8 +169,6 @@ def run_at_time_once(self, time_info):
               Args:
                 @param time_info dictionary containing timing information
         """
-        self.clear()
-
         var_list = util.sub_var_list(self.c_dict['VAR_LIST_TEMP'],
                                      time_info)
 
@@ -191,6 +189,7 @@ def run_at_time_once(self, time_info):
             if var_list:
                 self.c_dict['CURRENT_VAR_INFO'] = var_list[0]
 
+            self.clear()
             self.run_at_time_all_fields(time_info)
 
     def run_at_time_one_field(self, time_info, var_info):

metplus/wrappers/pb2nc_wrapper.py

@@ -274,14 +274,12 @@ def run_at_time(self, input_dict):
                 input_dict['custom'] = custom_string
 
                 # Run for given init/valid time and forecast lead combination
+                self.clear()
                 self.run_at_time_once(input_dict)
 
 
     def run_at_time_once(self, input_dict):
         """!Find files needed to run pb2nc and run if found"""
-        # clear out information set from previous run
-        self.clear()
-
         # look for input files to process
         time_info = self.find_input_files(input_dict)
 

metplus/wrappers/runtime_freq_wrapper.py

@@ -138,7 +138,7 @@ def run_all_times_custom(self, custom):
         elif runtime_freq == 'RUN_ONCE_PER_LEAD':
             self.run_once_per_lead(custom)
         elif runtime_freq == 'RUN_ONCE_FOR_EACH':
-            self.all_commands = super().run_all_times()
+            self.all_commands = super().run_all_times(custom)
 
     def run_once(self, custom):
         self.logger.debug("Running once for all files")
@@ -178,6 +178,7 @@ def run_once_per_init_or_valid(self, custom):
 
             time_input['lead'] = '*'
 
+            self.clear()
             if not self.run_at_time_once(time_input):
                 success = False
 
@@ -204,6 +205,7 @@ def run_once_per_lead(self, custom):
             time_input['init'] = '*'
             time_input['valid'] = '*'
 
+            self.clear()
             if not self.run_at_time_once(time_input):
                 success = False
 
@@ -216,42 +218,37 @@ def run_at_time(self, input_dict):
 
             @param input_dict dictionary containing time information
         """
-        for custom_string in self.c_dict['CUSTOM_LOOP_LIST']:
-            if custom_string:
-                self.logger.info(f"Processing custom string: {custom_string}")
-
-            input_dict['custom'] = custom_string
-
-            # loop of forecast leads and process each
-            lead_seq = get_lead_sequence(self.config, input_dict)
-            for lead in lead_seq:
-                input_dict['lead'] = lead
-
-                # set current lead time config and environment variables
-                time_info = time_util.ti_calculate(input_dict)
-
-                self.logger.info(
-                    f"Processing forecast lead {time_info['lead_string']}"
-                )
+        # loop of forecast leads and process each
+        lead_seq = get_lead_sequence(self.config, input_dict)
+        for lead in lead_seq:
+            input_dict['lead'] = lead
 
-                if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})):
-                    self.logger.debug('Skipping run time')
-                    continue
+            # set current lead time config and environment variables
+            time_info = time_util.ti_calculate(input_dict)
 
-                # since run_all_times was not called (LOOP_BY=times) then
-                # get files for current run time
-                file_dict = self.get_files_from_time(time_info)
-                all_files = []
-                if file_dict:
-                    if isinstance(file_dict, list):
-                        all_files = file_dict
-                    else:
-                        all_files = [file_dict]
+            self.logger.info(
+                f"Processing forecast lead {time_info['lead_string']}"
+            )
 
-                self.c_dict['ALL_FILES'] = all_files
+            if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})):
+                self.logger.debug('Skipping run time')
+                continue
 
-                # Run for given init/valid time and forecast lead combination
-                self.run_at_time_once(time_info)
+            # since run_all_times was not called (LOOP_BY=times) then
+            # get files for current run time
+            file_dict = self.get_files_from_time(time_info)
+            all_files = []
+            if file_dict:
+                if isinstance(file_dict, list):
+                    all_files = file_dict
+                else:
+                    all_files = [file_dict]
+
+            self.c_dict['ALL_FILES'] = all_files
+
+            # Run for given init/valid time and forecast lead combination
+            self.clear()
+            self.run_at_time_once(time_info)
 
     def get_all_files(self, custom=None):
         """! Get all files that can be processed with the app.

metplus/wrappers/tc_gen_wrapper.py

@@ -368,6 +368,7 @@ def run_at_time(self, input_dict):
                 self.logger.debug('Skipping run time')
                 continue
 
+            self.clear()
             self.run_at_time_once(time_info)
 
     def run_at_time_once(self, time_info):

metplus/wrappers/tcrmw_wrapper.py

@@ -225,6 +225,7 @@ def run_at_time(self, input_dict):
                 self.logger.debug('Skipping run time')
                 continue
 
+            self.clear()
             self.run_at_time_once(time_info)
 
     def run_at_time_once(self, time_info):