Added checker to class

run.py

@@ -13,5 +13,8 @@
     stl_runner.process_argv(sys.argv)
     logger = stl_runner.set_logger()
     # logging.warn(stl_runner.params)
-    stl_runner.run()
+    if stl_runner.checker:
+        stl_runner.check_progress()
+    else:
+        stl_runner.run()
     logging.warning("FINISHED!!")

streamline/runner.py

@@ -11,6 +11,9 @@
 from streamline.utils.config_loader import load_default_config
 from streamline.utils.parser import parser_function_definition
 from streamline.utils.parser import parser_function_all, single_parse, process_params
+from streamline.utils.parser_helpers import parse_checker, parse_general
+from streamline.utils.checker import check_phase
+
 
 warnings.filterwarnings("ignore")
 optuna.logging.set_verbosity(optuna.logging.WARNING)
@@ -27,6 +30,8 @@ class STREAMLINERunner:
 
     def __init__(self, argv=None, run=False):
         self.params = dict()
+        self.runner = run
+        self.checker = None
         if argv:
             self.process_argv(argv)
         if run:
@@ -47,6 +52,8 @@ def check_cli(mode_params):
             flag = False
             if mode_params['do_till_report']:
                 flag = True
+            if mode_params['checker']:
+                flag = True
             for key in mode_params:
                 if mode_params[key] and key not in ['config', 'do_till_report', 'verbose', 'checker']:
                     flag = True
@@ -55,7 +62,8 @@ def check_cli(mode_params):
         if check_cli(mode_params):
             self.load_cli_params(mode_params, argv)
 
-        self.params = process_params(self.params)
+        if not self.checker:
+            self.params = process_params(self.params)
 
         for param in self.essential_params:
             if param not in self.params:
@@ -86,7 +94,14 @@ def load_cli_params(self, mode_params, argv):
             self.params.update(config)
             save_config(self.params['output_path'],
                         self.params['experiment_name'],
-                        self.params)       
+                        self.params)
+        if mode_params['checker']:
+            print("Checking Progress")
+            config = parse_general(argv, self.params)
+            config = parse_checker(argv, config)
+            self.params.update(config)
+            self.checker = True
+            return
 
         for key in mode_params:
             if mode_params[key] and key not in ['config', 'do_till_report', 'verbose', 'checker']:
@@ -95,7 +110,7 @@ def load_cli_params(self, mode_params, argv):
                 self.params.update(mode_params)
                 save_config(self.params['output_path'],
                             self.params['experiment_name'],
-                            self.params)  
+                            self.params)
 
     def save_params(self):
         save_config(self.params['output_path'], self.params['experiment_name'], self.params)
@@ -184,6 +199,11 @@ def run_phase(self, obj, phase):
             time.sleep(2)
         del obj
 
+    def check_progress(self):
+        check_phase(self.params['output_path'], self.params['experiment_name'], 
+                    self.params['phase'], self.params['len_only'],
+                    self.params['rep_data_path'], self.params['dataset_for_rep'])
+
     def run(self):
         start_g = time.time()
 

streamline/utils/parser.py

@@ -1,7 +1,7 @@
 import argparse
 import configparser
 from streamline.utils.parser_helpers import str2bool, save_config, load_config
-from streamline.utils.parser_helpers import parse_general, parse_replicate
+from streamline.utils.parser_helpers import parse_general, parse_replicate, parse_checker
 from streamline.utils.parser_helpers import parse_logistic
 from streamline.utils.parser_helpers import parser_function_all
 from streamline.utils.parser_helpers import PARSER_LIST
@@ -70,7 +70,8 @@ def single_parse(mode_params, argv, config_dict=None):
             'do_report',
             'do_replicate',
             'do_rep_report',
-            'do_cleanup', ]
+            'do_cleanup', 
+            'checker']
     for i in range(len(keys)):
         if mode_params[keys[i]]:
             if i == 0:
@@ -99,6 +100,17 @@ def single_parse(mode_params, argv, config_dict=None):
                             = config_dict_copy['rep_export_feature_correlations']
                     if not config_dict_copy['exclude_rep_plots'] == 'None':
                         config_dict['exclude_rep_plots'] = config_dict_copy['exclude_rep_plots']
+                if i == 11:
+                    config_dict_copy = parse_checker(argv, config_dict)
+                    if not config_dict_copy['rep_data_path'] == "":
+                        config_dict['rep_data_path'] = config_dict_copy['rep_data_path']
+                    if not config_dict_copy['dataset_for_rep'] == "":
+                        config_dict['dataset_for_rep'] = config_dict_copy['dataset_for_rep']
+                    if not config_dict_copy['rep_export_feature_correlations']:
+                        config_dict['rep_export_feature_correlations'] \
+                            = config_dict_copy['rep_export_feature_correlations']
+                    if not config_dict_copy['exclude_rep_plots'] == 'None':
+                        config_dict['exclude_rep_plots'] = config_dict_copy['exclude_rep_plots']
             config_dict = parse_logistic(argv, config_dict)
     return config_dict
 

streamline/utils/parser_helpers.py

@@ -92,6 +92,17 @@ def parse_general(argv, params_dict=None):
     return update_dict_from_parser(argv, parser, params_dict)
 
 
+def parse_checker(argv, params_dict=None):
+    parser = argparse.ArgumentParser(description='program to check if run is complete')
+    parser.add_argument('--phase', dest='phase', type=int, help='phase to check')
+    parser.add_argument('--count-only', dest='len_only', type=bool, default=False, help='show only no of jobs')
+    parser.add_argument('--rep-data-path', dest='rep_data_path', type=str,
+                        help='replication dataset path')
+    parser.add_argument('--dataset-for-rep', dest='dataset_for_rep',
+                        type=str, help='train dataset for replication path')
+    return update_dict_from_parser(argv, parser, params_dict)
+
+
 def parse_eda(argv, params_dict=None):
     # Parse arguments
     parser = argparse.ArgumentParser(description="",