Merge pull request ibis-project#26 from dr-pain/census2db

add writing results to db

census/census_pandas_ibis.py

@@ -278,12 +278,20 @@ def etl_ibis(
     return X, y, etl_times
 
 
-def print_times(etl_times, name=None):
-    if name:
-        print(f"{name} times:")
+def print_times(etl_times, backend, db_reporter=None):
+    print(f"{backend} times:")
     for time_name, time in etl_times.items():
         print("{} = {:.5f} s".format(time_name, time))
-
+        if db_reporter is not None:
+            db_reporter.submit({
+                'QueryName': time_name,
+                'FirstExecTimeMS': time*1000,
+                'WorstExecTimeMS': time*1000,
+                'BestExecTimeMS': time*1000,
+                'AverageExecTimeMS': time*1000,
+                'TotalTimeMS': time*1000,
+                'BackEnd': backend
+            })
 
 def mse(y_test, y_pred):
     return ((y_test - y_pred) ** 2).mean()
@@ -411,7 +419,7 @@ def main():
     )
     optional.add_argument(
         "-db-pass",
-        dest="db_password",
+        dest="db_pass",
         default="omniscidb",
         help="Password to use to connect to MySQL database.",
     )
@@ -636,6 +644,26 @@ def main():
             from server_worker import OmnisciServerWorker
             omnisci_server_worker = OmnisciServerWorker(omnisci_server)
 
+            db_reporter = None
+            if args.db_user is not "":
+                print("Connecting to database")
+                db = mysql.connector.connect(host=args.db_server, port=args.db_port, user=args.db_user,
+                                             passwd=args.db_pass, db=args.db_name)
+                db_reporter = DbReport(db, args.db_table, {
+                    'QueryName': 'VARCHAR(500) NOT NULL',
+                    'FirstExecTimeMS': 'BIGINT UNSIGNED',
+                    'WorstExecTimeMS': 'BIGINT UNSIGNED',
+                    'BestExecTimeMS': 'BIGINT UNSIGNED',
+                    'AverageExecTimeMS': 'BIGINT UNSIGNED',
+                    'TotalTimeMS': 'BIGINT UNSIGNED',
+                    'IbisCommitHash': 'VARCHAR(500) NOT NULL',
+                    'BackEnd': 'VARCHAR(100) NOT NULL'
+                }, {
+                    'ScriptName': 'census_pandas_ibis.py',
+                    'CommitHash': args.commit_omnisci,
+                    'IbisCommitHash': args.commit_ibis
+                })
+
             X_ibis, y_ibis, etl_times_ibis = etl_ibis(
                 filename=args.file,
                 columns_names=columns_names,
@@ -648,13 +676,13 @@ def main():
             )
             omnisci_server.terminate()
             omnisci_server = None
-            print_times(etl_times_ibis, name='Ibis')
+            print_times(etl_times_ibis, 'Ibis', db_reporter)
 
             if not args.no_ml:
                 mse_mean, cod_mean, mse_dev, cod_dev, ml_times = ml(
                     X_ibis, y_ibis, RANDOM_STATE, N_RUNS, TRAIN_SIZE, args.optimizer
                 )
-                print_times(ml_times)
+                print_times(ml_times, 'Ibis')
                 print("mean MSE ± deviation: {:.9f} ± {:.9f}".format(mse_mean, mse_dev))
                 print("mean COD ± deviation: {:.9f} ± {:.9f}".format(cod_mean, cod_dev))
 
@@ -663,13 +691,13 @@ def main():
         X, y, etl_times = etl_pandas(
             args.file, columns_names=columns_names, columns_types=columns_types
         )
-        print_times(etl_times, name=args.pandas_mode)
+        print_times(etl_times, args.pandas_mode, db_reporter)
 
         if not args.no_ml:
             mse_mean, cod_mean, mse_dev, cod_dev, ml_times = ml(
                 X, y, RANDOM_STATE, N_RUNS, TRAIN_SIZE, args.optimizer
             )
-            print_times(ml_times)
+            print_times(ml_times, args.pandas_mode)
             print("mean MSE ± deviation: {:.9f} ± {:.9f}".format(mse_mean, mse_dev))
             print("mean COD ± deviation: {:.9f} ± {:.9f}".format(cod_mean, cod_dev))