only apply the limit when specified

Author: Matt Severini

config.json.example

@@ -19,7 +19,6 @@
         "port": 5432
     },
     "keep_disconnected_tables": false,
-    "max_rows_per_table": ALL,
     "excluded_tables": [ ],
     "passthrough_tables": [ ],
     "dependency_breaks": [ ],

config_reader.py

@@ -60,7 +60,7 @@ def get_post_subset_sql():
     return _config["post_subset_sql"] if "post_subset_sql" in _config else []
 
 def get_max_rows_per_table():
-    return _config["max_rows_per_table"]
+    return _config["max_rows_per_table"] if "max_rows_per_table" in _config else None
 
 def __convert_tonic_format(obj):
     if "fk_schema" in obj:

subset.py

@@ -71,7 +71,9 @@ def run_middle_out(self):
         start_time = time.time()
         for idx, t in enumerate(passthrough_tables):
             print_progress(t, idx+1, len(passthrough_tables))
-            q = 'SELECT * FROM {} LIMIT {}'.format(fully_qualified_table(t), config_reader.get_max_rows_per_table())
+            q = 'SELECT * FROM {}'.format(fully_qualified_table(t))
+            if config_reader.get_max_rows_per_table() is not None:
+                q += 'LIMIT {}'.format(config_reader.get_max_rows_per_table())
             self.__db_helper.copy_rows(self.__source_conn, self.__destination_conn, q, mysql_db_name_hack(t, self.__destination_conn))
         print('Pass-through completed in {}s'.format(time.time()-start_time))
 
@@ -140,7 +142,8 @@ def __subset_upstream(self, target, processed_tables, relationships):
             clauses.extend(upstream_filter_match(target, table_columns))
 
             select_query = 'SELECT * FROM {} WHERE TRUE AND {}'.format(quoter(temp_target_name), ' AND '.join(clauses))
-            select_query += " LIMIT {}".format(config_reader.get_max_rows_per_table())
+            if config_reader.get_max_rows_per_table() is not None:
+                select_query += " LIMIT {}".format(config_reader.get_max_rows_per_table())
             insert_query = 'INSERT INTO {} {}'.format(fully_qualified_table(mysql_db_name_hack(target, self.__destination_conn)), select_query)
             self.__db_helper.run_query(insert_query, self.__destination_conn)
             self.__destination_conn.commit()