diff --git a/caravel/bin/caravel b/caravel/bin/caravel index c53cdc2b338fb..fcf21be0a7cad 100755 --- a/caravel/bin/caravel +++ b/caravel/bin/caravel @@ -7,6 +7,7 @@ from __future__ import unicode_literals import logging from datetime import datetime from subprocess import Popen +from collections import defaultdict from flask_migrate import MigrateCommand from flask_script import Manager @@ -125,6 +126,67 @@ def refresh_druid(): "[" + cluster.cluster_name + "]") session.commit() +@manager.option( + '-p', '--prefix', default="", + help="Sync Table Prefix") +@manager.option( + '-d', '--database', default=None, + help="Specifies database (use caravel db name, not real db name)") +def synctable(prefix, database): + ''' Sync DB Table with Caravel Table''' + print("") + + if (prefix == "" and database is None) or (database == 'main' and prefix == ""): + print("If you not set prefix and db name, some system table may be sync to caravel.") + + check_status = raw_input("Are you sure do this? (Y/N)").lower().strip() + if check_status not in ['y', 'yes']: + print("Exit sync table") + exit() + + if database is None: + caravel_dbs = db.session.query(caravel.models.Database).all() + else: + caravel_dbs = (db.session.query(caravel.models.Database) + .filter_by(database_name=database).all()) + + if caravel_dbs == []: + print("Database not exists, please check database name") + exit() + + existing_tables = [] + for row in db.session.query(caravel.models.SqlaTable).all(): + existing_tables += [(row.database.database_name, row.name)] + + # Get all need insert table + insert_dict = defaultdict(list) + for caravel_db in caravel_dbs: + for table_name in caravel_db.get_sqla_engine().table_names(): + table = (caravel_db.database_name, table_name) + + if (table not in existing_tables) and table_name.startswith(prefix): + insert_dict[caravel_db].append(table_name) + + # Insert to caravel tables + count_insert = 0 + for caravel_db, tables in insert_dict.items(): + for table_name in tables: + count_insert += 1 + + tbl = caravel.models.SqlaTable(table_name=table_name) + tbl.description = "" + tbl.is_featured = False + tbl.database = caravel_db + db.session.add(tbl) + db.session.commit() + tbl.fetch_metadata() + + print("[{db}] {table} insert success.".format( + db=caravel_db.database_name, + table=table_name + )) + + print("[{}] Sync table complete.".format(count_insert)) if __name__ == "__main__": manager.run()