Generalize switch between different datasources (#1078)

* Generalize switch between different datasources. * Fix previous migration since slice model changed * Fix warm up cache and other small stuff * Adding modules and datasources through config * Replace tabs w/ spaces * Fix other style issues * Change add method for SliceModelView to pick the first non-empty ds * Remove tests on slice add redirect * Change way of db migration * Fix styling * Fix create slice * Small fixes * Fix code climate check * Adding notes on how to create new datasource in CONTRIBUTING.md * Fix last merge * A commit just to trigger travis build again * Add migration to merge two heads * Fix codeclimate * Simplify source_registry * Fix codeclimate * Remove all getter methods
apache · Sep 21, 2016 · 5a0e06e · 5a0e06e
1 parent ed2feaf
commit 5a0e06e
Show file tree

Hide file tree

Showing 13 changed files with 223 additions and 110 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -251,3 +251,20 @@ You can then translate the strings gathered in files located under
 to take effect, they need to be compiled using this command:
 
     fabmanager babel-compile --target caravel/translations/
+
+
+## Adding new datasources
+
+1. Create Models and Views for the datasource, add them under caravel folder, like a new my_models.py
+    with models for cluster, datasources, columns and metrics and my_views.py with clustermodelview
+    and datasourcemodelview.
+
+2. Create db migration files for the new models
+
+3. Specify this variable to add the datasource model and from which module it is from in config.py:
+
+    For example:
+
+    `ADDITIONAL_MODULE_DS_MAP = {'caravel.my_models': ['MyDatasource', 'MyOtherDatasource']}`
+
+    This means it'll register MyDatasource and MyOtherDatasource in caravel.my_models module in the source registry.
diff --git a/caravel/__init__.py b/caravel/__init__.py
@@ -14,6 +14,7 @@
 from flask_appbuilder.baseviews import expose
 from flask_cache import Cache
 from flask_migrate import Migrate
+from caravel import source_registry
 from werkzeug.contrib.fixers import ProxyFix
 
 
@@ -95,5 +96,7 @@ def index(self):
 
 sm = appbuilder.sm
 
+src_registry = source_registry.SourceRegistry()
+
 get_session = appbuilder.get_session
-from caravel import config, views  # noqa
+from caravel import views, config  # noqa
diff --git a/caravel/bin/caravel b/caravel/bin/caravel
@@ -20,6 +20,14 @@ config = app.config
 
 manager = Manager(app)
 manager.add_command('db', MigrateCommand)
+module_datasource_map = config.get("DEFAULT_MODULE_DS_MAP")
+module_datasource_map.update(config.get("ADDITIONAL_MODULE_DS_MAP"))
+
+datasources = {}
+for module in module_datasource_map:
+    datasources[module] = __import__(module, fromlist=module_datasource_map[module])
+
+utils.register_sources(datasources, module_datasource_map, caravel.src_registry)
 
 
 @manager.option(

diff --git a/caravel/config.py b/caravel/config.py
@@ -164,6 +164,13 @@
 
 DRUID_DATA_SOURCE_BLACKLIST = []
 
+# --------------------------------------------------
+# Modules and datasources to be registered
+# --------------------------------------------------
+DEFAULT_MODULE_DS_MAP = {'caravel.models': ['DruidDatasource', 'SqlaTable']}
+ADDITIONAL_MODULE_DS_MAP = {}
+
+
 """
 1) http://docs.python-guide.org/en/latest/writing/logging/
 2) https://docs.python.org/2/library/logging.config.html

diff --git a/caravel/data/__init__.py b/caravel/data/__init__.py
@@ -75,7 +75,7 @@ def load_energy():
             slice_name="Energy Sankey",
             viz_type='sankey',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=textwrap.dedent("""\
             {
                 "collapsed_fieldsets": "",
@@ -105,7 +105,7 @@ def load_energy():
             slice_name="Energy Force Layout",
             viz_type='directed_force',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=textwrap.dedent("""\
             {
                 "charge": "-500",
@@ -136,7 +136,7 @@ def load_energy():
             slice_name="Heatmap",
             viz_type='heatmap',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=textwrap.dedent("""\
             {
                 "all_columns_x": "source",
@@ -224,7 +224,7 @@ def load_world_bank_health_n_pop():
             slice_name="Region Filter",
             viz_type='filter_box',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type='filter_box',
@@ -233,7 +233,7 @@ def load_world_bank_health_n_pop():
             slice_name="World's Population",
             viz_type='big_number',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 since='2000',
@@ -245,7 +245,7 @@ def load_world_bank_health_n_pop():
             slice_name="Most Populated Countries",
             viz_type='table',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type='table',
@@ -255,7 +255,7 @@ def load_world_bank_health_n_pop():
             slice_name="Growth Rate",
             viz_type='line',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type='line',
@@ -267,7 +267,7 @@ def load_world_bank_health_n_pop():
             slice_name="% Rural",
             viz_type='world_map',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type='world_map',
@@ -277,7 +277,7 @@ def load_world_bank_health_n_pop():
             slice_name="Life Expectancy VS Rural %",
             viz_type='bubble',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type='bubble',
@@ -298,7 +298,7 @@ def load_world_bank_health_n_pop():
             slice_name="Rural Breakdown",
             viz_type='sunburst',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type='sunburst',
@@ -310,7 +310,7 @@ def load_world_bank_health_n_pop():
             slice_name="World's Pop Growth",
             viz_type='area',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 since="1960-01-01",
@@ -321,7 +321,7 @@ def load_world_bank_health_n_pop():
             slice_name="Box plot",
             viz_type='box_plot',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 since="1960-01-01",
@@ -333,7 +333,7 @@ def load_world_bank_health_n_pop():
             slice_name="Treemap",
             viz_type='treemap',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 since="1960-01-01",
@@ -345,7 +345,7 @@ def load_world_bank_health_n_pop():
             slice_name="Parallel Coordinates",
             viz_type='para',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 since="2011-01-01",
@@ -615,7 +615,7 @@ def load_birth_names():
             slice_name="Girls",
             viz_type='table',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 groupby=['name'],
@@ -625,7 +625,7 @@ def load_birth_names():
             slice_name="Boys",
             viz_type='table',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 groupby=['name'],
@@ -636,7 +636,7 @@ def load_birth_names():
             slice_name="Participants",
             viz_type='big_number',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type="big_number", granularity="ds",
@@ -645,15 +645,15 @@ def load_birth_names():
             slice_name="Genders",
             viz_type='pie',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type="pie", groupby=['gender'])),
         Slice(
             slice_name="Genders by State",
             viz_type='dist_bar',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 flt_eq_1="other", viz_type="dist_bar",
@@ -663,7 +663,7 @@ def load_birth_names():
             slice_name="Trends",
             viz_type='line',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type="line", groupby=['name'],
@@ -672,7 +672,7 @@ def load_birth_names():
             slice_name="Title",
             viz_type='markup',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type="markup", markup_type="html",
@@ -690,7 +690,7 @@ def load_birth_names():
             slice_name="Name Cloud",
             viz_type='word_cloud',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type="word_cloud", size_from="10",
@@ -700,7 +700,7 @@ def load_birth_names():
             slice_name="Pivot Table",
             viz_type='pivot_table',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type="pivot_table", metrics=['sum__num'],
@@ -709,7 +709,7 @@ def load_birth_names():
             slice_name="Number of Girls",
             viz_type='big_number_total',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(
                 defaults,
                 viz_type="big_number_total", granularity="ds",
@@ -862,7 +862,7 @@ def load_unicode_test_data():
         slice_name="Unicode Cloud",
         viz_type='word_cloud',
         datasource_type='table',
-        table=tbl,
+        datasource_id=tbl.id,
         params=get_slice_json(slice_data),
     )
     merge_slice(slc)
@@ -935,7 +935,7 @@ def load_random_time_series_data():
         slice_name="Calendar Heatmap",
         viz_type='cal_heatmap',
         datasource_type='table',
-        table=tbl,
+        datasource_id=tbl.id,
         params=get_slice_json(slice_data),
     )
     merge_slice(slc)
@@ -1005,7 +1005,7 @@ def load_long_lat_data():
         slice_name="Mapbox Long/Lat",
         viz_type='mapbox',
         datasource_type='table',
-        table=tbl,
+        datasource_id=tbl.id,
         params=get_slice_json(slice_data),
     )
     merge_slice(slc)
@@ -1084,7 +1084,7 @@ def load_multiformat_time_series_data():
             slice_name="Calendar Heatmap multiformat" + str(i),
             viz_type='cal_heatmap',
             datasource_type='table',
-            table=tbl,
+            datasource_id=tbl.id,
             params=get_slice_json(slice_data),
         )
         merge_slice(slc)
diff --git a/caravel/migrations/versions/27ae655e4247_make_creator_owners.py b/caravel/migrations/versions/27ae655e4247_make_creator_owners.py
@@ -11,15 +11,34 @@
 down_revision = 'd8bc074f7aad'
 
 from alembic import op
-from caravel import db, models
+from caravel import db
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+from flask_appbuilder import Model
+from sqlalchemy import (
+    Column, Integer, ForeignKey, Table)
+
+Base = declarative_base()
+
+
+class Slice(Base):
+    """Declarative class to do query in upgrade"""
+    __tablename__ = 'slices'
+    id = Column(Integer, primary_key=True)
+
+
+class Dashboard(Base):
+    """Declarative class to do query in upgrade"""
+    __tablename__ = 'dashboards'
+    id = Column(Integer, primary_key=True)
 
 
 def upgrade():
     bind = op.get_bind()
     session = db.Session(bind=bind)
 
-    objects = session.query(models.Slice).all()
-    objects += session.query(models.Dashboard).all()
+    objects = session.query(Slice).all()
+    objects += session.query(Dashboard).all()
     for obj in objects:
         if obj.created_by and obj.created_by not in obj.owners:
             obj.owners.append(obj.created_by)