diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15ba0d51b2b0a..e5abc1182a7d0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -251,3 +251,20 @@ You can then translate the strings gathered in files located under to take effect, they need to be compiled using this command: fabmanager babel-compile --target caravel/translations/ + + +## Adding new datasources + +1. Create Models and Views for the datasource, add them under caravel folder, like a new my_models.py + with models for cluster, datasources, columns and metrics and my_views.py with clustermodelview + and datasourcemodelview. + +2. Create db migration files for the new models + +3. Specify this variable to add the datasource model and from which module it is from in config.py: + + For example: + + `ADDITIONAL_MODULE_DS_MAP = {'caravel.my_models': ['MyDatasource', 'MyOtherDatasource']}` + + This means it'll register MyDatasource and MyOtherDatasource in caravel.my_models module in the source registry. diff --git a/caravel/__init__.py b/caravel/__init__.py index b01a8e59cf10c..f06729d7a47c8 100644 --- a/caravel/__init__.py +++ b/caravel/__init__.py @@ -14,6 +14,7 @@ from flask_appbuilder.baseviews import expose from flask_cache import Cache from flask_migrate import Migrate +from caravel import source_registry from werkzeug.contrib.fixers import ProxyFix @@ -95,5 +96,7 @@ def index(self): sm = appbuilder.sm +src_registry = source_registry.SourceRegistry() + get_session = appbuilder.get_session -from caravel import config, views # noqa +from caravel import views, config # noqa diff --git a/caravel/bin/caravel b/caravel/bin/caravel index a582c3e5ecb63..bf09f4d61c47b 100755 --- a/caravel/bin/caravel +++ b/caravel/bin/caravel @@ -20,6 +20,14 @@ config = app.config manager = Manager(app) manager.add_command('db', MigrateCommand) +module_datasource_map = config.get("DEFAULT_MODULE_DS_MAP") +module_datasource_map.update(config.get("ADDITIONAL_MODULE_DS_MAP")) + +datasources = {} +for module in module_datasource_map: + datasources[module] = __import__(module, fromlist=module_datasource_map[module]) + +utils.register_sources(datasources, module_datasource_map, caravel.src_registry) @manager.option( diff --git a/caravel/config.py b/caravel/config.py index b2bcb3d929d81..8871d95b55e28 100644 --- a/caravel/config.py +++ b/caravel/config.py @@ -164,6 +164,13 @@ DRUID_DATA_SOURCE_BLACKLIST = [] +# -------------------------------------------------- +# Modules and datasources to be registered +# -------------------------------------------------- +DEFAULT_MODULE_DS_MAP = {'caravel.models': ['DruidDatasource', 'SqlaTable']} +ADDITIONAL_MODULE_DS_MAP = {} + + """ 1) http://docs.python-guide.org/en/latest/writing/logging/ 2) https://docs.python.org/2/library/logging.config.html diff --git a/caravel/data/__init__.py b/caravel/data/__init__.py index 014e1eb99a65f..5198a12af7a75 100644 --- a/caravel/data/__init__.py +++ b/caravel/data/__init__.py @@ -75,7 +75,7 @@ def load_energy(): slice_name="Energy Sankey", viz_type='sankey', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=textwrap.dedent("""\ { "collapsed_fieldsets": "", @@ -105,7 +105,7 @@ def load_energy(): slice_name="Energy Force Layout", viz_type='directed_force', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=textwrap.dedent("""\ { "charge": "-500", @@ -136,7 +136,7 @@ def load_energy(): slice_name="Heatmap", viz_type='heatmap', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=textwrap.dedent("""\ { "all_columns_x": "source", @@ -224,7 +224,7 @@ def load_world_bank_health_n_pop(): slice_name="Region Filter", viz_type='filter_box', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type='filter_box', @@ -233,7 +233,7 @@ def load_world_bank_health_n_pop(): slice_name="World's Population", viz_type='big_number', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, since='2000', @@ -245,7 +245,7 @@ def load_world_bank_health_n_pop(): slice_name="Most Populated Countries", viz_type='table', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type='table', @@ -255,7 +255,7 @@ def load_world_bank_health_n_pop(): slice_name="Growth Rate", viz_type='line', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type='line', @@ -267,7 +267,7 @@ def load_world_bank_health_n_pop(): slice_name="% Rural", viz_type='world_map', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type='world_map', @@ -277,7 +277,7 @@ def load_world_bank_health_n_pop(): slice_name="Life Expectancy VS Rural %", viz_type='bubble', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type='bubble', @@ -298,7 +298,7 @@ def load_world_bank_health_n_pop(): slice_name="Rural Breakdown", viz_type='sunburst', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type='sunburst', @@ -310,7 +310,7 @@ def load_world_bank_health_n_pop(): slice_name="World's Pop Growth", viz_type='area', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, since="1960-01-01", @@ -321,7 +321,7 @@ def load_world_bank_health_n_pop(): slice_name="Box plot", viz_type='box_plot', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, since="1960-01-01", @@ -333,7 +333,7 @@ def load_world_bank_health_n_pop(): slice_name="Treemap", viz_type='treemap', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, since="1960-01-01", @@ -345,7 +345,7 @@ def load_world_bank_health_n_pop(): slice_name="Parallel Coordinates", viz_type='para', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, since="2011-01-01", @@ -615,7 +615,7 @@ def load_birth_names(): slice_name="Girls", viz_type='table', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, groupby=['name'], @@ -625,7 +625,7 @@ def load_birth_names(): slice_name="Boys", viz_type='table', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, groupby=['name'], @@ -636,7 +636,7 @@ def load_birth_names(): slice_name="Participants", viz_type='big_number', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type="big_number", granularity="ds", @@ -645,7 +645,7 @@ def load_birth_names(): slice_name="Genders", viz_type='pie', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type="pie", groupby=['gender'])), @@ -653,7 +653,7 @@ def load_birth_names(): slice_name="Genders by State", viz_type='dist_bar', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, flt_eq_1="other", viz_type="dist_bar", @@ -663,7 +663,7 @@ def load_birth_names(): slice_name="Trends", viz_type='line', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type="line", groupby=['name'], @@ -672,7 +672,7 @@ def load_birth_names(): slice_name="Title", viz_type='markup', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type="markup", markup_type="html", @@ -690,7 +690,7 @@ def load_birth_names(): slice_name="Name Cloud", viz_type='word_cloud', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type="word_cloud", size_from="10", @@ -700,7 +700,7 @@ def load_birth_names(): slice_name="Pivot Table", viz_type='pivot_table', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type="pivot_table", metrics=['sum__num'], @@ -709,7 +709,7 @@ def load_birth_names(): slice_name="Number of Girls", viz_type='big_number_total', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json( defaults, viz_type="big_number_total", granularity="ds", @@ -862,7 +862,7 @@ def load_unicode_test_data(): slice_name="Unicode Cloud", viz_type='word_cloud', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json(slice_data), ) merge_slice(slc) @@ -935,7 +935,7 @@ def load_random_time_series_data(): slice_name="Calendar Heatmap", viz_type='cal_heatmap', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json(slice_data), ) merge_slice(slc) @@ -1005,7 +1005,7 @@ def load_long_lat_data(): slice_name="Mapbox Long/Lat", viz_type='mapbox', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json(slice_data), ) merge_slice(slc) @@ -1084,7 +1084,7 @@ def load_multiformat_time_series_data(): slice_name="Calendar Heatmap multiformat" + str(i), viz_type='cal_heatmap', datasource_type='table', - table=tbl, + datasource_id=tbl.id, params=get_slice_json(slice_data), ) merge_slice(slc) diff --git a/caravel/migrations/versions/27ae655e4247_make_creator_owners.py b/caravel/migrations/versions/27ae655e4247_make_creator_owners.py index 71c627305dbc7..e9f84b63bf219 100644 --- a/caravel/migrations/versions/27ae655e4247_make_creator_owners.py +++ b/caravel/migrations/versions/27ae655e4247_make_creator_owners.py @@ -11,15 +11,34 @@ down_revision = 'd8bc074f7aad' from alembic import op -from caravel import db, models +from caravel import db +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship +from flask_appbuilder import Model +from sqlalchemy import ( + Column, Integer, ForeignKey, Table) + +Base = declarative_base() + + +class Slice(Base): + """Declarative class to do query in upgrade""" + __tablename__ = 'slices' + id = Column(Integer, primary_key=True) + + +class Dashboard(Base): + """Declarative class to do query in upgrade""" + __tablename__ = 'dashboards' + id = Column(Integer, primary_key=True) def upgrade(): bind = op.get_bind() session = db.Session(bind=bind) - objects = session.query(models.Slice).all() - objects += session.query(models.Dashboard).all() + objects = session.query(Slice).all() + objects += session.query(Dashboard).all() for obj in objects: if obj.created_by and obj.created_by not in obj.owners: obj.owners.append(obj.created_by) diff --git a/caravel/migrations/versions/33d996bcc382_update_slice_model.py b/caravel/migrations/versions/33d996bcc382_update_slice_model.py new file mode 100644 index 0000000000000..5450a1151dcf2 --- /dev/null +++ b/caravel/migrations/versions/33d996bcc382_update_slice_model.py @@ -0,0 +1,59 @@ +from alembic import op +import sqlalchemy as sa +from caravel import db +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import ( + Column, Integer, String) + +"""update slice model + +Revision ID: 33d996bcc382 +Revises: 41f6a59a61f2 +Create Date: 2016-09-07 23:50:59.366779 + +""" + +# revision identifiers, used by Alembic. +revision = '33d996bcc382' +down_revision = '41f6a59a61f2' + +Base = declarative_base() + + +class Slice(Base): + """Declarative class to do query in upgrade""" + __tablename__ = 'slices' + id = Column(Integer, primary_key=True) + datasource_id = Column(Integer) + druid_datasource_id = Column(Integer) + table_id = Column(Integer) + datasource_type = Column(String(200)) + + +def upgrade(): + bind = op.get_bind() + op.add_column('slices', sa.Column('datasource_id', sa.Integer())) + session = db.Session(bind=bind) + + for slc in session.query(Slice).all(): + if slc.druid_datasource_id: + slc.datasource_id = slc.druid_datasource_id + if slc.table_id: + slc.datasource_id = slc.table_id + session.merge(slc) + session.commit() + session.close() + + +def downgrade(): + bind = op.get_bind() + session = db.Session(bind=bind) + for slc in session.query(Slice).all(): + if slc.datasource_type == 'druid': + slc.druid_datasource_id = slc.datasource_id + if slc.datasource_type == 'table': + slc.table_id = slc.datasource_id + session.merge(slc) + session.commit() + session.close() + op.drop_column('slices', 'datasource_id') diff --git a/caravel/migrations/versions/b347b202819b_.py b/caravel/migrations/versions/b347b202819b_.py new file mode 100644 index 0000000000000..e73751814879a --- /dev/null +++ b/caravel/migrations/versions/b347b202819b_.py @@ -0,0 +1,19 @@ +"""empty message + +Revision ID: b347b202819b +Revises: ('33d996bcc382', '65903709c321') +Create Date: 2016-09-19 17:22:40.138601 + +""" + +# revision identifiers, used by Alembic. +revision = 'b347b202819b' +down_revision = ('33d996bcc382', '65903709c321') + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/caravel/models.py b/caravel/models.py index e885c95ff1caf..eaa9ec3f9dbd8 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -49,7 +49,7 @@ from werkzeug.datastructures import ImmutableMultiDict import caravel -from caravel import app, db, get_session, utils, sm +from caravel import app, db, get_session, utils, sm, src_registry from caravel.viz import viz_types from caravel.utils import flasher, MetricPermException, DimSelector @@ -156,8 +156,7 @@ class Slice(Model, AuditMixinNullable): __tablename__ = 'slices' id = Column(Integer, primary_key=True) slice_name = Column(String(250)) - druid_datasource_id = Column(Integer, ForeignKey('datasources.id')) - table_id = Column(Integer, ForeignKey('tables.id')) + datasource_id = Column(Integer) datasource_type = Column(String(200)) datasource_name = Column(String(2000)) viz_type = Column(String(250)) @@ -165,33 +164,34 @@ class Slice(Model, AuditMixinNullable): description = Column(Text) cache_timeout = Column(Integer) perm = Column(String(2000)) - - table = relationship( - 'SqlaTable', foreign_keys=[table_id], backref='slices') - druid_datasource = relationship( - 'DruidDatasource', foreign_keys=[druid_datasource_id], backref='slices') owners = relationship("User", secondary=slice_user) def __repr__(self): return self.slice_name + @property + def cls_model(self): + return src_registry.sources[self.datasource_type] + @property def datasource(self): - return self.table or self.druid_datasource + return self.get_datasource + + @datasource.getter + @utils.memoized + def get_datasource(self): + ds = db.session.query( + self.cls_model).filter_by( + id=self.datasource_id).first() + return ds @renders('datasource_name') def datasource_link(self): - if self.table: - return self.table.link - elif self.druid_datasource: - return self.druid_datasource.link + return self.datasource.link @property def datasource_edit_url(self): - if self.table: - return self.table.url - elif self.druid_datasource: - return self.druid_datasource.url + self.datasource.url @property @utils.memoized @@ -204,10 +204,6 @@ def viz(self): def description_markeddown(self): return utils.markdown(self.description) - @property - def datasource_id(self): - return self.table_id or self.druid_datasource_id - @property def data(self): """Data used to render slice in templates""" @@ -283,12 +279,8 @@ def get_viz(self, url_params_multidict=None): def set_perm(mapper, connection, target): # noqa - if target.table_id: - src_class = SqlaTable - id_ = target.table_id - elif target.druid_datasource_id: - src_class = DruidDatasource - id_ = target.druid_datasource_id + src_class = target.cls_model + id_ = target.datasource_id ds = db.session.query(src_class).filter_by(id=int(id_)).first() target.perm = ds.perm diff --git a/caravel/source_registry.py b/caravel/source_registry.py new file mode 100644 index 0000000000000..5e253aa3d3370 --- /dev/null +++ b/caravel/source_registry.py @@ -0,0 +1,15 @@ +from flask import flash + + +class SourceRegistry(object): + """ Central Registry for all available datasource engines""" + + sources = {} + + def add_source(self, ds_type, cls_model): + if ds_type not in self.sources: + self.sources[ds_type] = cls_model + if self.sources[ds_type] is not cls_model: + raise Exception( + 'source type: {} is already associated with Model: {}'.format( + ds_type, self.sources[ds_type])) diff --git a/caravel/utils.py b/caravel/utils.py index 0e65767e7ab6e..9e2c7f8d780e9 100644 --- a/caravel/utils.py +++ b/caravel/utils.py @@ -410,6 +410,14 @@ def readfile(filepath): return content +def register_sources(datasources, module_datasource_map, registry): + for m in datasources: + datasource_list = module_datasource_map[m] + for ds in datasource_list: + ds_class = getattr(datasources[m], ds) + registry.add_source(ds_class.type, ds_class) + + def generic_find_constraint_name(table, columns, referenced, db): """Utility to find a constraint name in alembic migrations""" t = sa.Table(table, db.metadata, autoload=True, autoload_with=db.engine) diff --git a/caravel/views.py b/caravel/views.py index acc48dead9848..1e842a4f80a9e 100755 --- a/caravel/views.py +++ b/caravel/views.py @@ -33,7 +33,8 @@ import caravel from caravel import ( - appbuilder, cache, db, models, viz, utils, app, sm, ascii_art, sql_lab + appbuilder, cache, db, models, viz, utils, app, + sm, ascii_art, sql_lab, src_registry ) config = app.config @@ -675,8 +676,7 @@ class SliceModelView(CaravelModelView, DeleteMixin): # noqa list_columns = [ 'slice_link', 'viz_type', 'datasource_link', 'creator', 'modified'] edit_columns = [ - 'slice_name', 'description', 'viz_type', 'druid_datasource', - 'table', 'owners', 'dashboards', 'params', 'cache_timeout'] + 'slice_name', 'description', 'viz_type', 'owners', 'dashboards', 'params', 'cache_timeout'] base_order = ('changed_on', 'desc') description_columns = { 'description': Markup( @@ -722,18 +722,13 @@ def add(self): if not widget: return redirect(self.get_redirect()) - a_druid_datasource = db.session.query(models.DruidDatasource).first() - if a_druid_datasource is not None: - url = "/druiddatasourcemodelview/list/" - msg = _( - "Click on a datasource link to create a Slice, " - "or click on a table link " - "here " - "to create a Slice for a table" - ) - else: - url = "/tablemodelview/list/" - msg = _("Click on a table link to create a Slice") + sources = src_registry.sources + for source in sources: + ds = db.session.query(src_registry.sources[source]).first() + if ds is not None: + url = "/{}/list/".format(ds.baselink) + msg = _("Click on a {} link to create a Slice".format(source)) + break redirect_url = "/r/msg/?url={}&msg={}".format(url, msg) return redirect(redirect_url) @@ -978,8 +973,8 @@ class Caravel(BaseCaravelView): @log_this def explore(self, datasource_type, datasource_id, slice_id=None): error_redirect = '/slicemodelview/list/' - datasource_class = models.SqlaTable \ - if datasource_type == "table" else models.DruidDatasource + datasource_class = src_registry.sources[datasource_type] + datasources = ( db.session .query(datasource_class) @@ -1093,12 +1088,8 @@ def save_or_overwrite_slice( if k not in as_list and isinstance(v, list): d[k] = v[0] - table_id = druid_datasource_id = None datasource_type = args.get('datasource_type') - if datasource_type in ('datasource', 'druid'): - druid_datasource_id = args.get('datasource_id') - elif datasource_type == 'table': - table_id = args.get('datasource_id') + datasource_id = args.get('datasource_id') if action in ('saveas'): d.pop('slice_id') # don't save old slice_id @@ -1107,9 +1098,8 @@ def save_or_overwrite_slice( slc.params = json.dumps(d, indent=4, sort_keys=True) slc.datasource_name = args.get('datasource_name') slc.viz_type = args.get('viz_type') - slc.druid_datasource_id = druid_datasource_id - slc.table_id = table_id slc.datasource_type = datasource_type + slc.datasource_id = datasource_id slc.slice_name = slice_name if action in ('saveas') and slice_add_perm: @@ -1330,7 +1320,9 @@ def warm_up_cache(self): json_error_response(__( "Table %(t)s wasn't found in the database %(d)s", t=table_name, s=db_name), status=404) - slices = table.slices + slices = session.query(models.Slice).filter_by( + datasource_id=table.id, + datasource_type=table.type).all() for slice in slices: try: diff --git a/tests/core_tests.py b/tests/core_tests.py index 01a7a81113d63..2b24357997016 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -210,32 +210,6 @@ def test_add_slices(self, username='admin'): assert new_slice in dash.slices assert len(set(dash.slices)) == len(dash.slices) - def test_add_slice_redirect_to_sqla(self, username='admin'): - self.login(username=username) - url = '/slicemodelview/add' - resp = self.client.get(url, follow_redirects=True) - assert ( - "Click on a table link to create a Slice" in - resp.data.decode('utf-8') - ) - - def test_add_slice_redirect_to_druid(self, username='admin'): - datasource = DruidDatasource( - datasource_name="datasource_name", - ) - db.session.add(datasource) - db.session.commit() - - self.login(username=username) - url = '/slicemodelview/add' - resp = self.client.get(url, follow_redirects=True) - assert ( - "Click on a datasource link to create a Slice" - in resp.data.decode('utf-8') - ) - - db.session.delete(datasource) - db.session.commit() def test_druid_sync_from_config(self): cluster = models.DruidCluster(cluster_name="new_druid")