Schema Viewer Drawer (getredash#3291)

* Process extra column metadata for a few sql-based data sources.

* Add Table and Column metadata tables.

* Periodically update table and column schema tables in a celery task.

* Fetching schema returns data from table and column metadata tables.

* Add tests for backend changes.

* Front-end shows extra table metadata and uses new schema response.

* Delete datasource schema data when deleting a data source.

* Process and store data source schema when a data source is first created or after a migration.

* Tables should have a unique name per datasource.

* Addressing review comments.

* Update migration file for mixins.

* Appease PEP8

* Upgrade migration file for rebase.

* Cascade delete.

* Adding org_id

* Remove redundant column and table prefixes.

* Non-existing tables and columns should be filtered out on the server side not client side.

* Fetching table samples should be optional and should happen in a separate task per table.

* Allow users to force a schema refresh.

* Use updated_at to help prune old schema metadata periodically.

* Using settings.SCHEMAS_REFRESH_QUEUE

Author: Marina Samuel

client/app/assets/less/ant.less

@@ -14,6 +14,7 @@
 @import '~antd/lib/radio/style/index';
 @import '~antd/lib/time-picker/style/index';
 @import '~antd/lib/pagination/style/index';
+@import '~antd/lib/drawer/style/index';
 @import '~antd/lib/table/style/index';
 @import '~antd/lib/popover/style/index';
 @import '~antd/lib/icon/style/index';

client/app/assets/less/inc/schema-browser.less

@@ -7,14 +7,14 @@ div.table-name {
   border-radius: @redash-radius;
   position: relative;
 
-  .copy-to-editor {
+  .copy-to-editor, .info {
     display: none;
   }
 
   &:hover {
     background: fade(@redash-gray, 10%);
 
-    .copy-to-editor {
+    .copy-to-editor, .info {
       display: flex;
     }
   }
@@ -36,7 +36,7 @@ div.table-name {
     background: transparent;
   }
 
-  .copy-to-editor {
+  .copy-to-editor, .info {
     color: fade(@redash-gray, 90%);
     cursor: pointer;
     position: absolute;
@@ -49,21 +49,25 @@ div.table-name {
     justify-content: center;
   }
 
+  .info {
+    right: 20px
+  }
+
   .table-open {
     padding: 0 22px 0 26px;
     overflow: hidden;
     text-overflow: ellipsis;
     white-space: nowrap;
     position: relative;
 
-    .copy-to-editor {
+    .copy-to-editor, .info {
       display: none;
     }
 
     &:hover {
       background: fade(@redash-gray, 10%);
 
-      .copy-to-editor {
+      .copy-to-editor, .info {
         display: flex;
       }
     }

client/app/components/proptypes.js

@@ -11,6 +11,13 @@ export const DataSource = PropTypes.shape({
   type_name: PropTypes.string,
 });
 
+export const DataSourceMetadata = PropTypes.shape({
+  key: PropTypes.number,
+  name: PropTypes.string,
+  type: PropTypes.string,
+  example: PropTypes.string,
+});
+
 export const Table = PropTypes.shape({
   columns: PropTypes.arrayOf(PropTypes.string).isRequired,
 });

client/app/components/queries/SchemaData.jsx

@@ -0,0 +1,65 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { react2angular } from 'react2angular';
+import Drawer from 'antd/lib/drawer';
+import Table from 'antd/lib/table';
+
+import { DataSourceMetadata } from '@/components/proptypes';
+
+class SchemaData extends React.PureComponent {
+  static propTypes = {
+    show: PropTypes.bool.isRequired,
+    onClose: PropTypes.func.isRequired,
+    tableName: PropTypes.string,
+    tableMetadata: PropTypes.arrayOf(DataSourceMetadata),
+  };
+
+  static defaultProps = {
+    tableName: '',
+    tableMetadata: [],
+  };
+
+  render() {
+    const columns = [{
+      title: 'Column Name',
+      dataIndex: 'name',
+      width: 400,
+      key: 'name',
+    }, {
+      title: 'Column Type',
+      dataIndex: 'type',
+      width: 400,
+      key: 'type',
+    }, {
+      title: 'Example',
+      dataIndex: 'example',
+      width: 400,
+      key: 'example',
+    }];
+
+    return (
+      <Drawer
+        title={this.props.tableName}
+        closable={false}
+        placement="bottom"
+        height={500}
+        onClose={this.props.onClose}
+        visible={this.props.show}
+      >
+        <Table
+          dataSource={this.props.tableMetadata}
+          pagination={false}
+          scroll={{ y: 350 }}
+          size="small"
+          columns={columns}
+        />
+      </Drawer>
+    );
+  }
+}
+
+export default function init(ngModule) {
+  ngModule.component('schemaData', react2angular(SchemaData, null, []));
+}
+
+init.init = true;

client/app/components/queries/schema-browser.html

@@ -16,15 +16,23 @@
           <span title="{{table.name}}">{{table.name}}</span>
           <span ng-if="table.size !== undefined"> ({{table.size}})</span>
         </strong>
+        <i ng-if="table.hasColumnMetadata" class="fa fa-question-circle info" title="More Info" aria-hidden="true"
+          ng-click="openSchemaInfo($event, table.name, table.columns)"></i>
         <i class="fa fa-angle-double-right copy-to-editor" aria-hidden="true"
           ng-click="$ctrl.itemSelected($event, [table.name])"></i>
       </div>
       <div uib-collapse="table.collapsed">
-        <div ng-repeat="column in table.columns | filter:$ctrl.schemaFilterColumn track by column" class="table-open">{{column}}
+        <div ng-repeat="column in table.columns | filter:$ctrl.schemaFilterColumn track by column.key" class="table-open">{{column.name}}
           <i class="fa fa-angle-double-right copy-to-editor" aria-hidden="true"
-            ng-click="$ctrl.itemSelected($event, [column])"></i>
+            ng-click="$ctrl.itemSelected($event, [column.name])"></i>
         </div>
       </div>
     </div>
   </div>
+  <schema-data
+    show="showSchemaInfo"
+    table-name="tableName"
+    table-metadata="tableMetadata"
+    on-close="closeSchemaInfo"
+  ></schema-data>
 </div>

client/app/components/queries/schema-browser.js

@@ -8,6 +8,17 @@ function SchemaBrowserCtrl($rootScope, $scope) {
     $scope.$broadcast('vsRepeatTrigger');
   };
 
+  $scope.showSchemaInfo = false;
+  $scope.openSchemaInfo = ($event, tableName, tableMetadata) => {
+    $scope.tableName = tableName;
+    $scope.tableMetadata = tableMetadata;
+    $scope.showSchemaInfo = true;
+    $event.stopPropagation();
+  };
+  $scope.closeSchemaInfo = () => {
+    $scope.$apply(() => { $scope.showSchemaInfo = false; });
+  };
+
   this.getSize = (table) => {
     let size = 22;
 

migrations/versions/280daa582976_.py

@@ -0,0 +1,55 @@
+"""Add column metadata and table metadata
+
+Revision ID: 280daa582976
+Revises: e5c7a4e2df4d
+Create Date: 2019-01-24 18:23:53.040608
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '280daa582976'
+down_revision = 'e5c7a4e2df4d'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        'table_metadata',
+        sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
+        sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('org_id', sa.Integer(), nullable=False),
+        sa.Column('data_source_id', sa.Integer(), nullable=False),
+        sa.Column('exists', sa.Boolean(), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False),
+        sa.Column('description', sa.String(length=4096), nullable=True),
+        sa.Column('column_metadata', sa.Boolean(), nullable=False),
+        sa.Column('sample_query', sa.Text(), nullable=True),
+        sa.ForeignKeyConstraint(['data_source_id'], ['data_sources.id'], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(['org_id'], ['organizations.id.id']),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table(
+        'column_metadata',
+        sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
+        sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('org_id', sa.Integer(), nullable=False),
+        sa.Column('table_id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(length=255), nullable=False),
+        sa.Column('type', sa.String(length=255), nullable=True),
+        sa.Column('example', sa.String(length=4096), nullable=True),
+        sa.Column('exists', sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(['table_id'], ['table_metadata.id'], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(['org_id'], ['organizations.id.id']),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+
+def downgrade():
+    op.drop_table('column_metadata')
+    op.drop_table('table_metadata')

redash/handlers/data_sources.py

@@ -6,10 +6,11 @@
 from six import text_type
 from sqlalchemy.exc import IntegrityError
 
-from redash import models
+from redash import models, settings
 from redash.handlers.base import BaseResource, get_object_or_404
 from redash.permissions import (require_access, require_admin,
                                 require_permission, view_only)
+from redash.tasks.queries import refresh_schemas
 from redash.query_runner import (get_configuration_schema_for_query_runner_type,
                                  query_runners, NotSupported)
 from redash.utils import filter_none
@@ -52,6 +53,9 @@ def post(self, data_source_id):
         data_source.name = req['name']
         models.db.session.add(data_source)
 
+        # Refresh the stored schemas when a data source is updated
+        refresh_schemas.apply_async(queue=settings.SCHEMAS_REFRESH_QUEUE)
+
         try:
             models.db.session.commit()
         except IntegrityError as e:
@@ -129,6 +133,9 @@ def post(self):
                                                              options=config)
 
             models.db.session.commit()
+
+            # Refresh the stored schemas when a new data source is added to the list
+            refresh_schemas.apply_async(queue=settings.SCHEMAS_REFRESH_QUEUE)
         except IntegrityError as e:
             if req['name'] in e.message:
                 abort(400, message="Data source with the name {} already exists.".format(req['name']))
@@ -151,9 +158,10 @@ def get(self, data_source_id):
         refresh = request.args.get('refresh') is not None
 
         response = {}
-
         try:
-            response['schema'] = data_source.get_schema(refresh)
+            if refresh:
+                refresh_schemas.apply(queue=settings.SCHEMAS_REFRESH_QUEUE)
+            response['schema'] = data_source.get_schema()
         except NotSupported:
             response['error'] = {
                 'code': 1,