Skip to content
This repository was archived by the owner on May 17, 2022. It is now read-only.

Commit cb4d81d

Browse files
Marina Samueljezdez
Marina Samuel
authored andcommittedMar 13, 2019
Schema Viewer Drawer (getredash#3291)
* Process extra column metadata for a few sql-based data sources. * Add Table and Column metadata tables. * Periodically update table and column schema tables in a celery task. * Fetching schema returns data from table and column metadata tables. * Add tests for backend changes. * Front-end shows extra table metadata and uses new schema response. * Delete datasource schema data when deleting a data source. * Process and store data source schema when a data source is first created or after a migration. * Tables should have a unique name per datasource. * Addressing review comments. * Update migration file for mixins. * Appease PEP8 * Upgrade migration file for rebase. * Cascade delete. * Adding org_id * Remove redundant column and table prefixes. * Non-existing tables and columns should be filtered out on the server side not client side. * Fetching table samples should be optional and should happen in a separate task per table. * Allow users to force a schema refresh. * Use updated_at to help prune old schema metadata periodically. * Using settings.SCHEMAS_REFRESH_QUEUE
1 parent adf935b commit cb4d81d

24 files changed

+765
-80
lines changed
 

‎client/app/assets/less/ant.less

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
@import '~antd/lib/radio/style/index';
1515
@import '~antd/lib/time-picker/style/index';
1616
@import '~antd/lib/pagination/style/index';
17+
@import '~antd/lib/drawer/style/index';
1718
@import '~antd/lib/table/style/index';
1819
@import '~antd/lib/popover/style/index';
1920
@import '~antd/lib/icon/style/index';

‎client/app/assets/less/inc/schema-browser.less

+9-5
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ div.table-name {
77
border-radius: @redash-radius;
88
position: relative;
99

10-
.copy-to-editor {
10+
.copy-to-editor, .info {
1111
display: none;
1212
}
1313

1414
&:hover {
1515
background: fade(@redash-gray, 10%);
1616

17-
.copy-to-editor {
17+
.copy-to-editor, .info {
1818
display: flex;
1919
}
2020
}
@@ -36,7 +36,7 @@ div.table-name {
3636
background: transparent;
3737
}
3838

39-
.copy-to-editor {
39+
.copy-to-editor, .info {
4040
color: fade(@redash-gray, 90%);
4141
cursor: pointer;
4242
position: absolute;
@@ -49,21 +49,25 @@ div.table-name {
4949
justify-content: center;
5050
}
5151

52+
.info {
53+
right: 20px
54+
}
55+
5256
.table-open {
5357
padding: 0 22px 0 26px;
5458
overflow: hidden;
5559
text-overflow: ellipsis;
5660
white-space: nowrap;
5761
position: relative;
5862

59-
.copy-to-editor {
63+
.copy-to-editor, .info {
6064
display: none;
6165
}
6266

6367
&:hover {
6468
background: fade(@redash-gray, 10%);
6569

66-
.copy-to-editor {
70+
.copy-to-editor, .info {
6771
display: flex;
6872
}
6973
}

‎client/app/components/proptypes.js

+7
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,13 @@ export const DataSource = PropTypes.shape({
1111
type_name: PropTypes.string,
1212
});
1313

14+
export const DataSourceMetadata = PropTypes.shape({
15+
key: PropTypes.number,
16+
name: PropTypes.string,
17+
type: PropTypes.string,
18+
example: PropTypes.string,
19+
});
20+
1421
export const Table = PropTypes.shape({
1522
columns: PropTypes.arrayOf(PropTypes.string).isRequired,
1623
});
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import React from 'react';
2+
import PropTypes from 'prop-types';
3+
import { react2angular } from 'react2angular';
4+
import Drawer from 'antd/lib/drawer';
5+
import Table from 'antd/lib/table';
6+
7+
import { DataSourceMetadata } from '@/components/proptypes';
8+
9+
class SchemaData extends React.PureComponent {
10+
static propTypes = {
11+
show: PropTypes.bool.isRequired,
12+
onClose: PropTypes.func.isRequired,
13+
tableName: PropTypes.string,
14+
tableMetadata: PropTypes.arrayOf(DataSourceMetadata),
15+
};
16+
17+
static defaultProps = {
18+
tableName: '',
19+
tableMetadata: [],
20+
};
21+
22+
render() {
23+
const columns = [{
24+
title: 'Column Name',
25+
dataIndex: 'name',
26+
width: 400,
27+
key: 'name',
28+
}, {
29+
title: 'Column Type',
30+
dataIndex: 'type',
31+
width: 400,
32+
key: 'type',
33+
}, {
34+
title: 'Example',
35+
dataIndex: 'example',
36+
width: 400,
37+
key: 'example',
38+
}];
39+
40+
return (
41+
<Drawer
42+
title={this.props.tableName}
43+
closable={false}
44+
placement="bottom"
45+
height={500}
46+
onClose={this.props.onClose}
47+
visible={this.props.show}
48+
>
49+
<Table
50+
dataSource={this.props.tableMetadata}
51+
pagination={false}
52+
scroll={{ y: 350 }}
53+
size="small"
54+
columns={columns}
55+
/>
56+
</Drawer>
57+
);
58+
}
59+
}
60+
61+
export default function init(ngModule) {
62+
ngModule.component('schemaData', react2angular(SchemaData, null, []));
63+
}
64+
65+
init.init = true;

‎client/app/components/queries/schema-browser.html

+10-2
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,23 @@
1616
<span title="{{table.name}}">{{table.name}}</span>
1717
<span ng-if="table.size !== undefined"> ({{table.size}})</span>
1818
</strong>
19+
<i ng-if="table.hasColumnMetadata" class="fa fa-question-circle info" title="More Info" aria-hidden="true"
20+
ng-click="openSchemaInfo($event, table.name, table.columns)"></i>
1921
<i class="fa fa-angle-double-right copy-to-editor" aria-hidden="true"
2022
ng-click="$ctrl.itemSelected($event, [table.name])"></i>
2123
</div>
2224
<div uib-collapse="table.collapsed">
23-
<div ng-repeat="column in table.columns | filter:$ctrl.schemaFilterColumn track by column" class="table-open">{{column}}
25+
<div ng-repeat="column in table.columns | filter:$ctrl.schemaFilterColumn track by column.key" class="table-open">{{column.name}}
2426
<i class="fa fa-angle-double-right copy-to-editor" aria-hidden="true"
25-
ng-click="$ctrl.itemSelected($event, [column])"></i>
27+
ng-click="$ctrl.itemSelected($event, [column.name])"></i>
2628
</div>
2729
</div>
2830
</div>
2931
</div>
32+
<schema-data
33+
show="showSchemaInfo"
34+
table-name="tableName"
35+
table-metadata="tableMetadata"
36+
on-close="closeSchemaInfo"
37+
></schema-data>
3038
</div>

‎client/app/components/queries/schema-browser.js

+11
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,17 @@ function SchemaBrowserCtrl($rootScope, $scope) {
88
$scope.$broadcast('vsRepeatTrigger');
99
};
1010

11+
$scope.showSchemaInfo = false;
12+
$scope.openSchemaInfo = ($event, tableName, tableMetadata) => {
13+
$scope.tableName = tableName;
14+
$scope.tableMetadata = tableMetadata;
15+
$scope.showSchemaInfo = true;
16+
$event.stopPropagation();
17+
};
18+
$scope.closeSchemaInfo = () => {
19+
$scope.$apply(() => { $scope.showSchemaInfo = false; });
20+
};
21+
1122
this.getSize = (table) => {
1223
let size = 22;
1324

‎migrations/versions/280daa582976_.py

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
"""Add column metadata and table metadata
2+
3+
Revision ID: 280daa582976
4+
Revises: e5c7a4e2df4d
5+
Create Date: 2019-01-24 18:23:53.040608
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
11+
12+
# revision identifiers, used by Alembic.
13+
revision = '280daa582976'
14+
down_revision = 'e5c7a4e2df4d'
15+
branch_labels = None
16+
depends_on = None
17+
18+
19+
def upgrade():
20+
op.create_table(
21+
'table_metadata',
22+
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
23+
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
24+
sa.Column('id', sa.Integer(), nullable=False),
25+
sa.Column('org_id', sa.Integer(), nullable=False),
26+
sa.Column('data_source_id', sa.Integer(), nullable=False),
27+
sa.Column('exists', sa.Boolean(), nullable=False),
28+
sa.Column('name', sa.String(length=255), nullable=False),
29+
sa.Column('description', sa.String(length=4096), nullable=True),
30+
sa.Column('column_metadata', sa.Boolean(), nullable=False),
31+
sa.Column('sample_query', sa.Text(), nullable=True),
32+
sa.ForeignKeyConstraint(['data_source_id'], ['data_sources.id'], ondelete="CASCADE"),
33+
sa.ForeignKeyConstraint(['org_id'], ['organizations.id.id']),
34+
sa.PrimaryKeyConstraint('id')
35+
)
36+
op.create_table(
37+
'column_metadata',
38+
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
39+
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
40+
sa.Column('id', sa.Integer(), nullable=False),
41+
sa.Column('org_id', sa.Integer(), nullable=False),
42+
sa.Column('table_id', sa.Integer(), nullable=False),
43+
sa.Column('name', sa.String(length=255), nullable=False),
44+
sa.Column('type', sa.String(length=255), nullable=True),
45+
sa.Column('example', sa.String(length=4096), nullable=True),
46+
sa.Column('exists', sa.Boolean(), nullable=False),
47+
sa.ForeignKeyConstraint(['table_id'], ['table_metadata.id'], ondelete="CASCADE"),
48+
sa.ForeignKeyConstraint(['org_id'], ['organizations.id.id']),
49+
sa.PrimaryKeyConstraint('id')
50+
)
51+
52+
53+
def downgrade():
54+
op.drop_table('column_metadata')
55+
op.drop_table('table_metadata')

‎redash/handlers/data_sources.py

+11-3
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,11 @@
66
from six import text_type
77
from sqlalchemy.exc import IntegrityError
88

9-
from redash import models
9+
from redash import models, settings
1010
from redash.handlers.base import BaseResource, get_object_or_404
1111
from redash.permissions import (require_access, require_admin,
1212
require_permission, view_only)
13+
from redash.tasks.queries import refresh_schemas
1314
from redash.query_runner import (get_configuration_schema_for_query_runner_type,
1415
query_runners, NotSupported)
1516
from redash.utils import filter_none
@@ -52,6 +53,9 @@ def post(self, data_source_id):
5253
data_source.name = req['name']
5354
models.db.session.add(data_source)
5455

56+
# Refresh the stored schemas when a data source is updated
57+
refresh_schemas.apply_async(queue=settings.SCHEMAS_REFRESH_QUEUE)
58+
5559
try:
5660
models.db.session.commit()
5761
except IntegrityError as e:
@@ -129,6 +133,9 @@ def post(self):
129133
options=config)
130134

131135
models.db.session.commit()
136+
137+
# Refresh the stored schemas when a new data source is added to the list
138+
refresh_schemas.apply_async(queue=settings.SCHEMAS_REFRESH_QUEUE)
132139
except IntegrityError as e:
133140
if req['name'] in e.message:
134141
abort(400, message="Data source with the name {} already exists.".format(req['name']))
@@ -151,9 +158,10 @@ def get(self, data_source_id):
151158
refresh = request.args.get('refresh') is not None
152159

153160
response = {}
154-
155161
try:
156-
response['schema'] = data_source.get_schema(refresh)
162+
if refresh:
163+
refresh_schemas.apply(queue=settings.SCHEMAS_REFRESH_QUEUE)
164+
response['schema'] = data_source.get_schema()
157165
except NotSupported:
158166
response['error'] = {
159167
'code': 1,

0 commit comments

Comments
 (0)
This repository has been archived.