Skip to content
This repository was archived by the owner on May 17, 2022. It is now read-only.

Commit cb4d81d

Browse files
Marina Samueljezdez
Marina Samuel
authored andcommittedMar 13, 2019
Schema Viewer Drawer (getredash#3291)
* Process extra column metadata for a few sql-based data sources. * Add Table and Column metadata tables. * Periodically update table and column schema tables in a celery task. * Fetching schema returns data from table and column metadata tables. * Add tests for backend changes. * Front-end shows extra table metadata and uses new schema response. * Delete datasource schema data when deleting a data source. * Process and store data source schema when a data source is first created or after a migration. * Tables should have a unique name per datasource. * Addressing review comments. * Update migration file for mixins. * Appease PEP8 * Upgrade migration file for rebase. * Cascade delete. * Adding org_id * Remove redundant column and table prefixes. * Non-existing tables and columns should be filtered out on the server side not client side. * Fetching table samples should be optional and should happen in a separate task per table. * Allow users to force a schema refresh. * Use updated_at to help prune old schema metadata periodically. * Using settings.SCHEMAS_REFRESH_QUEUE
1 parent adf935b commit cb4d81d

24 files changed

+765
-80
lines changed
 

‎client/app/assets/less/ant.less

+1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
@import '~antd/lib/radio/style/index';
1515
@import '~antd/lib/time-picker/style/index';
1616
@import '~antd/lib/pagination/style/index';
17+
@import '~antd/lib/drawer/style/index';
1718
@import '~antd/lib/table/style/index';
1819
@import '~antd/lib/popover/style/index';
1920
@import '~antd/lib/icon/style/index';

‎client/app/assets/less/inc/schema-browser.less

+9-5
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ div.table-name {
77
border-radius: @redash-radius;
88
position: relative;
99

10-
.copy-to-editor {
10+
.copy-to-editor, .info {
1111
display: none;
1212
}
1313

1414
&:hover {
1515
background: fade(@redash-gray, 10%);
1616

17-
.copy-to-editor {
17+
.copy-to-editor, .info {
1818
display: flex;
1919
}
2020
}
@@ -36,7 +36,7 @@ div.table-name {
3636
background: transparent;
3737
}
3838

39-
.copy-to-editor {
39+
.copy-to-editor, .info {
4040
color: fade(@redash-gray, 90%);
4141
cursor: pointer;
4242
position: absolute;
@@ -49,21 +49,25 @@ div.table-name {
4949
justify-content: center;
5050
}
5151

52+
.info {
53+
right: 20px
54+
}
55+
5256
.table-open {
5357
padding: 0 22px 0 26px;
5458
overflow: hidden;
5559
text-overflow: ellipsis;
5660
white-space: nowrap;
5761
position: relative;
5862

59-
.copy-to-editor {
63+
.copy-to-editor, .info {
6064
display: none;
6165
}
6266

6367
&:hover {
6468
background: fade(@redash-gray, 10%);
6569

66-
.copy-to-editor {
70+
.copy-to-editor, .info {
6771
display: flex;
6872
}
6973
}

‎client/app/components/proptypes.js

+7
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,13 @@ export const DataSource = PropTypes.shape({
1111
type_name: PropTypes.string,
1212
});
1313

14+
export const DataSourceMetadata = PropTypes.shape({
15+
key: PropTypes.number,
16+
name: PropTypes.string,
17+
type: PropTypes.string,
18+
example: PropTypes.string,
19+
});
20+
1421
export const Table = PropTypes.shape({
1522
columns: PropTypes.arrayOf(PropTypes.string).isRequired,
1623
});
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import React from 'react';
2+
import PropTypes from 'prop-types';
3+
import { react2angular } from 'react2angular';
4+
import Drawer from 'antd/lib/drawer';
5+
import Table from 'antd/lib/table';
6+
7+
import { DataSourceMetadata } from '@/components/proptypes';
8+
9+
class SchemaData extends React.PureComponent {
10+
static propTypes = {
11+
show: PropTypes.bool.isRequired,
12+
onClose: PropTypes.func.isRequired,
13+
tableName: PropTypes.string,
14+
tableMetadata: PropTypes.arrayOf(DataSourceMetadata),
15+
};
16+
17+
static defaultProps = {
18+
tableName: '',
19+
tableMetadata: [],
20+
};
21+
22+
render() {
23+
const columns = [{
24+
title: 'Column Name',
25+
dataIndex: 'name',
26+
width: 400,
27+
key: 'name',
28+
}, {
29+
title: 'Column Type',
30+
dataIndex: 'type',
31+
width: 400,
32+
key: 'type',
33+
}, {
34+
title: 'Example',
35+
dataIndex: 'example',
36+
width: 400,
37+
key: 'example',
38+
}];
39+
40+
return (
41+
<Drawer
42+
title={this.props.tableName}
43+
closable={false}
44+
placement="bottom"
45+
height={500}
46+
onClose={this.props.onClose}
47+
visible={this.props.show}
48+
>
49+
<Table
50+
dataSource={this.props.tableMetadata}
51+
pagination={false}
52+
scroll={{ y: 350 }}
53+
size="small"
54+
columns={columns}
55+
/>
56+
</Drawer>
57+
);
58+
}
59+
}
60+
61+
export default function init(ngModule) {
62+
ngModule.component('schemaData', react2angular(SchemaData, null, []));
63+
}
64+
65+
init.init = true;

‎client/app/components/queries/schema-browser.html

+10-2
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,23 @@
1616
<span title="{{table.name}}">{{table.name}}</span>
1717
<span ng-if="table.size !== undefined"> ({{table.size}})</span>
1818
</strong>
19+
<i ng-if="table.hasColumnMetadata" class="fa fa-question-circle info" title="More Info" aria-hidden="true"
20+
ng-click="openSchemaInfo($event, table.name, table.columns)"></i>
1921
<i class="fa fa-angle-double-right copy-to-editor" aria-hidden="true"
2022
ng-click="$ctrl.itemSelected($event, [table.name])"></i>
2123
</div>
2224
<div uib-collapse="table.collapsed">
23-
<div ng-repeat="column in table.columns | filter:$ctrl.schemaFilterColumn track by column" class="table-open">{{column}}
25+
<div ng-repeat="column in table.columns | filter:$ctrl.schemaFilterColumn track by column.key" class="table-open">{{column.name}}
2426
<i class="fa fa-angle-double-right copy-to-editor" aria-hidden="true"
25-
ng-click="$ctrl.itemSelected($event, [column])"></i>
27+
ng-click="$ctrl.itemSelected($event, [column.name])"></i>
2628
</div>
2729
</div>
2830
</div>
2931
</div>
32+
<schema-data
33+
show="showSchemaInfo"
34+
table-name="tableName"
35+
table-metadata="tableMetadata"
36+
on-close="closeSchemaInfo"
37+
></schema-data>
3038
</div>

‎client/app/components/queries/schema-browser.js

+11
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,17 @@ function SchemaBrowserCtrl($rootScope, $scope) {
88
$scope.$broadcast('vsRepeatTrigger');
99
};
1010

11+
$scope.showSchemaInfo = false;
12+
$scope.openSchemaInfo = ($event, tableName, tableMetadata) => {
13+
$scope.tableName = tableName;
14+
$scope.tableMetadata = tableMetadata;
15+
$scope.showSchemaInfo = true;
16+
$event.stopPropagation();
17+
};
18+
$scope.closeSchemaInfo = () => {
19+
$scope.$apply(() => { $scope.showSchemaInfo = false; });
20+
};
21+
1122
this.getSize = (table) => {
1223
let size = 22;
1324

‎migrations/versions/280daa582976_.py

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
"""Add column metadata and table metadata
2+
3+
Revision ID: 280daa582976
4+
Revises: e5c7a4e2df4d
5+
Create Date: 2019-01-24 18:23:53.040608
6+
7+
"""
8+
from alembic import op
9+
import sqlalchemy as sa
10+
11+
12+
# revision identifiers, used by Alembic.
13+
revision = '280daa582976'
14+
down_revision = 'e5c7a4e2df4d'
15+
branch_labels = None
16+
depends_on = None
17+
18+
19+
def upgrade():
20+
op.create_table(
21+
'table_metadata',
22+
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
23+
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
24+
sa.Column('id', sa.Integer(), nullable=False),
25+
sa.Column('org_id', sa.Integer(), nullable=False),
26+
sa.Column('data_source_id', sa.Integer(), nullable=False),
27+
sa.Column('exists', sa.Boolean(), nullable=False),
28+
sa.Column('name', sa.String(length=255), nullable=False),
29+
sa.Column('description', sa.String(length=4096), nullable=True),
30+
sa.Column('column_metadata', sa.Boolean(), nullable=False),
31+
sa.Column('sample_query', sa.Text(), nullable=True),
32+
sa.ForeignKeyConstraint(['data_source_id'], ['data_sources.id'], ondelete="CASCADE"),
33+
sa.ForeignKeyConstraint(['org_id'], ['organizations.id.id']),
34+
sa.PrimaryKeyConstraint('id')
35+
)
36+
op.create_table(
37+
'column_metadata',
38+
sa.Column('updated_at', sa.DateTime(timezone=True), nullable=False),
39+
sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
40+
sa.Column('id', sa.Integer(), nullable=False),
41+
sa.Column('org_id', sa.Integer(), nullable=False),
42+
sa.Column('table_id', sa.Integer(), nullable=False),
43+
sa.Column('name', sa.String(length=255), nullable=False),
44+
sa.Column('type', sa.String(length=255), nullable=True),
45+
sa.Column('example', sa.String(length=4096), nullable=True),
46+
sa.Column('exists', sa.Boolean(), nullable=False),
47+
sa.ForeignKeyConstraint(['table_id'], ['table_metadata.id'], ondelete="CASCADE"),
48+
sa.ForeignKeyConstraint(['org_id'], ['organizations.id.id']),
49+
sa.PrimaryKeyConstraint('id')
50+
)
51+
52+
53+
def downgrade():
54+
op.drop_table('column_metadata')
55+
op.drop_table('table_metadata')

‎redash/handlers/data_sources.py

+11-3
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,11 @@
66
from six import text_type
77
from sqlalchemy.exc import IntegrityError
88

9-
from redash import models
9+
from redash import models, settings
1010
from redash.handlers.base import BaseResource, get_object_or_404
1111
from redash.permissions import (require_access, require_admin,
1212
require_permission, view_only)
13+
from redash.tasks.queries import refresh_schemas
1314
from redash.query_runner import (get_configuration_schema_for_query_runner_type,
1415
query_runners, NotSupported)
1516
from redash.utils import filter_none
@@ -52,6 +53,9 @@ def post(self, data_source_id):
5253
data_source.name = req['name']
5354
models.db.session.add(data_source)
5455

56+
# Refresh the stored schemas when a data source is updated
57+
refresh_schemas.apply_async(queue=settings.SCHEMAS_REFRESH_QUEUE)
58+
5559
try:
5660
models.db.session.commit()
5761
except IntegrityError as e:
@@ -129,6 +133,9 @@ def post(self):
129133
options=config)
130134

131135
models.db.session.commit()
136+
137+
# Refresh the stored schemas when a new data source is added to the list
138+
refresh_schemas.apply_async(queue=settings.SCHEMAS_REFRESH_QUEUE)
132139
except IntegrityError as e:
133140
if req['name'] in e.message:
134141
abort(400, message="Data source with the name {} already exists.".format(req['name']))
@@ -151,9 +158,10 @@ def get(self, data_source_id):
151158
refresh = request.args.get('refresh') is not None
152159

153160
response = {}
154-
155161
try:
156-
response['schema'] = data_source.get_schema(refresh)
162+
if refresh:
163+
refresh_schemas.apply(queue=settings.SCHEMAS_REFRESH_QUEUE)
164+
response['schema'] = data_source.get_schema()
157165
except NotSupported:
158166
response['error'] = {
159167
'code': 1,

‎redash/models/__init__.py

+79-15
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import pytz
88

99
import xlsxwriter
10+
from operator import itemgetter
1011
from six import python_2_unicode_compatible, text_type
1112
from sqlalchemy import distinct, or_, and_, UniqueConstraint
1213
from sqlalchemy.dialects import postgresql
@@ -65,6 +66,62 @@ def get(self, query_id):
6566
scheduled_queries_executions = ScheduledQueriesExecutions()
6667

6768

69+
@python_2_unicode_compatible
70+
class TableMetadata(TimestampMixin, db.Model):
71+
id = Column(db.Integer, primary_key=True)
72+
org_id = Column(db.Integer, db.ForeignKey("organizations.id"))
73+
data_source_id = Column(db.Integer, db.ForeignKey("data_sources.id", ondelete="CASCADE"))
74+
exists = Column(db.Boolean, default=True)
75+
name = Column(db.String(255))
76+
description = Column(db.String(4096), nullable=True)
77+
column_metadata = Column(db.Boolean, default=False)
78+
sample_query = Column("sample_query", db.Text, nullable=True)
79+
80+
__tablename__ = 'table_metadata'
81+
82+
def __str__(self):
83+
return text_type(self.table_name)
84+
85+
def to_dict(self):
86+
return {
87+
'id': self.id,
88+
'org_id': self.org_id,
89+
'data_source_id': self.data_source_id,
90+
'exists': self.exists,
91+
'name': self.name,
92+
'description': self.description,
93+
'column_metadata': self.column_metadata,
94+
'sample_query': self.sample_query,
95+
}
96+
97+
98+
@python_2_unicode_compatible
99+
class ColumnMetadata(TimestampMixin, db.Model):
100+
id = Column(db.Integer, primary_key=True)
101+
org_id = Column(db.Integer, db.ForeignKey("organizations.id"))
102+
table_id = Column(db.Integer, db.ForeignKey("table_metadata.id", ondelete="CASCADE"))
103+
name = Column(db.String(255))
104+
type = Column(db.String(255), nullable=True)
105+
example = Column(db.String(4096), nullable=True)
106+
exists = Column(db.Boolean, default=True)
107+
108+
__tablename__ = 'column_metadata'
109+
110+
def __str__(self):
111+
return text_type(self.name)
112+
113+
def to_dict(self):
114+
return {
115+
'id': self.id,
116+
'org_id': self.org_id,
117+
'table_id': self.table_id,
118+
'name': self.name,
119+
'type': self.type,
120+
'example': self.example,
121+
'exists': self.exists,
122+
}
123+
124+
68125
@python_2_unicode_compatible
69126
@generic_repr('id', 'name', 'type', 'org_id', 'created_at')
70127
class DataSource(BelongsToOrgMixin, db.Model):
@@ -145,22 +202,29 @@ def delete(self):
145202
db.session.commit()
146203
return res
147204

148-
def get_schema(self, refresh=False):
149-
key = "data_source:schema:{}".format(self.id)
150-
151-
cache = None
152-
if not refresh:
153-
cache = redis_connection.get(key)
154-
155-
if cache is None:
156-
query_runner = self.query_runner
157-
schema = sorted(query_runner.get_schema(get_stats=refresh), key=lambda t: t['name'])
158-
159-
redis_connection.set(key, json_dumps(schema))
160-
else:
161-
schema = json_loads(cache)
205+
def get_schema(self):
206+
schema = []
207+
tables = TableMetadata.query.filter(TableMetadata.data_source_id == self.id).all()
208+
for table in tables:
209+
if not table.exists:
210+
continue
162211

163-
return schema
212+
table_info = {
213+
'name': table.name,
214+
'exists': table.exists,
215+
'hasColumnMetadata': table.column_metadata,
216+
'columns': []}
217+
columns = ColumnMetadata.query.filter(ColumnMetadata.table_id == table.id)
218+
table_info['columns'] = sorted([{
219+
'key': column.id,
220+
'name': column.name,
221+
'type': column.type,
222+
'exists': column.exists,
223+
'example': column.example
224+
} for column in columns if column.exists == True], key=itemgetter('name'))
225+
schema.append(table_info)
226+
227+
return sorted(schema, key=itemgetter('name'))
164228

165229
def _pause_key(self):
166230
return 'ds:{}:pause'.format(self.id)

‎redash/query_runner/__init__.py

+22
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ class NotSupported(Exception):
5454

5555
class BaseQueryRunner(object):
5656
noop_query = None
57+
data_sample_query = None
5758

5859
def __init__(self, configuration):
5960
self.syntax = 'sql'
@@ -118,6 +119,27 @@ def _run_query_internal(self, query):
118119
raise Exception("Failed running query [%s]." % query)
119120
return json_loads(results)['rows']
120121

122+
def get_table_sample(self, table_name):
123+
if not self.configuration.get('samples', False):
124+
return {}
125+
126+
if self.data_sample_query is None:
127+
raise NotImplementedError()
128+
129+
query = self.data_sample_query.format(table=table_name)
130+
131+
results, error = self.run_query(query, None)
132+
if error is not None:
133+
raise NotSupported()
134+
135+
rows = json_loads(results).get('rows', [])
136+
if len(rows) > 0:
137+
sample = rows[0]
138+
else:
139+
sample = {}
140+
141+
return sample
142+
121143
@classmethod
122144
def to_dict(cls):
123145
return {

‎redash/query_runner/athena.py

+14-3
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ def format(self, operation, parameters=None):
4343

4444
class Athena(BaseQueryRunner):
4545
noop_query = 'SELECT 1'
46+
data_sample_query = "SELECT * FROM {table} LIMIT 1"
4647

4748
@classmethod
4849
def name(cls):
@@ -78,6 +79,10 @@ def configuration_schema(cls):
7879
'type': 'boolean',
7980
'title': 'Use Glue Data Catalog',
8081
},
82+
'samples': {
83+
'type': 'boolean',
84+
'title': 'Show Data Samples'
85+
},
8186
},
8287
'required': ['region', 's3_staging_dir'],
8388
'order': ['region', 'aws_access_key', 'aws_secret_key', 's3_staging_dir', 'schema'],
@@ -143,7 +148,7 @@ def get_schema(self, get_stats=False):
143148

144149
schema = {}
145150
query = """
146-
SELECT table_schema, table_name, column_name
151+
SELECT table_schema, table_name, column_name, data_type AS column_type
147152
FROM information_schema.columns
148153
WHERE table_schema NOT IN ('information_schema')
149154
"""
@@ -153,11 +158,17 @@ def get_schema(self, get_stats=False):
153158
raise Exception("Failed getting schema.")
154159

155160
results = json_loads(results)
156-
for row in results['rows']:
161+
162+
for i, row in enumerate(results['rows']):
157163
table_name = '{0}.{1}'.format(row['table_schema'], row['table_name'])
158164
if table_name not in schema:
159-
schema[table_name] = {'name': table_name, 'columns': []}
165+
schema[table_name] = {'name': table_name, 'columns': [], 'metadata': []}
166+
160167
schema[table_name]['columns'].append(row['column_name'])
168+
schema[table_name]['metadata'].append({
169+
"name": row['column_name'],
170+
"type": row['column_type'],
171+
})
161172

162173
return schema.values()
163174

‎redash/query_runner/mysql.py

+14-4
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828

2929
class Mysql(BaseSQLQueryRunner):
3030
noop_query = "SELECT 1"
31+
data_sample_query = "SELECT * FROM {table} LIMIT 1"
3132

3233
@classmethod
3334
def configuration_schema(cls):
@@ -54,7 +55,11 @@ def configuration_schema(cls):
5455
'port': {
5556
'type': 'number',
5657
'default': 3306,
57-
}
58+
},
59+
'samples': {
60+
'type': 'boolean',
61+
'title': 'Show Data Samples'
62+
},
5863
},
5964
"order": ['host', 'port', 'user', 'passwd', 'db'],
6065
'required': ['db'],
@@ -100,7 +105,8 @@ def _get_tables(self, schema):
100105
query = """
101106
SELECT col.table_schema as table_schema,
102107
col.table_name as table_name,
103-
col.column_name as column_name
108+
col.column_name as column_name,
109+
col.data_type AS column_type
104110
FROM `information_schema`.`columns` col
105111
WHERE col.table_schema NOT IN ('information_schema', 'performance_schema', 'mysql', 'sys');
106112
"""
@@ -112,16 +118,20 @@ def _get_tables(self, schema):
112118

113119
results = json_loads(results)
114120

115-
for row in results['rows']:
121+
for i, row in enumerate(results['rows']):
116122
if row['table_schema'] != self.configuration['db']:
117123
table_name = u'{}.{}'.format(row['table_schema'], row['table_name'])
118124
else:
119125
table_name = row['table_name']
120126

121127
if table_name not in schema:
122-
schema[table_name] = {'name': table_name, 'columns': []}
128+
schema[table_name] = {'name': table_name, 'columns': [], 'metadata': []}
123129

124130
schema[table_name]['columns'].append(row['column_name'])
131+
schema[table_name]['metadata'].append({
132+
"name": row['column_name'],
133+
"type": row['column_type'],
134+
})
125135

126136
return schema.values()
127137

‎redash/query_runner/pg.py

+20-5
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ def _wait(conn, timeout=None):
6767

6868
class PostgreSQL(BaseSQLQueryRunner):
6969
noop_query = "SELECT 1"
70+
data_sample_query = "SELECT * FROM {table} LIMIT 1"
7071

7172
@classmethod
7273
def configuration_schema(cls):
@@ -95,7 +96,11 @@ def configuration_schema(cls):
9596
"type": "string",
9697
"title": "SSL Mode",
9798
"default": "prefer"
98-
}
99+
},
100+
"samples": {
101+
"type": "boolean",
102+
"title": "Show Data Samples"
103+
},
99104
},
100105
"order": ['host', 'port', 'user', 'password'],
101106
"required": ["dbname"],
@@ -121,9 +126,13 @@ def _get_definitions(self, schema, query):
121126
table_name = row['table_name']
122127

123128
if table_name not in schema:
124-
schema[table_name] = {'name': table_name, 'columns': []}
129+
schema[table_name] = {'name': table_name, 'columns': [], 'metadata': []}
125130

126131
schema[table_name]['columns'].append(row['column_name'])
132+
schema[table_name]['metadata'].append({
133+
"name": row['column_name'],
134+
"type": row['column_type'],
135+
})
127136

128137
def _get_tables(self, schema):
129138
'''
@@ -143,7 +152,8 @@ def _get_tables(self, schema):
143152
query = """
144153
SELECT s.nspname as table_schema,
145154
c.relname as table_name,
146-
a.attname as column_name
155+
a.attname as column_name,
156+
a.atttypid::regtype as column_type
147157
FROM pg_class c
148158
JOIN pg_namespace s
149159
ON c.relnamespace = s.oid
@@ -251,7 +261,11 @@ def configuration_schema(cls):
251261
"type": "string",
252262
"title": "SSL Mode",
253263
"default": "prefer"
254-
}
264+
},
265+
"samples": {
266+
"type": "boolean",
267+
"title": "Show Data Samples"
268+
},
255269
},
256270
"order": ['host', 'port', 'user', 'password'],
257271
"required": ["dbname", "user", "password", "host", "port"],
@@ -271,11 +285,12 @@ def _get_tables(self, schema):
271285
SELECT DISTINCT table_name,
272286
table_schema,
273287
column_name,
288+
data_type AS column_type,
274289
ordinal_position AS pos
275290
FROM svv_columns
276291
WHERE table_schema NOT IN ('pg_internal','pg_catalog','information_schema')
277292
)
278-
SELECT table_name, table_schema, column_name
293+
SELECT table_name, table_schema, column_name, column_type
279294
FROM tables
280295
WHERE
281296
HAS_SCHEMA_PRIVILEGE(table_schema, 'USAGE') AND

‎redash/query_runner/presto.py

+11-4
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131

3232
class Presto(BaseQueryRunner):
3333
noop_query = 'SHOW TABLES'
34+
data_sample_query = "SELECT * FROM {table} LIMIT 1"
3435

3536
@classmethod
3637
def configuration_schema(cls):
@@ -56,6 +57,10 @@ def configuration_schema(cls):
5657
'username': {
5758
'type': 'string'
5859
},
60+
'samples': {
61+
'type': 'boolean',
62+
'title': 'Show Data Samples'
63+
},
5964
},
6065
'order': ['host', 'protocol', 'port', 'username', 'schema', 'catalog'],
6166
'required': ['host']
@@ -72,25 +77,27 @@ def type(cls):
7277
def get_schema(self, get_stats=False):
7378
schema = {}
7479
query = """
75-
SELECT table_schema, table_name, column_name
80+
SELECT table_schema, table_name, column_name, data_type AS column_type
7681
FROM information_schema.columns
7782
WHERE table_schema NOT IN ('pg_catalog', 'information_schema')
7883
"""
7984

8085
results, error = self.run_query(query, None)
81-
8286
if error is not None:
8387
raise Exception("Failed getting schema.")
8488

8589
results = json_loads(results)
8690

8791
for row in results['rows']:
8892
table_name = '{}.{}'.format(row['table_schema'], row['table_name'])
89-
9093
if table_name not in schema:
91-
schema[table_name] = {'name': table_name, 'columns': []}
94+
schema[table_name] = {'name': table_name, 'columns': [], 'metadata': []}
9295

9396
schema[table_name]['columns'].append(row['column_name'])
97+
schema[table_name]['metadata'].append({
98+
"name": row['column_name'],
99+
"type": row['column_type'],
100+
})
94101

95102
return schema.values()
96103

‎redash/settings/__init__.py

+3
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,9 @@ def all_settings():
259259
# Enhance schema fetching
260260
SCHEMA_RUN_TABLE_SIZE_CALCULATIONS = parse_boolean(os.environ.get("REDASH_SCHEMA_RUN_TABLE_SIZE_CALCULATIONS", "false"))
261261

262+
# Frequency of clearing out old schema metadata.
263+
SCHEMA_METADATA_TTL_DAYS = int(os.environ.get("REDASH_SCHEMA_METADATA_TTL_DAYS", 60))
264+
262265
# Allow Parameters in Embeds
263266
# WARNING: With this option enabled, Redash reads query parameters from the request URL (risk of SQL injection!)
264267
ALLOW_PARAMETERS_IN_EMBEDS = parse_boolean(os.environ.get("REDASH_ALLOW_PARAMETERS_IN_EMBEDS", "false"))

‎redash/tasks/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
from .general import record_event, version_check, send_mail, sync_user_details
2-
from .queries import QueryTask, refresh_queries, refresh_schemas, cleanup_query_results, execute_query
2+
from .queries import QueryTask, refresh_queries, refresh_schemas, refresh_schema, cleanup_query_results, execute_query, get_table_sample_data, cleanup_schema_metadata
33
from .alerts import check_alerts_for_query

‎redash/tasks/queries.py

+135-2
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
import logging
22
import signal
33
import time
4+
import datetime
45

56
import redis
67
from celery.exceptions import SoftTimeLimitExceeded, TimeLimitExceeded
78
from celery.result import AsyncResult
89
from celery.utils.log import get_task_logger
910
from six import text_type
11+
from sqlalchemy.orm import load_only
1012

11-
from redash import models, redis_connection, settings, statsd_client
13+
from redash import models, redis_connection, settings, statsd_client, utils
14+
from redash.models import TableMetadata, ColumnMetadata, db
1215
from redash.query_runner import InterruptException
1316
from redash.tasks.alerts import check_alerts_for_query
1417
from redash.utils import gen_query_hash, json_dumps, json_loads, utcnow, mustache_render
@@ -229,13 +232,143 @@ def cleanup_query_results():
229232
logger.info("Deleted %d unused query results.", deleted_count)
230233

231234

235+
@celery.task(name="redash.tasks.get_table_sample_data")
236+
def get_table_sample_data(data_source_id, table, table_id):
237+
ds = models.DataSource.get_by_id(data_source_id)
238+
sample = ds.query_runner.get_table_sample(table['name'])
239+
if not sample:
240+
return
241+
242+
# If a column exists, add a sample to it.
243+
for i, column in enumerate(table['columns']):
244+
persisted_column = ColumnMetadata.query.filter(
245+
ColumnMetadata.name == column,
246+
ColumnMetadata.table_id == table_id,
247+
).options(load_only('id')).first()
248+
249+
if persisted_column:
250+
column_example = str(sample.get(column, None))
251+
if column_example and len(column_example) > 4000:
252+
column_example = u'{}...'.format(column_example[:4000])
253+
254+
ColumnMetadata.query.filter(
255+
ColumnMetadata.id == persisted_column.id,
256+
).update({
257+
'example': column_example,
258+
})
259+
models.db.session.commit()
260+
261+
def cleanup_data_in_table(table_model):
262+
removed_metadata = table_model.query.filter(
263+
table_model.exists == False,
264+
).options(load_only('updated_at'))
265+
266+
for removed_metadata_row in removed_metadata:
267+
is_old_data = (
268+
utils.utcnow() - removed_metadata_row.updated_at
269+
) > datetime.timedelta(days=settings.SCHEMA_METADATA_TTL_DAYS)
270+
271+
table_model.query.filter(
272+
table_model.id == removed_metadata_row.id,
273+
).delete()
274+
275+
db.session.commit()
276+
277+
@celery.task(name="redash.tasks.cleanup_schema_metadata")
278+
def cleanup_schema_metadata():
279+
cleanup_data_in_table(TableMetadata)
280+
cleanup_data_in_table(ColumnMetadata)
281+
232282
@celery.task(name="redash.tasks.refresh_schema", time_limit=90, soft_time_limit=60)
233283
def refresh_schema(data_source_id):
234284
ds = models.DataSource.get_by_id(data_source_id)
235285
logger.info(u"task=refresh_schema state=start ds_id=%s", ds.id)
236286
start_time = time.time()
287+
237288
try:
238-
ds.get_schema(refresh=True)
289+
existing_tables = set()
290+
schema = ds.query_runner.get_schema(get_stats=True)
291+
for table in schema:
292+
table_name = table['name']
293+
existing_tables.add(table_name)
294+
295+
# Assume that there will only exist 1 table with a given name for a given data source so we use first()
296+
persisted_table = TableMetadata.query.filter(
297+
TableMetadata.name == table_name,
298+
TableMetadata.data_source_id == ds.id,
299+
).first()
300+
301+
if persisted_table:
302+
TableMetadata.query.filter(
303+
TableMetadata.id == persisted_table.id,
304+
).update({"exists": True})
305+
else:
306+
metadata = 'metadata' in table
307+
persisted_table = TableMetadata(
308+
org_id=ds.org_id,
309+
name=table_name,
310+
data_source_id=ds.id,
311+
column_metadata=metadata
312+
)
313+
models.db.session.add(persisted_table)
314+
models.db.session.flush()
315+
316+
existing_columns = set()
317+
for i, column in enumerate(table['columns']):
318+
existing_columns.add(column)
319+
column_metadata = {
320+
'org_id': ds.org_id,
321+
'table_id': persisted_table.id,
322+
'name': column,
323+
'type': None,
324+
'example': None,
325+
'exists': True
326+
}
327+
if 'metadata' in table:
328+
column_metadata['type'] = table['metadata'][i]['type']
329+
330+
# If the column exists, update it, otherwise create a new one.
331+
persisted_column = ColumnMetadata.query.filter(
332+
ColumnMetadata.name == column,
333+
ColumnMetadata.table_id == persisted_table.id,
334+
).options(load_only('id')).first()
335+
if persisted_column:
336+
ColumnMetadata.query.filter(
337+
ColumnMetadata.id == persisted_column.id,
338+
).update(column_metadata)
339+
else:
340+
models.db.session.add(ColumnMetadata(**column_metadata))
341+
models.db.session.commit()
342+
343+
get_table_sample_data.apply_async(
344+
args=(data_source_id, table, persisted_table.id),
345+
queue=settings.SCHEMAS_REFRESH_QUEUE
346+
)
347+
348+
# If a column did not exist, set the 'column_exists' flag to false.
349+
existing_columns_list = tuple(existing_columns)
350+
ColumnMetadata.query.filter(
351+
ColumnMetadata.exists == True,
352+
ColumnMetadata.table_id == persisted_table.id,
353+
~ColumnMetadata.name.in_(existing_columns_list),
354+
).update({
355+
"exists": False,
356+
"updated_at": db.func.now()
357+
}, synchronize_session='fetch')
358+
359+
# If a table did not exist in the get_schema() response above, set the 'exists' flag to false.
360+
existing_tables_list = tuple(existing_tables)
361+
tables_to_update = TableMetadata.query.filter(
362+
TableMetadata.exists == True,
363+
TableMetadata.data_source_id == ds.id,
364+
~TableMetadata.name.in_(existing_tables_list)
365+
).update({
366+
"exists": False,
367+
"updated_at": db.func.now()
368+
}, synchronize_session='fetch')
369+
370+
models.db.session.commit()
371+
239372
logger.info(u"task=refresh_schema state=finished ds_id=%s runtime=%.2f", ds.id, time.time() - start_time)
240373
statsd_client.incr('refresh_schema.success')
241374
except SoftTimeLimitExceeded:

‎redash/worker.py

+4
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@
2727
'sync_user_details': {
2828
'task': 'redash.tasks.sync_user_details',
2929
'schedule': timedelta(minutes=1),
30+
},
31+
'cleanup_schema_metadata': {
32+
'task': 'redash.tasks.cleanup_schema_metadata',
33+
'schedule': timedelta(days=3),
3034
}
3135
}
3236

‎tests/factories.py

+15
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,15 @@ def __call__(self):
7979
data_source=data_source_factory.create,
8080
org_id=1)
8181

82+
table_metadata_factory = ModelFactory(redash.models.TableMetadata,
83+
data_source_id=1,
84+
exists=True,
85+
name='table')
86+
87+
column_metadata_factory = ModelFactory(redash.models.ColumnMetadata,
88+
table_id=1,
89+
name='column')
90+
8291
query_with_params_factory = ModelFactory(redash.models.Query,
8392
name='New Query with Params',
8493
description='',
@@ -176,6 +185,12 @@ def create_org(self, **kwargs):
176185

177186
return org
178187

188+
def create_table_metadata(self, **kwargs):
189+
return table_metadata_factory.create(**kwargs)
190+
191+
def create_column_metadata(self, **kwargs):
192+
return column_metadata_factory.create(**kwargs)
193+
179194
def create_user(self, **kwargs):
180195
args = {
181196
'org': self.org,

‎tests/models/test_data_sources.py

+36-32
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import mock
21
from tests import BaseTestCase
32

43
from redash.models import DataSource, Query, QueryResult
@@ -7,38 +6,43 @@
76

87
class DataSourceTest(BaseTestCase):
98
def test_get_schema(self):
10-
return_value = [{'name': 'table', 'columns': []}]
11-
12-
with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema:
13-
patched_get_schema.return_value = return_value
14-
15-
schema = self.factory.data_source.get_schema()
16-
17-
self.assertEqual(return_value, schema)
18-
19-
def test_get_schema_uses_cache(self):
20-
return_value = [{'name': 'table', 'columns': []}]
21-
with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema:
22-
patched_get_schema.return_value = return_value
23-
24-
self.factory.data_source.get_schema()
25-
schema = self.factory.data_source.get_schema()
26-
27-
self.assertEqual(return_value, schema)
28-
self.assertEqual(patched_get_schema.call_count, 1)
29-
30-
def test_get_schema_skips_cache_with_refresh_true(self):
31-
return_value = [{'name': 'table', 'columns': []}]
32-
with mock.patch('redash.query_runner.pg.PostgreSQL.get_schema') as patched_get_schema:
33-
patched_get_schema.return_value = return_value
34-
35-
self.factory.data_source.get_schema()
36-
new_return_value = [{'name': 'new_table', 'columns': []}]
37-
patched_get_schema.return_value = new_return_value
38-
schema = self.factory.data_source.get_schema(refresh=True)
9+
data_source = self.factory.create_data_source()
3910

40-
self.assertEqual(new_return_value, schema)
41-
self.assertEqual(patched_get_schema.call_count, 2)
11+
# Create an existing table with a non-existing column
12+
table_metadata = self.factory.create_table_metadata(
13+
data_source_id=data_source.id,
14+
org_id=data_source.org_id
15+
)
16+
column_metadata = self.factory.create_column_metadata(
17+
table_id=table_metadata.id,
18+
org_id=data_source.org_id,
19+
type='boolean',
20+
example=True,
21+
exists=False
22+
)
23+
24+
# Create a non-existing table with an existing column
25+
table_metadata = self.factory.create_table_metadata(
26+
data_source_id=data_source.id,
27+
org_id=data_source.org_id,
28+
name='table_doesnt_exist',
29+
exists=False
30+
)
31+
column_metadata = self.factory.create_column_metadata(
32+
table_id=table_metadata.id,
33+
org_id=data_source.org_id,
34+
type='boolean',
35+
example=True,
36+
)
37+
38+
return_value = [{
39+
'name': 'table',
40+
'hasColumnMetadata': False,
41+
'exists': True,
42+
'columns': []
43+
}]
44+
schema = data_source.get_schema()
45+
self.assertEqual(return_value, schema)
4246

4347

4448
class TestDataSourceCreate(BaseTestCase):
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import json
2+
import mock
3+
4+
from unittest import TestCase
5+
6+
from redash.query_runner.presto import Presto
7+
from redash.query_runner.athena import Athena
8+
from redash.query_runner.mysql import Mysql
9+
from redash.query_runner.pg import PostgreSQL, Redshift
10+
11+
class TestBaseQueryRunner(TestCase):
12+
def setUp(self):
13+
self.query_runners = [{
14+
'instance': Presto({}),
15+
'mock_location': 'presto.Presto'
16+
}, {
17+
'instance': Athena({}),
18+
'mock_location': 'athena.Athena'
19+
}, {
20+
'instance': Mysql({'db': None}),
21+
'mock_location': 'mysql.Mysql'
22+
}, {
23+
'instance': PostgreSQL({}),
24+
'mock_location': 'pg.PostgreSQL'
25+
}, {
26+
'instance': Redshift({}),
27+
'mock_location': 'pg.Redshift'
28+
}]
29+
30+
def _setup_mock(self, function_to_patch):
31+
patcher = mock.patch(function_to_patch)
32+
patched_function = patcher.start()
33+
self.addCleanup(patcher.stop)
34+
return patched_function
35+
36+
def assert_correct_schema_format(self, query_runner, mock_location):
37+
EXPECTED_SCHEMA_RESULT = [{
38+
'columns': ['created_date'],
39+
'metadata': [{
40+
'name': 'created_date',
41+
'type': 'varchar',
42+
}],
43+
'name': 'default.table_name'
44+
}]
45+
46+
get_schema_query_response = {
47+
"rows": [{
48+
"table_schema": "default",
49+
"table_name": "table_name",
50+
"column_type": "varchar",
51+
"column_name": "created_date"
52+
}]
53+
}
54+
get_samples_query_response = {
55+
"rows": [{
56+
"created_date": "2017-10-26"
57+
}]
58+
}
59+
60+
self.run_count = 0
61+
def query_runner_resonses(query, user):
62+
response = (json.dumps(get_schema_query_response), None)
63+
if self.run_count > 0:
64+
response = (json.dumps(get_samples_query_response), None)
65+
self.run_count += 1
66+
return response
67+
68+
self.patched_run_query = self._setup_mock(
69+
'redash.query_runner.{location}.run_query'.format(location=mock_location))
70+
self.patched_run_query.side_effect = query_runner_resonses
71+
72+
schema = query_runner.get_schema()
73+
self.assertEqual(schema, EXPECTED_SCHEMA_RESULT)
74+
75+
def test_get_schema_format(self):
76+
for runner in self.query_runners:
77+
self.assert_correct_schema_format(runner['instance'], runner['mock_location'])

‎tests/tasks/test_queries.py

+26-2
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,14 @@
33
import uuid
44

55
import mock
6+
import datetime
67

78
from tests import BaseTestCase
8-
from redash import redis_connection, models
9+
from redash import redis_connection, models, utils
10+
from redash.models import TableMetadata
911
from redash.query_runner.pg import PostgreSQL
10-
from redash.tasks.queries import QueryExecutionError, enqueue_query, execute_query
12+
from redash.tasks.queries import (QueryExecutionError, enqueue_query,
13+
execute_query, cleanup_data_in_table)
1114

1215

1316
FakeResult = namedtuple('FakeResult', 'id')
@@ -114,3 +117,24 @@ def test_success_after_failure(self):
114117
scheduled_query_id=q.id)
115118
q = models.Query.get_by_id(q.id)
116119
self.assertEqual(q.schedule_failures, 0)
120+
121+
122+
class TestPruneSchemaMetadata(BaseTestCase):
123+
124+
def test_cleanup_data_in_table(self):
125+
data_source = self.factory.create_data_source()
126+
127+
# Create an existing table with a non-existing column
128+
table_metadata = self.factory.create_table_metadata(
129+
data_source_id=data_source.id,
130+
org_id=data_source.org_id,
131+
exists=False,
132+
updated_at=(utils.utcnow() - datetime.timedelta(days=70))
133+
)
134+
all_tables = TableMetadata.query.all()
135+
self.assertEqual(len(all_tables), 1)
136+
137+
cleanup_data_in_table(TableMetadata)
138+
139+
all_tables = TableMetadata.query.all()
140+
self.assertEqual(len(all_tables), 0)

‎tests/tasks/test_refresh_schemas.py

+138-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,49 @@
1+
import copy
2+
13
from mock import patch
24
from tests import BaseTestCase
35

4-
from redash.tasks import refresh_schemas
6+
from redash import models
7+
from redash.tasks import refresh_schemas, refresh_schema, get_table_sample_data
8+
from redash.models import TableMetadata, ColumnMetadata
59

610

711
class TestRefreshSchemas(BaseTestCase):
12+
def setUp(self):
13+
super(TestRefreshSchemas, self).setUp()
14+
15+
self.COLUMN_NAME = 'first_column'
16+
self.COLUMN_TYPE = 'text'
17+
self.COLUMN_EXAMPLE = 'some text for column value'
18+
self.EXPECTED_COLUMN_METADATA = {
19+
'id': 1,
20+
'org_id': 1,
21+
'table_id': 1,
22+
'name': self.COLUMN_NAME,
23+
'type': self.COLUMN_TYPE,
24+
'example': self.COLUMN_EXAMPLE,
25+
'exists': True,
26+
}
27+
28+
get_schema_patcher = patch('redash.query_runner.pg.PostgreSQL.get_schema')
29+
self.patched_get_schema = get_schema_patcher.start()
30+
self.addCleanup(get_schema_patcher.stop)
31+
self.default_schema_return_value = [{
32+
'name': 'table',
33+
'columns': [self.COLUMN_NAME],
34+
'metadata': [{
35+
'name': self.COLUMN_NAME,
36+
'type': self.COLUMN_TYPE,
37+
}]
38+
}]
39+
self.patched_get_schema.return_value = self.default_schema_return_value
40+
41+
42+
get_table_sample_patcher = patch('redash.query_runner.BaseQueryRunner.get_table_sample')
43+
patched_get_table_sample = get_table_sample_patcher.start()
44+
self.addCleanup(get_table_sample_patcher.stop)
45+
patched_get_table_sample.return_value = {self.COLUMN_NAME: self.COLUMN_EXAMPLE}
46+
847
def test_calls_refresh_of_all_data_sources(self):
948
self.factory.data_source # trigger creation
1049
with patch('redash.tasks.queries.refresh_schema.apply_async') as refresh_job:
@@ -23,3 +62,101 @@ def test_skips_paused_data_sources(self):
2362
with patch('redash.tasks.queries.refresh_schema.apply_async') as refresh_job:
2463
refresh_schemas()
2564
refresh_job.assert_called()
65+
66+
def test_refresh_schema_creates_tables(self):
67+
EXPECTED_TABLE_METADATA = {
68+
'id': 1,
69+
'org_id': 1,
70+
'exists': True,
71+
'name': 'table',
72+
'sample_query': None,
73+
'description': None,
74+
'column_metadata': True,
75+
'data_source_id': 1
76+
}
77+
78+
refresh_schema(self.factory.data_source.id)
79+
get_table_sample_data(
80+
self.factory.data_source.id, {
81+
"name": 'table',
82+
"columns": [self.COLUMN_NAME]
83+
}, 1
84+
)
85+
table_metadata = TableMetadata.query.all()
86+
column_metadata = ColumnMetadata.query.all()
87+
88+
self.assertEqual(len(table_metadata), 1)
89+
self.assertEqual(len(column_metadata), 1)
90+
self.assertEqual(table_metadata[0].to_dict(), EXPECTED_TABLE_METADATA)
91+
self.assertEqual(column_metadata[0].to_dict(), self.EXPECTED_COLUMN_METADATA)
92+
93+
def test_refresh_schema_deleted_table_marked(self):
94+
refresh_schema(self.factory.data_source.id)
95+
table_metadata = TableMetadata.query.all()
96+
column_metadata = ColumnMetadata.query.all()
97+
98+
self.assertEqual(len(table_metadata), 1)
99+
self.assertEqual(len(column_metadata), 1)
100+
self.assertTrue(table_metadata[0].to_dict()['exists'])
101+
102+
# Table is gone, `exists` should be False.
103+
self.patched_get_schema.return_value = []
104+
105+
refresh_schema(self.factory.data_source.id)
106+
table_metadata = TableMetadata.query.all()
107+
column_metadata = ColumnMetadata.query.all()
108+
109+
self.assertEqual(len(table_metadata), 1)
110+
self.assertEqual(len(column_metadata), 1)
111+
self.assertFalse(table_metadata[0].to_dict()['exists'])
112+
113+
# Table is back, `exists` should be True again.
114+
self.patched_get_schema.return_value = self.default_schema_return_value
115+
refresh_schema(self.factory.data_source.id)
116+
table_metadata = TableMetadata.query.all()
117+
self.assertTrue(table_metadata[0].to_dict()['exists'])
118+
119+
def test_refresh_schema_delete_column(self):
120+
NEW_COLUMN_NAME = 'new_column'
121+
refresh_schema(self.factory.data_source.id)
122+
column_metadata = ColumnMetadata.query.all()
123+
124+
self.assertTrue(column_metadata[0].to_dict()['exists'])
125+
126+
self.patched_get_schema.return_value = [{
127+
'name': 'table',
128+
'columns': [NEW_COLUMN_NAME],
129+
'metadata': [{
130+
'name': NEW_COLUMN_NAME,
131+
'type': self.COLUMN_TYPE,
132+
}]
133+
}]
134+
135+
refresh_schema(self.factory.data_source.id)
136+
column_metadata = ColumnMetadata.query.all()
137+
self.assertEqual(len(column_metadata), 2)
138+
139+
self.assertFalse(column_metadata[1].to_dict()['exists'])
140+
self.assertTrue(column_metadata[0].to_dict()['exists'])
141+
142+
def test_refresh_schema_update_column(self):
143+
UPDATED_COLUMN_TYPE = 'varchar'
144+
145+
refresh_schema(self.factory.data_source.id)
146+
get_table_sample_data(
147+
self.factory.data_source.id, {
148+
"name": 'table',
149+
"columns": [self.COLUMN_NAME]
150+
}, 1
151+
)
152+
column_metadata = ColumnMetadata.query.all()
153+
self.assertEqual(column_metadata[0].to_dict(), self.EXPECTED_COLUMN_METADATA)
154+
155+
updated_schema = copy.deepcopy(self.default_schema_return_value)
156+
updated_schema[0]['metadata'][0]['type'] = UPDATED_COLUMN_TYPE
157+
self.patched_get_schema.return_value = updated_schema
158+
159+
refresh_schema(self.factory.data_source.id)
160+
column_metadata = ColumnMetadata.query.all()
161+
self.assertNotEqual(column_metadata[0].to_dict(), self.EXPECTED_COLUMN_METADATA)
162+
self.assertEqual(column_metadata[0].to_dict()['type'], UPDATED_COLUMN_TYPE)

‎tests/test_cli.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ def test_interactive_new(self):
1616
result = runner.invoke(
1717
manager,
1818
['ds', 'new'],
19-
input="test\n%s\n\n\nexample.com\n\n\ntestdb\n" % (pg_i,))
19+
input="test\n%s\n\n\nexample.com\n\n\n\ntestdb\n" % (pg_i,))
2020
self.assertFalse(result.exception)
2121
self.assertEqual(result.exit_code, 0)
2222
self.assertEqual(DataSource.query.count(), 1)

0 commit comments

Comments
 (0)
This repository has been archived.