From 109eac83cbcfc078b0d17eae81405ebf51f79250 Mon Sep 17 00:00:00 2001 From: Marina Samuel Date: Mon, 25 Mar 2019 15:01:59 -0400 Subject: [PATCH] Schema Viewer Drawer (getredash#3291) * Process extra column metadata for a few sql-based data sources. * Add Table and Column metadata tables. * Periodically update table and column schema tables in a celery task. * Fetching schema returns data from table and column metadata tables. * Add tests for backend changes. * Front-end shows extra table metadata and uses new schema response. * Delete datasource schema data when deleting a data source. * Process and store data source schema when a data source is first created or after a migration. * Tables should have a unique name per datasource. * Addressing review comments. * Update migration file for mixins. * Appease PEP8 * Upgrade migration file for rebase. * Cascade delete. * Adding org_id * Remove redundant column and table prefixes. * Non-existing tables and columns should be filtered out on the server side not client side. * Fetching table samples should be optional and should happen in a separate task per table. * Allow users to force a schema refresh. * Use updated_at to help prune old schema metadata periodically. * Using settings.SCHEMAS_REFRESH_QUEUE * fix for getredash#2426 test * more stable test_interactive_new * Closes #927, #928: Schema refresh improvements. * Closes #934, #935: Remove type from schema browser and don't show empty example column in schema drawer (#936) * Speed up schema fetch requests with fewer postgres queries. * Add column metadata to Athena glue processing. * Fix bug assuming 'metadata' exists for every table. * Closes #939: Persisted, existing table metadata should be updated. * Sample processing should be rate-limited. * Add cli command for refreshing data samples. * Schema refreshes should not overwrite column 'example' field. * refresh_samples() should filter tables_to_sample on the datasource's id being sampled * Correctly wrap long text in schema drawer. Schema Improvements Part 2: Add data source config options. Adding BigQuery schema drawer with data types and samples. Add empty migration to replace the removed schedule_until migration Add merge migration. Co-authored-by: Alison Co-authored-by: Jannis Leidel --- client/app/assets/less/ant.less | 1 + client/app/assets/less/inc/base.less | 4 + client/app/assets/less/inc/popover.less | 4 +- .../app/assets/less/inc/schema-browser.less | 14 +- client/app/assets/less/redash/query.less | 14 + .../dynamic-form/dynamicFormHelper.js | 8 + client/app/components/proptypes.js | 17 +- .../app/components/queries/QueryEditor/ace.js | 8 +- .../components/queries/QueryEditor/index.jsx | 2 +- client/app/components/queries/SchemaData.jsx | 141 +++++++ .../app/pages/data-sources/EditDataSource.jsx | 12 + .../schema-table-components/EditableTable.jsx | 83 +++++ .../schema-table-components/QueryListItem.jsx | 40 ++ .../QuerySearchDialog.jsx | 106 ++++++ .../SampleQueryList.jsx | 85 +++++ .../schema-table-components/SchemaTable.jsx | 276 ++++++++++++++ .../TableVisibilityCheckbox.jsx | 24 ++ .../schema-table-components/schema-table.css | 24 ++ client/app/pages/queries/QuerySource.jsx | 7 + .../queries/components/SchemaBrowser.jsx | 73 +++- client/app/services/data-source.js | 2 + migrations/versions/118aa16f565b_.py | 38 ++ migrations/versions/151a4c333e96_.py | 24 ++ .../171aaafb2d52_add_more_db_indexes.py | 101 +++++ migrations/versions/280daa582976_.py | 59 +++ migrations/versions/6adb92e75691_.py | 27 ++ migrations/versions/ba150362b02e_.py | 26 ++ migrations/versions/cf135a57332e_.py | 32 ++ migrations/versions/eb2f788f997e_.py | 23 ++ redash/cli/data_sources.py | 34 +- redash/handlers/api.py | 4 +- redash/handlers/data_sources.py | 37 +- redash/models/__init__.py | 324 ++++++++++++++-- redash/query_runner/__init__.py | 20 + redash/query_runner/athena.py | 25 +- redash/query_runner/big_query.py | 117 +++++- redash/query_runner/mysql.py | 10 +- redash/query_runner/pg.py | 26 +- redash/query_runner/presto.py | 9 +- redash/serializers/__init__.py | 82 +++- redash/settings/__init__.py | 22 +- redash/tasks/__init__.py | 4 + redash/tasks/queries/__init__.py | 2 + redash/tasks/queries/maintenance.py | 158 ++++++-- redash/tasks/queries/samples.py | 128 +++++++ redash/tasks/schedule.py | 2 + tests/factories.py | 15 + tests/handlers/test_data_sources.py | 42 +++ tests/models/test_data_sources.py | 98 ----- tests/query_runner/test_athena.py | 27 +- tests/query_runner/test_bigquery.py | 54 +++ tests/query_runner/test_get_schema_format.py | 70 ++++ tests/query_runner/test_pg.py | 15 +- tests/tasks/test_queries.py | 23 +- tests/tasks/test_refresh_schemas.py | 349 +++++++++++++++++- tests/test_cli.py | 15 +- 56 files changed, 2764 insertions(+), 223 deletions(-) create mode 100644 client/app/components/queries/SchemaData.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/EditableTable.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/QueryListItem.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/QuerySearchDialog.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/SampleQueryList.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/SchemaTable.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/TableVisibilityCheckbox.jsx create mode 100644 client/app/pages/data-sources/schema-table-components/schema-table.css create mode 100644 migrations/versions/118aa16f565b_.py create mode 100644 migrations/versions/151a4c333e96_.py create mode 100644 migrations/versions/171aaafb2d52_add_more_db_indexes.py create mode 100644 migrations/versions/280daa582976_.py create mode 100644 migrations/versions/6adb92e75691_.py create mode 100644 migrations/versions/ba150362b02e_.py create mode 100644 migrations/versions/cf135a57332e_.py create mode 100644 migrations/versions/eb2f788f997e_.py create mode 100644 redash/tasks/queries/samples.py create mode 100644 tests/query_runner/test_bigquery.py create mode 100644 tests/query_runner/test_get_schema_format.py diff --git a/client/app/assets/less/ant.less b/client/app/assets/less/ant.less index 672393a1c2..e15cef2bfd 100644 --- a/client/app/assets/less/ant.less +++ b/client/app/assets/less/ant.less @@ -14,6 +14,7 @@ @import '~antd/lib/radio/style/index'; @import '~antd/lib/time-picker/style/index'; @import '~antd/lib/pagination/style/index'; +@import '~antd/lib/drawer/style/index'; @import '~antd/lib/table/style/index'; @import '~antd/lib/popover/style/index'; @import '~antd/lib/icon/style/index'; diff --git a/client/app/assets/less/inc/base.less b/client/app/assets/less/inc/base.less index 171b60fff5..2770b5d91a 100755 --- a/client/app/assets/less/inc/base.less +++ b/client/app/assets/less/inc/base.less @@ -155,6 +155,10 @@ strong { transition: height 0s, width 0s !important; } +.admin-schema-editor { + padding: 50px 0; +} + .bg-ace { background-color: fade(@redash-gray, 12%) !important; } diff --git a/client/app/assets/less/inc/popover.less b/client/app/assets/less/inc/popover.less index 5fcad7089b..c687a089a2 100755 --- a/client/app/assets/less/inc/popover.less +++ b/client/app/assets/less/inc/popover.less @@ -1,5 +1,7 @@ .popover { box-shadow: fade(@redash-gray, 25%) 0px 0px 15px 0px; + color: #000000; + z-index: 1000000001; // So that it can popover a dropdown menu } .popover-title { @@ -19,4 +21,4 @@ p { margin-bottom: 0; } -} \ No newline at end of file +} diff --git a/client/app/assets/less/inc/schema-browser.less b/client/app/assets/less/inc/schema-browser.less index 3da8fc9212..883d60def4 100644 --- a/client/app/assets/less/inc/schema-browser.less +++ b/client/app/assets/less/inc/schema-browser.less @@ -7,14 +7,14 @@ div.table-name { border-radius: @redash-radius; position: relative; - .copy-to-editor { + .copy-to-editor, .info { display: none; } &:hover { background: fade(@redash-gray, 10%); - .copy-to-editor { + .copy-to-editor, .info { display: flex; } } @@ -38,7 +38,7 @@ div.table-name { background: transparent; } - .copy-to-editor { + .copy-to-editor, .info { color: fade(@redash-gray, 90%); cursor: pointer; position: absolute; @@ -51,6 +51,10 @@ div.table-name { justify-content: center; } + .info { + right: 20px + } + .table-open { padding: 0 22px 0 26px; overflow: hidden; @@ -58,14 +62,14 @@ div.table-name { white-space: nowrap; position: relative; - .copy-to-editor { + .copy-to-editor, .info { display: none; } &:hover { background: fade(@redash-gray, 10%); - .copy-to-editor { + .copy-to-editor, .info { display: flex; } } diff --git a/client/app/assets/less/redash/query.less b/client/app/assets/less/redash/query.less index 7b3598d129..0627390b0d 100644 --- a/client/app/assets/less/redash/query.less +++ b/client/app/assets/less/redash/query.less @@ -514,3 +514,17 @@ nav .rg-bottom { padding-right: 0; } } + +.ui-select-choices-row .info { + display: none; +} + +.ui-select-choices-row { + &:hover { + .info { + cursor: pointer; + width: 20px; + display: inline; + } + } +} diff --git a/client/app/components/dynamic-form/dynamicFormHelper.js b/client/app/components/dynamic-form/dynamicFormHelper.js index b59b9d3ee8..9b3c7d262c 100644 --- a/client/app/components/dynamic-form/dynamicFormHelper.js +++ b/client/app/components/dynamic-form/dynamicFormHelper.js @@ -100,6 +100,13 @@ function getFields(type = {}, target = { options: {} }) { placeholder: `My ${type.name}`, autoFocus: isNewTarget, }, + { + name: "description", + title: "Description", + type: "text", + required: false, + initialValue: target.description, + }, ...orderedInputs(configurationSchema.properties, configurationSchema.order, target.options), ]; @@ -108,6 +115,7 @@ function getFields(type = {}, target = { options: {} }) { function updateTargetWithValues(target, values) { target.name = values.name; + target.description = values.description; Object.keys(values).forEach(key => { if (key !== "name") { target.options[key] = values[key]; diff --git a/client/app/components/proptypes.js b/client/app/components/proptypes.js index 355b912da4..1d815b50c6 100644 --- a/client/app/components/proptypes.js +++ b/client/app/components/proptypes.js @@ -11,8 +11,16 @@ export const DataSource = PropTypes.shape({ type_name: PropTypes.string, }); +export const DataSourceMetadata = PropTypes.shape({ + key: PropTypes.number, + name: PropTypes.string, + type: PropTypes.string, + example: PropTypes.string, + description: PropTypes.string, +}); + export const Table = PropTypes.shape({ - columns: PropTypes.arrayOf(PropTypes.string).isRequired, + columns: PropTypes.arrayOf(PropTypes.object).isRequired, }); export const Schema = PropTypes.arrayOf(Table); @@ -31,6 +39,13 @@ export const RefreshScheduleDefault = { until: null, }; +export const TableMetadata = PropTypes.shape({ + key: PropTypes.number.isRequired, + name: PropTypes.string.isRequired, + description: PropTypes.string, + visible: PropTypes.bool.isRequired, +}); + export const Field = PropTypes.shape({ name: PropTypes.string.isRequired, title: PropTypes.string, diff --git a/client/app/components/queries/QueryEditor/ace.js b/client/app/components/queries/QueryEditor/ace.js index b4c4689942..1579988ec1 100644 --- a/client/app/components/queries/QueryEditor/ace.js +++ b/client/app/components/queries/QueryEditor/ace.js @@ -31,9 +31,9 @@ function buildTableColumnKeywords(table) { const keywords = []; table.columns.forEach(column => { keywords.push({ - caption: column, - name: `${table.name}.${column}`, - value: `${table.name}.${column}`, + caption: column.name, + name: `${table.name}.${column.name}`, + value: `${table.name}.${column.name}`, score: 100, meta: "Column", className: "completion", @@ -56,7 +56,7 @@ function buildKeywordsFromSchema(schema) { }); tableColumnKeywords[table.name] = buildTableColumnKeywords(table); table.columns.forEach(c => { - columnKeywords[c] = "Column"; + columnKeywords[c.name] = "Column"; }); }); diff --git a/client/app/components/queries/QueryEditor/index.jsx b/client/app/components/queries/QueryEditor/index.jsx index 9c1b978fb0..04b8b37f06 100644 --- a/client/app/components/queries/QueryEditor/index.jsx +++ b/client/app/components/queries/QueryEditor/index.jsx @@ -161,7 +161,7 @@ QueryEditor.propTypes = { PropTypes.shape({ name: PropTypes.string.isRequired, size: PropTypes.number, - columns: PropTypes.arrayOf(PropTypes.string).isRequired, + columns: PropTypes.arrayOf(PropTypes.object).isRequired, }) ), onChange: PropTypes.func, diff --git a/client/app/components/queries/SchemaData.jsx b/client/app/components/queries/SchemaData.jsx new file mode 100644 index 0000000000..55b8aeaf67 --- /dev/null +++ b/client/app/components/queries/SchemaData.jsx @@ -0,0 +1,141 @@ +import { some } from "lodash"; +import React from "react"; +import PropTypes from "prop-types"; +import Drawer from "antd/lib/drawer"; +import Table from "antd/lib/table"; + +import { DataSourceMetadata, Query } from "@/components/proptypes"; + +function textWrapRenderer(text) { + return
{text}
; +} + +export default class SchemaData extends React.PureComponent { + static propTypes = { + show: PropTypes.bool.isRequired, + onClose: PropTypes.func.isRequired, + tableName: PropTypes.string, + tableDescription: PropTypes.string, + tableMetadata: PropTypes.arrayOf(DataSourceMetadata), + sampleQueries: PropTypes.arrayOf(Query), + }; + + static defaultProps = { + tableName: "", + tableDescription: "", + tableMetadata: [], + sampleQueries: [], + }; + + render() { + const tableDataColumns = [ + { + title: "Metadata", + dataIndex: "metadata", + width: 400, + key: "metadata", + }, + { + title: "Value", + dataIndex: "value", + width: 400, + key: "value", + render: text => { + if (typeof text === "string") { + return text; + } + return ( + + ); + }, + }, + ]; + + const columnDataColumns = [ + { + title: "Column Name", + dataIndex: "name", + width: 400, + key: "name", + render: textWrapRenderer, + }, + { + title: "Column Type", + dataIndex: "type", + width: 400, + key: "type", + render: textWrapRenderer, + }, + ]; + + const hasDescription = some(this.props.tableMetadata, columnMetadata => columnMetadata.description); + + const hasExample = some(this.props.tableMetadata, columnMetadata => columnMetadata.example); + + if (hasDescription) { + columnDataColumns.push({ + title: "Description", + dataIndex: "description", + width: 400, + key: "description", + render: textWrapRenderer, + }); + } + + if (hasExample) { + columnDataColumns.push({ + title: "Example", + dataIndex: "example", + width: 400, + key: "example", + render: textWrapRenderer, + }); + } + const tableData = [ + { + metadata: "Table Description", + value: this.props.tableDescription || "N/A", + key: "description", + }, + { + metadata: "Sample Usage", + value: this.props.sampleQueries.length > 0 ? this.props.sampleQueries : "N/A", + key: "sample", + }, + ]; + + return ( + +

{this.props.tableName}

+
+
Table Data
+ +
+
Column Data
+
+ + ); + } +} diff --git a/client/app/pages/data-sources/EditDataSource.jsx b/client/app/pages/data-sources/EditDataSource.jsx index f04ea18183..d25c4474ee 100644 --- a/client/app/pages/data-sources/EditDataSource.jsx +++ b/client/app/pages/data-sources/EditDataSource.jsx @@ -8,6 +8,7 @@ import navigateTo from "@/components/ApplicationArea/navigateTo"; import notification from "@/services/notification"; import LoadingState from "@/components/items-list/components/LoadingState"; import DynamicForm from "@/components/dynamic-form/DynamicForm"; +import SchemaTable from "@/pages/data-sources/schema-table-components/SchemaTable"; import helper from "@/components/dynamic-form/dynamicFormHelper"; import HelpTrigger, { TYPES as HELP_TRIGGER_TYPES } from "@/components/HelpTrigger"; import wrapSettingsTab from "@/components/SettingsWrapper"; @@ -26,6 +27,7 @@ class EditDataSource extends React.Component { dataSource: null, type: null, loading: true, + schema: null, }; componentDidMount() { @@ -34,6 +36,7 @@ class EditDataSource extends React.Component { const { type } = dataSource; this.setState({ dataSource }); DataSource.types().then(types => this.setState({ type: find(types, { type }), loading: false })); + DataSource.fetchSchema({ id: this.props.dataSourceId }).then(data => this.setState({ schema: data.schema, loading: false })); }) .catch(error => this.props.onError(error)); } @@ -75,6 +78,12 @@ class EditDataSource extends React.Component { }); }; + updateSchema = (schema, tableId, columnId) => { + const { dataSource } = this.state; + const data = { tableId, columnId, schema }; + DataSource.updateSchema({ id: dataSource.id, data: data }); + }; + testConnection = callback => { const { dataSource } = this.state; DataSource.test({ id: dataSource.id }) @@ -127,6 +136,9 @@ class EditDataSource extends React.Component {
+
+ +
); } diff --git a/client/app/pages/data-sources/schema-table-components/EditableTable.jsx b/client/app/pages/data-sources/schema-table-components/EditableTable.jsx new file mode 100644 index 0000000000..4150b7802b --- /dev/null +++ b/client/app/pages/data-sources/schema-table-components/EditableTable.jsx @@ -0,0 +1,83 @@ +import React from "react"; +import Form from "antd/lib/form"; +import Input from "antd/lib/input"; +import PropTypes from "prop-types"; +import { TableMetadata } from "@/components/proptypes"; +import TableVisibilityCheckbox from "./TableVisibilityCheckbox"; +import SampleQueryList from "./SampleQueryList"; + +import "./schema-table.css"; + +const FormItem = Form.Item; +const { TextArea } = Input; +export const EditableContext = React.createContext(); + +// eslint-disable-next-line react/prop-types +const EditableRow = ({ form, index, ...props }) => ( + + + +); + +export const EditableFormRow = Form.create()(EditableRow); + +export class EditableCell extends React.Component { + static propTypes = { + dataIndex: PropTypes.string, + input_type: PropTypes.string, + editing: PropTypes.bool, + record: TableMetadata, + }; + + static defaultProps = { + dataIndex: undefined, + input_type: undefined, + editing: false, + record: {}, + }; + + constructor(props) { + super(props); + this.state = { + visible: this.props.record ? this.props.record.visible : false, + }; + } + + onChange = () => { + this.setState(prevState => ({ visible: !prevState.visible })); + }; + + getInput = () => { + if (this.props.input_type === "visible") { + return ; + } else if (this.props.input_type === "sample_queries") { + return ; + } + return