Schema Viewer Drawer (getredash#3291)

* Process extra column metadata for a few sql-based data sources. * Add Table and Column metadata tables. * Periodically update table and column schema tables in a celery task. * Fetching schema returns data from table and column metadata tables. * Add tests for backend changes. * Front-end shows extra table metadata and uses new schema response. * Delete datasource schema data when deleting a data source. * Process and store data source schema when a data source is first created or after a migration. * Tables should have a unique name per datasource. * Addressing review comments. * Update migration file for mixins. * Appease PEP8 * Upgrade migration file for rebase. * Cascade delete. * Adding org_id * Remove redundant column and table prefixes. * Non-existing tables and columns should be filtered out on the server side not client side. * Fetching table samples should be optional and should happen in a separate task per table. * Allow users to force a schema refresh. * Use updated_at to help prune old schema metadata periodically. * Using settings.SCHEMAS_REFRESH_QUEUE * fix for getredash#2426 test * more stable test_interactive_new * Closes #927, #928: Schema refresh improvements. * Closes #934, #935: Remove type from schema browser and don't show empty example column in schema drawer (#936) * Speed up schema fetch requests with fewer postgres queries. * Add column metadata to Athena glue processing. * Fix bug assuming 'metadata' exists for every table. * Closes #939: Persisted, existing table metadata should be updated. * Sample processing should be rate-limited. * Add cli command for refreshing data samples. * Schema refreshes should not overwrite column 'example' field. * refresh_samples() should filter tables_to_sample on the datasource's id being sampled * Correctly wrap long text in schema drawer. Schema Improvements Part 2: Add data source config options. Adding BigQuery schema drawer with data types and samples. Add empty migration to replace the removed schedule_until migration Add merge migration. Fix spacing issue with data scanned value in query execution metadata. Increase schema refresh timeout. Remove old migrations. Co-authored-by: Alison <github@bankofknowledge.net> Co-authored-by: Jannis Leidel <jannis@leidel.info>
mozilla · Jul 15, 2020 · 9695b85 · 9695b85
1 parent 57dd32d
commit 9695b85
Show file tree

Hide file tree

Showing 55 changed files with 2,743 additions and 237 deletions.
diff --git a/client/app/assets/less/inc/base.less b/client/app/assets/less/inc/base.less
@@ -116,6 +116,10 @@ strong {
   transition: height 0s, width 0s !important;
 }
 
+.admin-schema-editor {
+  padding: 50px 0;
+}
+
 .bg-ace {
   background-color: fade(@redash-gray, 12%) !important;
 }

diff --git a/client/app/assets/less/inc/popover.less b/client/app/assets/less/inc/popover.less
@@ -1,5 +1,7 @@
 .popover {
     box-shadow: fade(@redash-gray, 25%) 0px 0px 15px 0px;
+    color: #000000;
+    z-index: 1000000001; // So that it can popover a dropdown menu
 }
 
 .popover-title {
@@ -19,4 +21,4 @@
     p {
         margin-bottom: 0; 
     }
-} 
+} 
diff --git a/client/app/assets/less/inc/schema-browser.less b/client/app/assets/less/inc/schema-browser.less
@@ -8,14 +8,14 @@ div.table-name {
   position: relative;
   height: 22px;
 
-  .copy-to-editor {
+  .copy-to-editor, .info {
     display: none;
   }
 
   &:hover {
     background: fade(@redash-gray, 10%);
 
-    .copy-to-editor {
+    .copy-to-editor, .info {
       display: flex;
     }
   }
@@ -45,7 +45,7 @@ div.table-name {
     background: transparent;
   }
 
-  .copy-to-editor {
+  .copy-to-editor, .info {
     color: fade(@redash-gray, 90%);
     cursor: pointer;
     position: absolute;
@@ -58,6 +58,10 @@ div.table-name {
     justify-content: center;
   }
 
+  .info {
+    right: 20px
+  }
+
   .table-open {
     padding: 0 22px 0 26px;
     overflow: hidden;
@@ -66,14 +70,14 @@ div.table-name {
     position: relative;
     height: 18px;
 
-    .copy-to-editor {
+    .copy-to-editor, .info {
       display: none;
     }
 
     &:hover {
       background: fade(@redash-gray, 10%);
 
-      .copy-to-editor {
+      .copy-to-editor, .info {
         display: flex;
       }
     }

diff --git a/client/app/assets/less/redash/query.less b/client/app/assets/less/redash/query.less
@@ -493,3 +493,17 @@ nav .rg-bottom {
     padding-right: 0;
   }
 }
+
+.ui-select-choices-row .info {
+  display: none;
+}
+
+.ui-select-choices-row {
+    &:hover {
+        .info {
+            cursor: pointer;
+            width: 20px;
+            display: inline;
+        }
+    }
+}
diff --git a/client/app/components/dynamic-form/dynamicFormHelper.js b/client/app/components/dynamic-form/dynamicFormHelper.js
@@ -100,6 +100,13 @@ function getFields(type = {}, target = { options: {} }) {
       placeholder: `My ${type.name}`,
       autoFocus: isNewTarget,
     },
+    {
+      name: "description",
+      title: "Description",
+      type: "text",
+      required: false,
+      initialValue: target.description,
+    },
     ...orderedInputs(configurationSchema.properties, configurationSchema.order, target.options),
   ];
 
@@ -108,6 +115,7 @@ function getFields(type = {}, target = { options: {} }) {
 
 function updateTargetWithValues(target, values) {
   target.name = values.name;
+  target.description = values.description;
   Object.keys(values).forEach(key => {
     if (key !== "name") {
       target.options[key] = values[key];

diff --git a/client/app/components/proptypes.js b/client/app/components/proptypes.js
@@ -11,8 +11,16 @@ export const DataSource = PropTypes.shape({
   type_name: PropTypes.string,
 });
 
+export const DataSourceMetadata = PropTypes.shape({
+  key: PropTypes.number,
+  name: PropTypes.string,
+  type: PropTypes.string,
+  example: PropTypes.string,
+  description: PropTypes.string,
+});
+
 export const Table = PropTypes.shape({
-  columns: PropTypes.arrayOf(PropTypes.string).isRequired,
+  columns: PropTypes.arrayOf(PropTypes.object).isRequired,
 });
 
 export const Schema = PropTypes.arrayOf(Table);
@@ -31,6 +39,13 @@ export const RefreshScheduleDefault = {
   until: null,
 };
 
+export const TableMetadata = PropTypes.shape({
+  key: PropTypes.number.isRequired,
+  name: PropTypes.string.isRequired,
+  description: PropTypes.string,
+  visible: PropTypes.bool.isRequired,
+});
+
 export const Field = PropTypes.shape({
   name: PropTypes.string.isRequired,
   title: PropTypes.string,

diff --git a/client/app/components/queries/QueryEditor/ace.js b/client/app/components/queries/QueryEditor/ace.js
@@ -31,9 +31,9 @@ function buildTableColumnKeywords(table) {
   const keywords = [];
   table.columns.forEach(column => {
     keywords.push({
-      caption: column,
-      name: `${table.name}.${column}`,
-      value: `${table.name}.${column}`,
+      caption: column.name,
+      name: `${table.name}.${column.name}`,
+      value: `${table.name}.${column.name}`,
       score: 100,
       meta: "Column",
       className: "completion",
@@ -56,7 +56,7 @@ function buildKeywordsFromSchema(schema) {
     });
     tableColumnKeywords[table.name] = buildTableColumnKeywords(table);
     table.columns.forEach(c => {
-      columnKeywords[c] = "Column";
+      columnKeywords[c.name] = "Column";
     });
   });
 

diff --git a/client/app/components/queries/QueryEditor/index.jsx b/client/app/components/queries/QueryEditor/index.jsx
@@ -161,7 +161,7 @@ QueryEditor.propTypes = {
     PropTypes.shape({
       name: PropTypes.string.isRequired,
       size: PropTypes.number,
-      columns: PropTypes.arrayOf(PropTypes.string).isRequired,
+      columns: PropTypes.arrayOf(PropTypes.object).isRequired,
     })
   ),
   onChange: PropTypes.func,

diff --git a/client/app/components/queries/SchemaBrowser.jsx b/client/app/components/queries/SchemaBrowser.jsx
@@ -15,17 +15,18 @@ import useImmutableCallback from "@/lib/hooks/useImmutableCallback";
 import LoadingState from "../items-list/components/LoadingState";
 import { clientConfig } from "@/services/auth";
 import notification from "@/services/notification";
+import SchemaData from "@/components/queries/SchemaData";
 
 const SchemaItemType = PropTypes.shape({
   name: PropTypes.string.isRequired,
   size: PropTypes.number,
-  columns: PropTypes.arrayOf(PropTypes.string).isRequired,
+  columns: PropTypes.arrayOf(PropTypes.object).isRequired,
 });
 
 const schemaTableHeight = 22;
 const schemaColumnHeight = 18;
 
-function SchemaItem({ item, expanded, onToggle, onSelect, ...props }) {
+function SchemaItem({ item, expanded, onToggle, onSelect, onShowSchema, ...props }) {
   const handleSelect = useCallback(
     (event, ...args) => {
       event.preventDefault();
@@ -35,6 +36,15 @@ function SchemaItem({ item, expanded, onToggle, onSelect, ...props }) {
     [onSelect]
   );
 
+  const handleShowSchema = useCallback(
+    (event, ...args) => {
+      event.preventDefault();
+      event.stopPropagation();
+      onShowSchema(...args);
+    },
+    [onShowSchema]
+  );
+
   if (!item) {
     return null;
   }
@@ -47,6 +57,12 @@ function SchemaItem({ item, expanded, onToggle, onSelect, ...props }) {
           <span title={item.name}>{item.name}</span>
           {!isNil(item.size) && <span> ({item.size})</span>}
         </strong>
+        {item.column_metadata && <i
+            className="fa fa-question-circle info"
+            title="More Info"
+            aria-hidden="true"
+            onClick={e => handleShowSchema(e, item)}
+          />}
         <i
           className="fa fa-angle-double-right copy-to-editor"
           aria-hidden="true"
@@ -56,12 +72,12 @@ function SchemaItem({ item, expanded, onToggle, onSelect, ...props }) {
       {expanded && (
         <div>
           {map(item.columns, column => (
-            <div key={column} className="table-open">
-              {column}
+            <div key={column.id} className="table-open">
+              {column.name}
               <i
                 className="fa fa-angle-double-right copy-to-editor"
                 aria-hidden="true"
-                onClick={e => handleSelect(e, column)}
+                onClick={e => handleSelect(e, column.name)}
               />
             </div>
           ))}
@@ -93,7 +109,7 @@ function SchemaLoadingState() {
   );
 }
 
-export function SchemaList({ loading, schema, expandedFlags, onTableExpand, onItemSelect }) {
+export function SchemaList({ loading, schema, expandedFlags, onTableExpand, onItemSelect, openSchemaInfo, closeSchemaInfo }) {
   const [listRef, setListRef] = useState(null);
 
   useEffect(() => {
@@ -128,6 +144,7 @@ export function SchemaList({ loading, schema, expandedFlags, onTableExpand, onIt
                     expanded={expandedFlags[item.name]}
                     onToggle={() => onTableExpand(item.name)}
                     onSelect={onItemSelect}
+                    onShowSchema={openSchemaInfo}
                   />
                 );
               }}
@@ -139,6 +156,14 @@ export function SchemaList({ loading, schema, expandedFlags, onTableExpand, onIt
   );
 }
 
+function itemExists(item) {
+  if ("visible" in item) {
+    return item.visible;
+  } else {
+    return false;
+  }
+};
+
 export function applyFilterOnSchema(schema, filterString, showHidden, toggleString) {
   const filters = filter(filterString.toLowerCase().split(/\s+/), s => s.length > 0);
 
@@ -155,6 +180,9 @@ export function applyFilterOnSchema(schema, filterString, showHidden, toggleStri
       }
   }
 
+  // Filter out all columns set to invisible
+  schema = filter(schema, itemExists);
+
   // Empty string: return original schema
   if (filters.length === 0) {
     return schema;
@@ -168,7 +196,7 @@ export function applyFilterOnSchema(schema, filterString, showHidden, toggleStri
       schema,
       item =>
         includes(item.name.toLowerCase(), nameFilter) ||
-        some(item.columns, column => includes(column.toLowerCase(), columnFilter))
+        some(item.columns, column => includes(column.name.toLowerCase(), columnFilter))
     );
   }
 
@@ -178,7 +206,7 @@ export function applyFilterOnSchema(schema, filterString, showHidden, toggleStri
   return filter(
     map(schema, item => {
       if (includes(item.name.toLowerCase(), nameFilter)) {
-        item = { ...item, columns: filter(item.columns, column => includes(column.toLowerCase(), columnFilter)) };
+        item = { ...item, columns: filter(item.columns, column => includes(column.name.toLowerCase(), columnFilter)) };
         return item.columns.length > 0 ? item : null;
       }
     })
@@ -203,8 +231,17 @@ export default function SchemaBrowser({
   const [handleToggleChange] = useDebouncedCallback(setShowHidden, 100);
   const [expandedFlags, setExpandedFlags] = useState({});
 
+  const [showSchemaInfo, setShowSchemaInfo] = useState(false);
+  const [tableName, setTableName] = useState("");
+  const [tableDescription, setTableDescription] = useState("");
+  const [tableMetadata, setTableMetadata] = useState([]);
+  const [sampleQueries, setSampleQueries] = useState([]);
+
   const handleSchemaUpdate = useImmutableCallback(onSchemaUpdate);
 
+  useEffect(() => {
+    setExpandedFlags({});
+  }, [schema]);
 
   useEffect(() => {
     setExpandedFlags({});
@@ -237,6 +274,18 @@ export default function SchemaBrowser({
     return toggleString;
   }
 
+  function openSchemaInfo(table) {
+    setTableName(table.name);
+    setTableDescription(table.description);
+    setTableMetadata(table.columns);
+    setSampleQueries(Object.values(table.sample_queries));
+    setShowSchemaInfo(true);
+  }
+
+  function closeSchemaInfo() {
+    setShowSchemaInfo(false);
+  };
+
   return (
     <div className="schema-container" {...props}>
       <div className="schema-control">
@@ -267,6 +316,17 @@ export default function SchemaBrowser({
         expandedFlags={expandedFlags}
         onTableExpand={toggleTable}
         onItemSelect={onItemSelect}
+        toggleString={toggleString}
+        openSchemaInfo={openSchemaInfo}
+        closeSchemaInfo={closeSchemaInfo}
+      />
+      <SchemaData
+        show={showSchemaInfo}
+        tableName={tableName}
+        tableDescription={tableDescription}
+        tableMetadata={tableMetadata}
+        sampleQueries={sampleQueries}
+        onClose={closeSchemaInfo}
       />
     </div>
   );