Closes #934, #935: Remove type from schema browser and don't show emp…

…ty example column in schema drawer (#936)

* Closes #934, #935: Remove type from schema browser and don't show empty example column

* Speed up schema fetch requests with fewer postgres queries.

* Add column metadata to Athena glue processing.

* Fix bug assuming 'metadata' exists for every table.

client/app/components/queries/SchemaData.jsx

@@ -30,13 +30,20 @@ class SchemaData extends React.PureComponent {
       dataIndex: 'type',
       width: 400,
       key: 'type',
-    }, {
-      title: 'Example',
-      dataIndex: 'example',
-      width: 400,
-      key: 'example',
     }];
 
+    const hasExample =
+      this.props.tableMetadata.some(columnMetadata => columnMetadata.example);
+
+    if (hasExample) {
+      columns.push({
+        title: 'Example',
+        dataIndex: 'example',
+        width: 400,
+        key: 'example',
+      });
+    }
+
     return (
       <Drawer
         title={this.props.tableName}

client/app/components/queries/schema-browser.html

@@ -34,7 +34,6 @@
       <div uib-collapse="table.collapsed">
         <div ng-repeat="column in table.columns | filter:$ctrl.schemaFilterColumn track by column.key" class="table-open">
           {{column.name}}
-          <span ng-if="column.type !== undefined">({{column.type}})</span>
           <i class="fa fa-angle-double-right copy-to-editor" aria-hidden="true"
             ng-click="$ctrl.itemSelected($event, [column.name])"></i>
         </div>

redash/models/__init__.py

@@ -206,24 +206,37 @@ def delete(self):
 
     def get_schema(self):
         schema = []
-        tables = TableMetadata.query.filter(TableMetadata.data_source_id == self.id).all()
-        for table in tables:
-            if not table.exists:
-                continue
+        columns_by_table_id = {}
 
-            table_info = {
-                'name': table.name,
-                'exists': table.exists,
-                'hasColumnMetadata': table.column_metadata,
-                'columns': []}
-            columns = ColumnMetadata.query.filter(ColumnMetadata.table_id == table.id)
-            table_info['columns'] = sorted([{
+        tables = TableMetadata.query.filter(
+            TableMetadata.data_source_id == self.id,
+            TableMetadata.exists.is_(True),
+        ).all()
+        table_ids = [table.id for table in tables]
+
+        columns = ColumnMetadata.query.filter(
+            ColumnMetadata.exists.is_(True),
+            ColumnMetadata.table_id.in_(table_ids),
+        ).all()
+
+        for column in columns:
+            columns_by_table_id.setdefault(column.table_id, []).append({
                 'key': column.id,
                 'name': column.name,
                 'type': column.type,
                 'exists': column.exists,
                 'example': column.example
-            } for column in columns if column.exists == True], key=itemgetter('name'))
+            })
+
+        for table in tables:
+            table_info = {
+                'name': table.name,
+                'exists': table.exists,
+                'hasColumnMetadata': table.column_metadata,
+                'columns': []}
+
+            table_info['columns'] = sorted(
+                columns_by_table_id.get(table.id, []), key=itemgetter('name'))
             schema.append(table_info)
 
         return sorted(schema, key=itemgetter('name'))

redash/query_runner/athena.py

@@ -143,9 +143,18 @@ def __get_schema_from_glue(self):
                     table_name = '%s.%s' % (database['Name'], table['Name'])
                     if table_name not in schema:
                         column = [columns['Name'] for columns in table['StorageDescriptor']['Columns']]
-                        schema[table_name] = {'name': table_name, 'columns': column}
+                        metadata = [{
+                            "name": column_data['Name'],
+                            "type": column_data['Type']
+                        } for column_data in table['StorageDescriptor']['Columns']]
+                        schema[table_name] = {'name': table_name, 'columns': column, 'metadata': metadata}
                         for partition in table.get('PartitionKeys', []):
                             schema[table_name]['columns'].append(partition['Name'])
+                            schema[table_name]['metadata'].append({
+                                "name": partition['Name'],
+                                "type": partition['Type']
+                            })
+
         return schema.values()
 
     def get_schema(self, get_stats=False):

redash/tasks/queries.py

@@ -378,7 +378,7 @@ def refresh_schema(data_source_id):
                 "column_metadata": "metadata" in table
             }
             new_column_names[table_name] = table['columns']
-            new_column_metadata[table_name] = table['metadata']
+            new_column_metadata[table_name] = table.get('metadata', None)
 
         insert_or_update_table_metadata(ds, existing_tables_set, table_data)
         models.db.session.flush()

tests/query_runner/test_athena.py

@@ -72,7 +72,11 @@ def test_external_table(self):
             {'DatabaseName': 'test1'},
         )
         with self.stubber:
-            assert query_runner.get_schema() == [{'columns': ['row_id'], 'name': 'test1.jdbc_table'}]
+            assert query_runner.get_schema() == [{
+                'columns': ['row_id'],
+                'name': 'test1.jdbc_table',
+                'metadata': [{'type': 'int', 'name': 'row_id'}]
+            }]
 
     def test_partitioned_table(self):
         """
@@ -118,7 +122,11 @@ def test_partitioned_table(self):
             {'DatabaseName': 'test1'},
         )
         with self.stubber:
-            assert query_runner.get_schema() == [{'columns': ['sk', 'category'], 'name': 'test1.partitioned_table'}]
+            assert query_runner.get_schema() == [{
+                'columns': ['sk', 'category'],
+                'name': 'test1.partitioned_table',
+                'metadata': [{'type': 'int', 'name': 'sk'}, {'type': 'int', 'name': 'category'}]
+            }]
 
     def test_view(self):
         query_runner = Athena({'glue': True, 'region': 'mars-east-1'})
@@ -150,7 +158,11 @@ def test_view(self):
             {'DatabaseName': 'test1'},
         )
         with self.stubber:
-            assert query_runner.get_schema() == [{'columns': ['sk'], 'name': 'test1.view'}]
+            assert query_runner.get_schema() == [{
+                'columns': ['sk'],
+                'name': 'test1.view',
+                'metadata': [{'type': 'int', 'name': 'sk'}]
+            }]
 
     def test_dodgy_table_does_not_break_schema_listing(self):
         """
@@ -187,4 +199,8 @@ def test_dodgy_table_does_not_break_schema_listing(self):
             {'DatabaseName': 'test1'},
         )
         with self.stubber:
-            assert query_runner.get_schema() == [{'columns': ['region'], 'name': 'test1.csv'}]
+            assert query_runner.get_schema() == [{
+                'columns': ['region'],
+                'name': 'test1.csv',
+                'metadata': [{'type': 'string', 'name': 'region'}]
+            }]