diff --git a/querybook/migrations/versions/27ed76f75106_extend_datatablecolumn_type_length.py b/querybook/migrations/versions/27ed76f75106_extend_datatablecolumn_type_length.py new file mode 100644 index 000000000..5dbb24ccb --- /dev/null +++ b/querybook/migrations/versions/27ed76f75106_extend_datatablecolumn_type_length.py @@ -0,0 +1,40 @@ +"""Extend DataTableColumn type length + +Revision ID: 27ed76f75106 +Revises: 63bde0162416 +Create Date: 2023-01-05 09:43:06.639449 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = "27ed76f75106" +down_revision = "63bde0162416" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "data_table_column", + "type", + nullable=True, + existing_type=sa.String(length=255), + type_=sa.String(length=4096), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "data_table_column", + "type", + nullable=True, + existing_type=sa.String(length=4096), + type_=sa.String(length=255), + ) + # ### end Alembic commands ### diff --git a/querybook/server/const/db.py b/querybook/server/const/db.py index 7964f4517..086b679df 100644 --- a/querybook/server/const/db.py +++ b/querybook/server/const/db.py @@ -3,6 +3,7 @@ now = datetime.datetime.utcnow utf8mb4_name_length = 191 name_length = 255 +type_length = 4096 password_length = 255 description_length = 5000 text_length = 65535 diff --git a/querybook/server/models/metastore.py b/querybook/server/models/metastore.py index ae5c0e397..5b9bd6d3b 100644 --- a/querybook/server/models/metastore.py +++ b/querybook/server/models/metastore.py @@ -9,6 +9,7 @@ description_length, url_length, mediumtext_length, + type_length, ) from const.metastore import DataTableWarningSeverity from lib.sqlalchemy import CRUDMixin, TruncateString @@ -283,7 +284,7 @@ class DataTableColumn(TruncateString("name", "type", "comment"), Base): updated_at = sql.Column(sql.DateTime, default=now) name = sql.Column(sql.String(length=name_length), index=True) - type = sql.Column(sql.String(length=name_length)) + type = sql.Column(sql.String(length=type_length)) comment = sql.Column(sql.String(length=description_length)) description = sql.Column(sql.Text(length=mediumtext_length)) diff --git a/querybook/webapp/__tests__/lib/utils/complex-types.test.ts b/querybook/webapp/__tests__/lib/utils/complex-types.test.ts new file mode 100644 index 000000000..431621fd4 --- /dev/null +++ b/querybook/webapp/__tests__/lib/utils/complex-types.test.ts @@ -0,0 +1,258 @@ +import { parseType, prettyPrintType } from 'lib/utils/complex-types'; + +test('simple type', () => { + expect(parseType('column', 'string')).toEqual({ + key: 'column', + type: 'string', + }); +}); + +test('truncated type', () => { + expect( + parseType( + 'column', + 'struct,hour:int,minute:int,second:int,' + ) + ).toEqual({ + key: 'column', + type: 'struct,hour:int,minute:int,second:int,', + }); + + // Truncated, but coincidentally matches the regex + expect(parseType('column', 'struct')).toEqual({ + key: 'column', + type: 'struct', + }); +}); + +test('malformed struct type', () => { + expect(parseType('column', 'STRUCT ')).toEqual({ + key: 'column', + type: 'STRUCT ', + }); +}); + +test('complex type', () => { + expect(parseType('column', 'STRUCT')).toEqual({ + key: 'column', + type: 'STRUCT', + children: [ + { + key: 'id', + type: 'string', + }, + ], + }); + + expect( + parseType( + 'column', + 'struct,hour:int,minute:int,second:int,timeZoneId:string>' + ) + ).toEqual({ + key: 'column', + type: 'struct,hour:int,minute:int,second:int,timeZoneId:string>', + children: [ + { + key: 'date', + type: 'struct', + children: [ + { key: 'year', type: 'int' }, + { key: 'month', type: 'int' }, + { key: 'day', type: 'int' }, + ], + }, + { key: 'hour', type: 'int' }, + { key: 'minute', type: 'int' }, + { key: 'second', type: 'int' }, + { key: 'timeZoneId', type: 'string' }, + ], + }); + + expect( + parseType( + 'column', + 'array>>' + ) + ).toEqual({ + key: 'column', + type: 'array>>', + children: [ + { + key: '', + type: 'struct>', + children: [ + { + key: 'size', + type: 'struct', + children: [ + { key: 'width', type: 'int' }, + { key: 'height', type: 'int' }, + { key: 'isAspectRatio', type: 'boolean' }, + ], + }, + ], + }, + ], + }); + + expect( + parseType( + 'column', + 'struct,tests:array>,user:struct>' + ) + ).toEqual({ + key: 'column', + type: 'struct,tests:array>,user:struct>', + children: [ + { key: 'purchasePath', type: 'string' }, + { key: 'resultToken', type: 'string' }, + { key: 'sessionId', type: 'string' }, + { + key: 'site', + type: 'struct', + children: [ + { key: 'eapid', type: 'bigint' }, + { key: 'tpid', type: 'bigint' }, + ], + }, + { + key: 'tests', + type: 'array>', + children: [ + { + key: '', + type: 'struct', + children: [ + { key: 'bucketValue', type: 'string' }, + { key: 'experimentId', type: 'string' }, + { key: 'instanceId', type: 'string' }, + ], + }, + ], + }, + { + key: 'user', + type: 'struct', + children: [ + { key: 'guid', type: 'string' }, + { key: 'tuid', type: 'string' }, + ], + }, + ], + }); + + expect(parseType('column', 'map')).toEqual({ + key: 'column', + type: 'map', + children: [ + { + key: '', + type: 'string', + }, + { + key: '', + type: 'float', + }, + ], + }); + + expect( + parseType( + 'column', + 'map>>' + ) + ).toEqual({ + key: 'column', + type: 'map>>', + children: [ + { + key: '', + type: 'string', + }, + { + key: '', + type: 'uniontype>', + children: [ + { key: '', type: 'string' }, + { key: '', type: 'int' }, + { key: '', type: 'bigint' }, + { key: '', type: 'float' }, + { key: '', type: 'double' }, + { + key: '', + type: 'struct', + children: [ + { key: 'year', type: 'int' }, + { key: 'month', type: 'int' }, + { key: 'day', type: 'int' }, + ], + }, + ], + }, + ], + }); +}); + +test('prettyPrintType', () => { + expect(prettyPrintType('map')).toEqual(`map< + string, + string +>`); + + expect( + prettyPrintType( + 'struct,data:uniontype>' + ) + ).toEqual(`struct< + ids: array< + string + >, + data: uniontype< + int, + float, + string + > +>`); + + expect( + prettyPrintType( + 'struct,comment:string,data:map>' + ) + ).toEqual(`struct< + ids: array< + string + >, + comment: string, + data: map< + int, + int + > +>`); + + expect( + prettyPrintType( + 'map,comment:string,data:map>,struct,event:map>>' + ) + ).toEqual(`map< + struct< + ids: array< + string + >, + comment: string, + data: map< + int, + int + > + >, + struct< + data: array< + string + >, + event: map< + int, + int + > + > +>`); +}); diff --git a/querybook/webapp/components/DataDocScheduleList/DataDocScheduleItem.tsx b/querybook/webapp/components/DataDocScheduleList/DataDocScheduleItem.tsx index 97006ed4c..8e7f537ea 100644 --- a/querybook/webapp/components/DataDocScheduleList/DataDocScheduleItem.tsx +++ b/querybook/webapp/components/DataDocScheduleList/DataDocScheduleItem.tsx @@ -38,7 +38,12 @@ export const DataDocScheduleItem: React.FC = ({ Runs - Next Run: + Next Run:{' '} + {schedule.enabled ? ( + + ) : ( + 'Disabled' + )} {lastRecord && ( @@ -99,7 +104,11 @@ export const DataDocScheduleItem: React.FC = ({
{doc.title ? ( - + {doc.title} ) : ( diff --git a/querybook/webapp/components/DataTableViewColumn/DataTableColumnCard.scss b/querybook/webapp/components/DataTableViewColumn/DataTableColumnCard.scss index f483f2cdd..6109876c0 100644 --- a/querybook/webapp/components/DataTableViewColumn/DataTableColumnCard.scss +++ b/querybook/webapp/components/DataTableViewColumn/DataTableColumnCard.scss @@ -29,3 +29,20 @@ } } } + +.DataTableColumnCardNestedType { + + .column-type { + min-width: 80px; + word-break: break-all; + } + + .nested-indent { + margin-left: 32px; + } + + .expand-icon { + margin-left: -32px; + padding: 0 8px; + } +} diff --git a/querybook/webapp/components/DataTableViewColumn/DataTableColumnCard.tsx b/querybook/webapp/components/DataTableViewColumn/DataTableColumnCard.tsx index 72be201dd..b0ca23afc 100644 --- a/querybook/webapp/components/DataTableViewColumn/DataTableColumnCard.tsx +++ b/querybook/webapp/components/DataTableViewColumn/DataTableColumnCard.tsx @@ -1,14 +1,18 @@ import { ContentState } from 'draft-js'; -import * as React from 'react'; +import React, { useMemo } from 'react'; import { DataTableColumnStats } from 'components/DataTableStats/DataTableColumnStats'; import { IDataColumn } from 'const/metastore'; +import { useToggleState } from 'hooks/useToggleState'; +import { parseType } from 'lib/utils/complex-types'; import { Card } from 'ui/Card/Card'; import { EditableTextField } from 'ui/EditableTextField/EditableTextField'; import { Icon } from 'ui/Icon/Icon'; import { KeyContentDisplay } from 'ui/KeyContentDisplay/KeyContentDisplay'; import { AccentText, StyledText } from 'ui/StyledText/StyledText'; +import { DataTableColumnCardNestedType } from './DataTableColumnCardNestedType'; + import './DataTableColumnCard.scss'; interface IProps { @@ -23,7 +27,8 @@ export const DataTableColumnCard: React.FunctionComponent = ({ column, updateDataColumnDescription, }) => { - const [expanded, setExpanded] = React.useState(false); + const [expanded, , toggleExpanded] = useToggleState(false); + const parsedType = useMemo(() => parseType('', column.type), [column.type]); const userCommentsContent = ( = ({
setExpanded(!expanded)} + onClick={() => toggleExpanded()} aria-label={ expanded ? 'click to collapse' : 'click to expand' } @@ -53,6 +58,13 @@ export const DataTableColumnCard: React.FunctionComponent = ({
{expanded ? (
+ {parsedType.children && ( + + + + )} {column.comment && ( {column.comment} diff --git a/querybook/webapp/components/DataTableViewColumn/DataTableColumnCardNestedType.tsx b/querybook/webapp/components/DataTableViewColumn/DataTableColumnCardNestedType.tsx new file mode 100644 index 000000000..88fd3419a --- /dev/null +++ b/querybook/webapp/components/DataTableViewColumn/DataTableColumnCardNestedType.tsx @@ -0,0 +1,58 @@ +import * as React from 'react'; + +import { ComplexType } from 'lib/utils/complex-types'; +import { IconButton } from 'ui/Button/IconButton'; +import { AccentText, StyledText } from 'ui/StyledText/StyledText'; + +interface IDataTableColumnCardNestedTypeProps { + complexType: ComplexType; +} +export const DataTableColumnCardNestedType: React.FunctionComponent< + IDataTableColumnCardNestedTypeProps +> = ({ complexType }) => { + const hasChildren = complexType.children?.length > 0; + const [expanded, setExpanded] = React.useState(false); + + const rowProps: React.HTMLAttributes = { + className: 'flex-row', + }; + + if (hasChildren) { + rowProps['onClick'] = () => setExpanded(!expanded); + rowProps['aria-label'] = expanded + ? 'click to collapse' + : 'click to expand'; + rowProps['data-balloon-pos'] = 'down-left'; + } + + return ( +
+
+ {hasChildren && ( + + )} + + {complexType.key} + + + {complexType.type} + +
+ {hasChildren && + expanded && + complexType.children.map((child) => ( +
+ +
+ ))} +
+ ); +}; diff --git a/querybook/webapp/components/DataTableViewMini/ColumnPanelView.tsx b/querybook/webapp/components/DataTableViewMini/ColumnPanelView.tsx index b9f95300f..60cc09b7f 100644 --- a/querybook/webapp/components/DataTableViewMini/ColumnPanelView.tsx +++ b/querybook/webapp/components/DataTableViewMini/ColumnPanelView.tsx @@ -2,6 +2,7 @@ import { ContentState } from 'draft-js'; import React from 'react'; import { useSelector } from 'react-redux'; +import { prettyPrintType } from 'lib/utils/complex-types'; import { IStoreState } from 'redux/store/types'; import { PanelSection, SubPanelSection } from './PanelSection'; @@ -22,7 +23,11 @@ export const ColumnPanelView: React.FunctionComponent< const overviewPanel = ( {column.name} - {column.type} + +
+ {prettyPrintType(column.type)} +
+ ); diff --git a/querybook/webapp/components/DataTableViewMini/TablePanelView.tsx b/querybook/webapp/components/DataTableViewMini/TablePanelView.tsx index 805485805..d90509fd8 100644 --- a/querybook/webapp/components/DataTableViewMini/TablePanelView.tsx +++ b/querybook/webapp/components/DataTableViewMini/TablePanelView.tsx @@ -1,11 +1,14 @@ import { ContentState } from 'draft-js'; -import React from 'react'; +import React, { useMemo } from 'react'; import styled from 'styled-components'; import { DataTableTags } from 'components/DataTableTags/DataTableTags'; import { useDataTable } from 'hooks/redux/useDataTable'; +import { ComplexType, parseType } from 'lib/utils/complex-types'; import { generateFormattedDate } from 'lib/utils/datetime'; +import { stopPropagationAndDefault } from 'lib/utils/noop'; import { getHumanReadableByteSize } from 'lib/utils/number'; +import { IconButton } from 'ui/Button/IconButton'; import { AllLucideIconNames } from 'ui/Icon/LucideIcons'; import { Loader } from 'ui/Loader/Loader'; @@ -111,6 +114,7 @@ export const TablePanelView: React.FunctionComponent = ({ }; interface IStyledColumnRowProps { + indent?: number; selected?: boolean; } const StyledColumnRow = styled.div` @@ -147,6 +151,13 @@ const StyledColumnRow = styled.div` max-width: 50%; } + .column-row-expand-icon { + margin-left: -16px; + padding-right: 4px; + } + + ${({ indent }) => indent && `margin-left: ${indent * 16}px`}; + ${({ selected }) => selected ? ` @@ -164,21 +175,65 @@ const StyledColumnRow = styled.div` const ColumnRow: React.FunctionComponent<{ name: string; type: string; + typeChildren?: ComplexType[]; onClick: () => any; selected?: boolean; icon?: AllLucideIconNames; -}> = ({ name, type, onClick, selected, icon }) => ( - - {name} - - {icon && ( - - )} - - {type} - -); + indent?: number; +}> = ({ name, type, typeChildren, onClick, selected, icon, indent = 0 }) => { + const [expanded, setExpanded] = React.useState(false); + const effectiveTypeChildren = useMemo( + () => + typeChildren ? typeChildren : parseType('', type).children ?? [], + [type, typeChildren] + ); + const hasChildren = effectiveTypeChildren.length > 0; + + return ( + <> + + {hasChildren && ( + { + stopPropagationAndDefault(e); + setExpanded(!expanded); + }} + /> + )} + + {name.startsWith('<') ? {name} : name} + + + {icon && ( + + )} + + {type} + + {hasChildren && + expanded && + effectiveTypeChildren?.map((child) => ( + + ))} + + ); +}; diff --git a/querybook/webapp/lib/utils/complex-types.ts b/querybook/webapp/lib/utils/complex-types.ts new file mode 100644 index 000000000..bd4a30d72 --- /dev/null +++ b/querybook/webapp/lib/utils/complex-types.ts @@ -0,0 +1,247 @@ +export interface ComplexType { + key: string; + type: string; + children?: ComplexType[]; +} + +const INDENT = ' '; + +/** + * Convert a complex Hive type string to a nested JSON object + * + * Example: 'column', 'struct,hour:int,minute:int,second:int,timeZoneId:string>' + * Output: { + key: 'column', + type: 'struct,hour:int,minute:int,second:int,timeZoneId:string>', + children: [ + { + key: 'date', + type: 'struct', + children: [ + { key: 'year', type: 'int' }, + { key: 'month', type: 'int' }, + { key: 'day', type: 'int' }, + ], + }, + { key: 'hour', type: 'int' }, + { key: 'minute', type: 'int' }, + { key: 'second', type: 'int' }, + { key: 'timeZoneId', type: 'string' }, + ], + } + */ +export function parseType(key: string, type: string): ComplexType { + const regex = /^(struct|array|map|uniontype)<(.*)>$/i; + const matches = type.match(regex); + + if (!matches || matches.length < 3) { + return { key, type }; + } + + const [_, typeName, typeContents] = matches; + + switch (typeName.toLowerCase()) { + case 'struct': + return parseStructType(key, type, typeContents); + case 'array': + return parseArrayType(key, type, typeContents); + case 'map': + return parseMapType(key, type, typeContents); + case 'uniontype': + return parseUnionType(key, type, typeContents); + default: + return { key, type }; + } +} + +export function parseStructType( + key: string, + type: string, + typeContents: string +): ComplexType { + const children = []; + let currentKey = ''; + let currentVal = ''; + let depth = 0; + + for (const char of typeContents) { + if (char === ':') { + if (depth > 0) { + currentVal += char; + } else { + currentKey = currentVal; + currentVal = ''; + } + } else if (char === ',') { + if (depth === 0) { + children.push(parseType(currentKey, currentVal)); + currentKey = ''; + currentVal = ''; + } else { + currentVal += char; + } + } else if (char === '<') { + depth += 1; + currentVal += char; + } else if (char === '>') { + depth -= 1; + currentVal += char; + } else { + currentVal += char; + } + } + + if (depth > 0) { + // Truncated or malformed type, return as-is + return { key, type }; + } + + children.push(parseType(currentKey, currentVal)); + + const structType: ComplexType = { + key, + type, + children, + }; + + return structType; +} + +export function parseMapType( + key: string, + type: string, + typeContents: string +): ComplexType { + const children: ComplexType[] = []; + let currentKey = ''; + let currentVal = ''; + let depth = 0; + + for (const char of typeContents) { + if (char === ',') { + if (depth > 0) { + currentVal += char; + } else { + currentKey = currentVal; + currentVal = ''; + } + } else if (char === '<') { + depth += 1; + currentVal += char; + } else if (char === '>') { + depth -= 1; + currentVal += char; + } else { + currentVal += char; + } + } + + if (depth > 0) { + // Truncated or malformed type, return as-is + return { key, type }; + } + + children.push(parseType('', currentKey)); + children.push(parseType('', currentVal)); + + const mapType: ComplexType = { + key, + type, + children, + }; + + return mapType; +} + +export function parseUnionType( + key: string, + type: string, + typeContents: string +): ComplexType { + const children: ComplexType[] = []; + let currentVal = ''; + let depth = 0; + + for (const char of typeContents) { + if (char === ',') { + if (depth > 0) { + currentVal += char; + } else { + children.push(parseType('', currentVal)); + currentVal = ''; + } + } else if (char === '<') { + depth += 1; + currentVal += char; + } else if (char === '>') { + depth -= 1; + currentVal += char; + } else { + currentVal += char; + } + } + + if (depth > 0) { + // Truncated or malformed type, return as-is + return { key, type }; + } + + children.push(parseType('', currentVal)); + + const unionType: ComplexType = { + key, + type, + children, + }; + + return unionType; +} + +export function parseArrayType( + key: string, + type: string, + typeContents: string +): ComplexType { + const regex = /array<(.*)>/; + const matches = type.match(regex); + + if (!matches) { + return { key, type }; + } + + return { + key, + type, + children: [parseType('', typeContents)], + }; +} + +/** + * Pretty-print a complex Hive type string using newlines and 2-space indentation. + */ +export function prettyPrintType(type: string): string { + let prettyString = ''; + let depth = 0; + + for (const char of type) { + if (char === '<') { + prettyString += '<\n'; + depth += 1; + prettyString += INDENT.repeat(depth); + } else if (char === '>') { + prettyString += '\n'; + depth -= 1; + prettyString += INDENT.repeat(depth); + prettyString += '>'; + } else if (char === ',') { + prettyString += ',\n'; + prettyString += INDENT.repeat(depth); + } else if (char === ':') { + prettyString += ': '; + } else { + prettyString += char; + } + } + + return prettyString; +} diff --git a/querybook/webapp/stylesheets/_utilities.scss b/querybook/webapp/stylesheets/_utilities.scss index 7e9fe404e..9993cb2eb 100644 --- a/querybook/webapp/stylesheets/_utilities.scss +++ b/querybook/webapp/stylesheets/_utilities.scss @@ -87,6 +87,11 @@ background-color: transparent; } +.preformatted { + white-space: pre-wrap; + word-break: break-word; +} + /* force scrollbar to appear on Mac & -webkit (Chrome, Safari) despite user 'hide' preference */ .force-scrollbar-x { overflow-x: scroll !important;