From bbe720abd1562434397fd774023d4dc351b3abc7 Mon Sep 17 00:00:00 2001 From: Andrew Schonfeld Date: Mon, 20 Apr 2020 07:43:07 -0400 Subject: [PATCH] 1.8.10 * #172: allowing groups to be specified in 3D scatter * #181: percentage sum/count charts * #179: confirmation for column deletion * #175: rename columns * #173: wider column input box for GroupBy in "Summarize Data" popup * #174, moved "Describe" popup to new browser tab * #170: filter "Value" dropdown for maps to only int or float columns * #164: show information about missing data in "Describe" popup * #184: "nan" not showing up for numeric columns * #176: highlight background of outliers/missing values --- .circleci/config.yml | 2 +- .gitignore | 1 + CHANGES.md | 12 + docker-compose.yml | 8 + .env => docker/dtale.env | 2 +- docs/source/conf.py | 4 +- dtale/charts/utils.py | 86 +++--- dtale/dash_application/charts.py | 19 +- dtale/dash_application/layout.py | 37 +-- dtale/global_state.py | 3 +- dtale/static/css/dash.css | 6 + dtale/static/css/main.css | 31 ++- dtale/views.py | 64 ++++- package.json | 2 +- setup.py | 2 +- .../__tests__/dtale/DataViewer-base-test.jsx | 3 +- .../dtale/DataViewer-correlations-test.jsx | 4 +- .../dtale/DataViewer-describe-test.jsx | 249 ++++++++---------- .../DataViewer-dtype-highlighting-test.jsx | 13 +- .../DataViewer-missing-highlighting-test.jsx | 64 +++++ .../DataViewer-outliers-highlighting-test.jsx | 64 +++++ .../dtale/create/type-conversion-test.jsx | 2 +- .../__tests__/iframe/DataViewer-base-test.jsx | 6 +- .../iframe/DataViewer-delete-test.jsx | 9 +- .../iframe/DataViewer-rename-test.jsx | 109 ++++++++ .../iframe/DataViewer-within-iframe-test.jsx | 4 +- static/__tests__/popups/Correlations-test.jsx | 5 + .../__tests__/popups/window/Describe-test.jsx | 4 +- static/__tests__/redux-test-utils.jsx | 50 ++-- static/dtale/DataViewer.jsx | 32 ++- static/dtale/DataViewerMenu.jsx | 80 ++++-- static/dtale/Header.jsx | 15 +- static/dtale/backgroundUtils.jsx | 114 ++++++++ static/dtale/gridUtils.jsx | 71 ++--- static/dtale/iframe/ColumnMenu.jsx | 39 ++- static/dtale/menu-descriptions.json | 10 +- static/dtale/serverStateManagement.jsx | 5 + static/main.jsx | 2 +- static/popups/Confirmation.css | 12 + static/popups/Confirmation.jsx | 54 ++++ static/popups/Describe.jsx | 37 +-- static/popups/Popup.jsx | 21 ++ static/popups/Rename.jsx | 102 +++++++ static/popups/describe/Details.jsx | 6 +- static/popups/reshape/Aggregate.jsx | 8 +- static/popups/reshape/Pivot.jsx | 2 +- static/popups/reshape/Reshape.css | 8 +- tests/dtale/test_dash.py | 13 +- tests/dtale/test_views.py | 38 ++- 49 files changed, 1120 insertions(+), 414 deletions(-) rename .env => docker/dtale.env (55%) create mode 100644 static/__tests__/dtale/DataViewer-missing-highlighting-test.jsx create mode 100644 static/__tests__/dtale/DataViewer-outliers-highlighting-test.jsx create mode 100644 static/__tests__/iframe/DataViewer-rename-test.jsx create mode 100644 static/dtale/backgroundUtils.jsx create mode 100644 static/popups/Confirmation.css create mode 100644 static/popups/Confirmation.jsx create mode 100644 static/popups/Rename.jsx diff --git a/.circleci/config.yml b/.circleci/config.yml index 510426c53..882d28b7f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ defaults: &defaults CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_TEST_REPORTS: /tmp/circleci-test-results CODECOV_TOKEN: b0d35139-0a75-427a-907b-2c78a762f8f0 - VERSION: 1.8.9 + VERSION: 1.8.10 PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases steps: - checkout diff --git a/.gitignore b/.gitignore index c29ee1f6a..08b4c56af 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ # test output files .cache .coverage +.DS_store .eggs .nyc_output js_junit.xml diff --git a/CHANGES.md b/CHANGES.md index f76979732..de9516d6a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,17 @@ ## Changelog +### 1.8.10 (2020-4-26) + * [#184](https://github.com/man-group/dtale/issues/184): "nan" not showing up for numeric columns + * [#181](https://github.com/man-group/dtale/issues/181): percentage sum/count charts + * [#179](https://github.com/man-group/dtale/issues/179): confirmation for column deletion + * [#176](https://github.com/man-group/dtale/issues/176): highlight background of outliers/missing values + * [#175](https://github.com/man-group/dtale/issues/175): column renaming + * [#174](https://github.com/man-group/dtale/issues/174): moved "Describe" popup to new browser tab + * [#173](https://github.com/man-group/dtale/issues/173): wider column input box for GroupBy in "Summarize Data" popup + * [#172](https://github.com/man-group/dtale/issues/172): allowing groups to be specified in 3D scatter + * [#170](https://github.com/man-group/dtale/issues/170): filter "Value" dropdown for maps to only int or float columns + * [#164](https://github.com/man-group/dtale/issues/164): show information about missing data in "Describe" popup + ### 1.8.9 (2020-4-18) * updated correlations & "Open Popup" to create new tabs instead * test fixes for dash 1.11.0 diff --git a/docker-compose.yml b/docker-compose.yml index 4fa67d95e..8950129ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,21 +3,29 @@ version: '3.4' services: dtale_2_7: image: dtale_2_7:latest + env_file: + - ./docker/dtale.env build: context: . dockerfile: docker/2_7/Dockerfile dtale_3_6: image: dtale_3_6:latest + env_file: + - ./docker/dtale.env build: context: . dockerfile: docker/3_6/Dockerfile dtale_3_7: image: dtale_3_7:latest + env_file: + - ./docker/dtale.env build: context: . dockerfile: docker/3_7/Dockerfile dtale_3_8: image: dtale_3_8:latest + env_file: + - ./docker/dtale.env build: context: . dockerfile: docker/3_8/Dockerfile diff --git a/.env b/docker/dtale.env similarity index 55% rename from .env rename to docker/dtale.env index a7a4ef52b..8a2665175 100644 --- a/.env +++ b/docker/dtale.env @@ -1,2 +1,2 @@ -VERSION=1.8.9 +VERSION=1.8.10 TZ=America/New_York \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index a5282df57..9b1910644 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -64,9 +64,9 @@ # built documents. # # The short X.Y version. -version = u'1.8.9' +version = u'1.8.10' # The full version, including alpha/beta/rc tags. -release = u'1.8.9' +release = u'1.8.10' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/dtale/charts/utils.py b/dtale/charts/utils.py index a8ea88e4c..77f411eb2 100644 --- a/dtale/charts/utils.py +++ b/dtale/charts/utils.py @@ -239,7 +239,7 @@ def check_exceptions(df, allow_duplicates, unlimited_data=False, data_limit=1500 raise ChartBuildingError(limit_msg.format(data_limit)) -def build_agg_data(df, x, y, inputs, agg, z=None, animate_by=None): +def build_agg_data(df, x, y, inputs, agg, z=None, group_col=None, animate_by=None): """ Builds aggregated data when an aggregation (sum, mean, max, min...) is selected from the front-end. @@ -282,29 +282,46 @@ def build_agg_data(df, x, y, inputs, agg, z=None, animate_by=None): ] return agg_df, code + idx_cols = make_list(animate_by) + make_list(group_col) + [x] + agg_cols = y if z_exists: - idx_cols = make_list(animate_by) + [x] + make_list(y) - groups = df.groupby(idx_cols) - groups = getattr(groups[make_list(z)], agg)() - if animate_by is not None: - full_idx = pd.MultiIndex.from_product([df[c].unique() for c in idx_cols], names=idx_cols) - groups = groups.reindex(full_idx).fillna(0) - return groups.reset_index(), [ - "chart_data = chart_data.groupby(['{cols}'])[['{z}']].{agg}().reset_index()".format( - cols="', '".join([x] + make_list(y)), z=z, agg=agg + idx_cols += make_list(y) + agg_cols = make_list(z) + + groups = df.groupby(idx_cols) + if agg in ['pctsum', 'pctct']: + func = 'sum' if agg == 'pctsum' else 'size' + subidx_cols = [c for c in idx_cols if c not in make_list(group_col)] + groups = getattr(groups[agg_cols], func)() + groups = groups / getattr(df.groupby(subidx_cols)[agg_cols], func)() * 100 + if len(agg_cols) > 1: + groups.columns = agg_cols + elif len(agg_cols) == 1: + groups.name = agg_cols[0] + code = ( + "chart_data = chart_data.groupby(['{cols}'])[['{agg_cols}']].{agg}()\n" + "chart_data = chart_data / chart_data.groupby(['{subidx_cols}']).{agg}()\n" + "chart_data = chart_data.reset_index()" + ) + code = code.format(cols="', '".join(idx_cols), subidx_cols="', '".join(subidx_cols), + agg_cols="', '".join(agg_cols), agg=func) + code = [code] + else: + groups = getattr(groups[agg_cols], agg)() + code = [ + "chart_data = chart_data.groupby(['{cols}'])[['{agg_cols}']].{agg}().reset_index()".format( + cols="', '".join(idx_cols), agg_cols="', '".join(agg_cols), agg=agg ) ] - idx_cols = make_list(animate_by) + [x] - groups = df.groupby(idx_cols) - groups = getattr(groups[y], agg)() if animate_by is not None: full_idx = pd.MultiIndex.from_product([df[c].unique() for c in idx_cols], names=idx_cols) groups = groups.reindex(full_idx).fillna(0) - return groups.reset_index(), [ - "chart_data = chart_data.groupby('{x}')[['{y}']].{agg}().reset_index()".format( - x=x, y=make_list(y)[0], agg=agg - ) - ] + code += [ + "idx_cols = ['{cols}']".format(cols="', '".join(idx_cols)), + 'full_idx = pd.MultiIndex.from_product([df[c].unique() for c in idx_cols], names=idx_cols)' + 'chart_data = chart_data.reindex(full_idx).fillna(0)' + ] + return groups.reset_index(), code def build_base_chart(raw_data, x, y, group_col=None, group_val=None, agg=None, allow_duplicates=False, return_raw=False, @@ -337,22 +354,21 @@ def build_base_chart(raw_data, x, y, group_col=None, group_val=None, agg=None, a y_cols = make_list(y) z_col = kwargs.get('z') z_cols = make_list(z_col) + sort_cols = (y_cols if len(z_cols) else []) if group_col is not None and len(group_col): main_group = group_col if animate_by is not None: main_group = [animate_by] + main_group - sort_cols = main_group + [x] + sort_cols = main_group + [x] + sort_cols data = data.sort_values(sort_cols) code.append("chart_data = chart_data.sort_values(['{cols}'])".format(cols="', '".join(sort_cols))) - check_all_nan(data, [x] + y_cols) + check_all_nan(data) data = data.rename(columns={x: x_col}) code.append("chart_data = chart_data.rename(columns={'" + x + "': '" + x_col + "'})") - if agg is not None and agg != 'raw': - data = data.groupby(main_group + [x_col]) - data = getattr(data, agg)().reset_index() - code.append("chart_data = chart_data.groupby(['{cols}']).{agg}().reset_index()".format( - cols="', '".join(main_group + [x]), agg=agg - )) + if agg is not None: + data, agg_code = build_agg_data(data, x_col, y_cols, kwargs, agg, z=z_col, group_col=group_col, + animate_by=animate_by) + code += agg_code MAX_GROUPS = 30 group_vals = data[group_col].drop_duplicates() if len(group_vals) > MAX_GROUPS: @@ -376,8 +392,8 @@ def build_base_chart(raw_data, x, y, group_col=None, group_val=None, agg=None, a data_f, range_f = build_formatters(data) ret_data = dict( data={}, - min={col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols}, - max={col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols}, + min={col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols}, + max={col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols}, ) dtypes = get_dtypes(data) @@ -408,15 +424,17 @@ def _group_filter(): main_group = [x] if animate_by is not None: main_group = [animate_by] + main_group - sort_cols = main_group + (y_cols if len(z_cols) else []) + sort_cols = main_group + sort_cols data = data.sort_values(sort_cols) code.append("chart_data = chart_data.sort_values(['{cols}'])".format(cols="', '".join(sort_cols))) - check_all_nan(data, main_group + y_cols + z_cols) + check_all_nan(data) y_cols = [str(y_col) for y_col in y_cols] data = data[main_group + y_cols + z_cols] - main_group[-1] = x_col - data.columns = main_group + y_cols + z_cols - code.append("chart_data.columns = ['{cols}']".format(cols="', '".join(main_group + y_cols + z_cols))) + + data = data.rename(columns={x: x_col}) + main_group = [x_col if c == x else c for c in main_group] + code.append("chart_data = chart_data.rename(columns={'" + x + "': '" + x_col + "'})") + if agg is not None: data, agg_code = build_agg_data(data, x_col, y_cols, kwargs, agg, z=z_col, animate_by=animate_by) code += agg_code @@ -427,7 +445,7 @@ def _group_filter(): dupe_cols = main_group + (y_cols if len(z_cols) else []) check_exceptions( - data[dupe_cols].rename(columns={'x': x}), + data[dupe_cols].rename(columns={x_col: x}), allow_duplicates or agg == 'raw', unlimited_data=unlimited_data, data_limit=40000 if len(z_cols) or animate_by is not None else 15000 diff --git a/dtale/dash_application/charts.py b/dtale/dash_application/charts.py index 4a104b991..5aa780a9b 100644 --- a/dtale/dash_application/charts.py +++ b/dtale/dash_application/charts.py @@ -385,7 +385,7 @@ def build_layout(cfg): :return: layout object :rtype: :plotly:`plotly.graph_objects.Layout ` """ - return go.Layout(**dict_merge(dict(legend=dict(orientation='h', y=1.2)), cfg)) + return go.Layout(**dict_merge(dict(legend=dict(orientation='h')), cfg)) def cpg_chunker(charts, columns=2): @@ -479,7 +479,6 @@ def build_frame(frame): id='scatter-{}-{}'.format(group or 'all', y_val), figure=figure_cfg ), group_filter=dict_merge(dict(y=y_val), {} if group is None else dict(group=group))) - return [_build_final_scatter(y2) for y2 in y] @@ -1147,7 +1146,6 @@ def build_figure_data(data_id, chart_type=None, query=None, x=None, y=None, z=No chart_kwargs['animate_by'] = animate_by if chart_type in ZAXIS_CHARTS: chart_kwargs['z'] = z - del chart_kwargs['group_col'] data, chart_code = build_base_chart(data, x, y, unlimited_data=True, **chart_kwargs) return data, code + chart_code @@ -1274,21 +1272,20 @@ def build_chart(data_id=None, **inputs): return pie_builder(data, x, y, chart_builder, **chart_inputs), range_data, code axes_builder = build_axes(data_id, x, axis_inputs, data['min'], data['max'], z=z, agg=agg) - if chart_type == 'scatter': + if chart_type in ['scatter', '3d_scatter']: + kwargs = dict(agg=agg) + if chart_type == '3d_scatter': + kwargs['z'] = z + kwargs['animate_by'] = animate_by if inputs['cpg']: scatter_charts = flatten_lists([ - scatter_builder(data, x, y, axes_builder, chart_builder, group=subgroup, agg=agg) + scatter_builder(data, x, y, axes_builder, chart_builder, group=subgroup, **kwargs) for subgroup in data['data'] ]) else: - scatter_charts = scatter_builder(data, x, y, axes_builder, chart_builder, agg=agg) + scatter_charts = scatter_builder(data, x, y, axes_builder, chart_builder, **kwargs) return cpg_chunker(scatter_charts), range_data, code - if chart_type == '3d_scatter': - chart = scatter_builder(data, x, y, axes_builder, chart_builder, z=z, agg=agg, - animate_by=animate_by) - return chart, range_data, code - if chart_type == 'surface': return surface_builder(data, x, y, z, axes_builder, chart_builder, agg=agg), range_data, code diff --git a/dtale/dash_application/layout.py b/dtale/dash_application/layout.py index 3e56e5bb3..ec1a837bd 100644 --- a/dtale/dash_application/layout.py +++ b/dtale/dash_application/layout.py @@ -172,9 +172,10 @@ def build_option(value, label=None): map_group=dict(display=True)), } AGGS = dict( - count='Count', nunique='Unique Count', sum='Sum', mean='Mean', rolling='Rolling', corr='Correlation', first='First', - last='Last', median='Median', min='Minimum', max='Maximum', std='Standard Deviation', var='Variance', - mad='Mean Absolute Deviation', prod='Product of All Items', raw='No Aggregation' + raw='No Aggregation', count='Count', nunique='Unique Count', sum='Sum', mean='Mean', rolling='Rolling', + corr='Correlation', first='First', last='Last', median='Median', min='Minimum', max='Maximum', + std='Standard Deviation', var='Variance', mad='Mean Absolute Deviation', prod='Product of All Items', + pctct='Percentage Count', pctsum='Percentage Sum' ) FREQS = ['H', 'H2', 'WD', 'D', 'W', 'M', 'Q', 'Y'] FREQ_LABELS = dict(H='Hourly', H2='Hour', WD='Weekday', W='Weekly', M='Monthly', Q='Quarterly', Y='Yearly') @@ -268,8 +269,7 @@ def build_loc_mode_hover(loc_mode): id='loc-mode-hover' ) ], - className='input-group-addon pt-1 pb-0', - style=dict(height='35.5px') + className='input-group-addon pt-1 pb-0' ) @@ -313,7 +313,7 @@ def build_error(error, tb): :param error: execption message :type error: str - :param tb: traceback + :param tb: tracebackF :type tb: str :return: error component :rtype: :dash:`dash_html_components.Div ` @@ -380,23 +380,26 @@ def build_input_options(df, **inputs): def build_map_options(df, type='choropleth', loc=None, lat=None, lon=None, map_val=None): dtypes = get_dtypes(df) cols = sorted(dtypes.keys()) - float_cols, str_cols = [], [] + float_cols, str_cols, num_cols = [], [], [] for c in cols: dtype = dtypes[c] - if classify_type(dtype) == 'F': - float_cols.append(c) - continue - if classify_type(dtype) == 'S': + classification = classify_type(dtype) + if classification == 'S': str_cols.append(c) + continue + if classification in ['F', 'I']: + num_cols.append(c) + if classification == 'F': + float_cols.append(c) lat_options = [build_option(c) for c in float_cols if c not in build_selections(lon, map_val)] lon_options = [build_option(c) for c in float_cols if c not in build_selections(lat, map_val)] loc_options = [build_option(c) for c in str_cols if c not in build_selections(map_val)] if type == 'choropleth': - val_options = [build_option(c) for c in cols if c not in build_selections(loc)] + val_options = [build_option(c) for c in num_cols if c not in build_selections(loc)] else: - val_options = [build_option(c) for c in cols if c not in build_selections(lon, lat)] + val_options = [build_option(c) for c in num_cols if c not in build_selections(lon, lat)] return loc_options, lat_options, lon_options, val_options @@ -636,7 +639,7 @@ def show_map_style(show): ) ], id='non-map-inputs', style={} if not show_map else {'display': 'none'}, - className='row pt-3 pb-3 charts-filters' + className='row p-0 charts-filters' ), html.Div( [ @@ -748,9 +751,9 @@ def show_map_style(show): html.Div([ build_input('Aggregation', dcc.Dropdown( id='agg-dropdown', - options=[build_option(v, AGGS[v]) for v in ['count', 'nunique', 'sum', 'mean', 'rolling', + options=[build_option(v, AGGS[v]) for v in ['raw', 'count', 'nunique', 'sum', 'mean', 'rolling', 'corr', 'first', 'last', 'median', 'min', 'max', - 'std', 'var', 'mad', 'prod', 'raw']], + 'std', 'var', 'mad', 'prod', 'pctsum', 'pctct']], placeholder='Select an aggregation', style=dict(width='inherit'), value=agg or 'raw', @@ -849,7 +852,7 @@ def show_map_style(show): value=inputs.get('animate_by') ), className='col-auto addon-min-width', style=animate_style, id='animate-by-input'), ], - className='row pt-3 pb-5 charts-filters' + className='row pt-3 pb-5 charts-filters', id='chart-inputs' )], id='main-inputs', className=main_input_class ), build_input('Group(s)', dcc.Dropdown( diff --git a/dtale/global_state.py b/dtale/global_state.py index 11439f7c0..cede8b21c 100644 --- a/dtale/global_state.py +++ b/dtale/global_state.py @@ -1,6 +1,5 @@ -from collections import MutableMapping - from six import PY3 +from six.moves.collections_abc import MutableMapping DATA = {} DTYPES = {} diff --git a/dtale/static/css/dash.css b/dtale/static/css/dash.css index d0fcbe4cc..4ec7ef264 100644 --- a/dtale/static/css/dash.css +++ b/dtale/static/css/dash.css @@ -530,3 +530,9 @@ div.modebar > div.modebar-group:first-child /* hide plotly "export to png" */ padding-left: 0; } +div#non-map-inputs > div, +div#chart-inputs > div { + padding-top: 0.25rem !important; + padding-bottom: 0.25rem !important; +} + diff --git a/dtale/static/css/main.css b/dtale/static/css/main.css index c26272128..ee29a65f1 100644 --- a/dtale/static/css/main.css +++ b/dtale/static/css/main.css @@ -4551,11 +4551,14 @@ button.close { } } -div.build-modal > div.modal-lg, -div.reshape-modal > div.modal-lg { +div.build-modal > div.modal-lg { min-width: 720px; } +div.reshape-modal > div.modal-lg { + min-width: 800px; +} + div.filter-modal > div.modal-lg { min-width: 1100px; } @@ -4564,10 +4567,12 @@ div.filter-modal > div.modal-lg { .modal-lg { max-width: 800px; } - div.build-modal > div.modal-lg, - div.reshape-modal > div.modal-lg { + div.build-modal > div.modal-lg { max-width: 720px; } + div.reshape-modal > div.modal-lg { + max-width: 800px; + } div.histogram-modal > div.modal-lg, div.code-modal > div.modal-lg { max-width: 700px; @@ -10599,8 +10604,7 @@ div.container-fluid.code-export > div#popup-content > div.modal-footer, { 100% { transform: rotate(360deg); } } -.dtype-highlighting -{ +.bg-icon { height: 18px; width: 18px; box-sizing: border-box; @@ -10611,7 +10615,20 @@ div.container-fluid.code-export > div#popup-content > div.modal-footer, { float: left; } -.dtype-highlighting.spin { + +.dtype-bg { + border-color: rgba(102,205,170,1.0) rgba(65,105,225,0.5) rgba(255,182,193,1.0) rgba(255,215,0,1.0); +} + +.missing-bg { + border-color: #FFF59D #FFF59D #FFCC80 #FFCC80; +} + +.outliers-bg { + border-color: dodgerblue dodgerblue red red; +} + +.dtype-bg.spin, .missing-bg.spin, .outliers-bg.spin { animation: dtype-spin 1s infinite linear; } diff --git a/dtale/views.py b/dtale/views.py index 109ce9666..31453c274 100644 --- a/dtale/views.py +++ b/dtale/views.py @@ -407,11 +407,23 @@ def _formatter(col_index, col): dtype = dtypes.get(col) if prev_dtypes and col in prev_dtypes: visible = prev_dtypes[col].get('visible', True) - dtype_data = dict(name=col, dtype=dtype, index=col_index, visible=visible) - if classify_type(dtype) in ['F', 'I'] and not data[col].isnull().all() and col in data_ranges: # floats/ints + s = data[col] + dtype_data = dict(name=col, dtype=dtype, index=col_index, visible=visible, + hasMissing=bool(s.isnull().any()), hasOutliers=False) + classification = classify_type(dtype) + if classification in ['F', 'I'] and not data[col].isnull().all() and col in data_ranges: # floats/ints col_ranges = data_ranges[col] if not any((np.isnan(v) or np.isinf(v) for v in col_ranges.values())): dtype_data = dict_merge(col_ranges, dtype_data) + + # load outlier information + o_s, o_e = calc_outlier_range(s) + if not any((np.isnan(v) or np.isinf(v) for v in [o_s, o_e])): + dtype_data['hasOutliers'] = bool(((s < o_s) | (s > o_e)).any()) + dtype_data['outlierRange'] = dict(lower=o_s, upper=o_e) + + if classification == 'S' and not dtype_data['hasMissing']: + dtype_data['hasMissing'] = bool((s.str.strip() == '').any()) return dtype_data return _formatter @@ -981,6 +993,10 @@ def load_describe(column_series, additional_aggs=None): if 'count' in desc: # pandas always returns 'count' as a float and it adds useless decimal points desc['count'] = desc['count'].split('.')[0] + missing_ct = column_series.isnull().sum() + desc['missing_pct'] = json_float((missing_ct / len(column_series) * 100).round(2)) + desc['missing_ct'] = json_int(missing_ct) + return desc, code @@ -1036,16 +1052,21 @@ def describe(data_id, column): return jsonify_error(e) +def calc_outlier_range(s): + q1 = s.quantile(0.25) + q3 = s.quantile(0.75) + iqr = q3 - q1 + iqr_lower = q1 - 1.5 * iqr + iqr_upper = q3 + 1.5 * iqr + return iqr_lower, iqr_upper + + @dtale.route('/outliers//') def outliers(data_id, column): try: df = global_state.get_data(data_id) s = df[column] - q1 = s.quantile(0.25) - q3 = s.quantile(0.75) - iqr = q3 - q1 - iqr_lower = q1 - 1.5 * iqr - iqr_upper = q3 + 1.5 * iqr + iqr_lower, iqr_upper = calc_outlier_range(s) formatter = find_dtype_formatter(find_dtype(df[column])) outliers = s[(s < iqr_lower) | (s > iqr_upper)].unique() top = len(outliers) > 100 @@ -1072,6 +1093,9 @@ def delete_col(data_id, column): try: data = global_state.get_data(data_id) data = data[[c for c in data.columns if c != column]] + curr_history = global_state.get_history(data_id) or [] + curr_history += ['df = df[[c for c in df.columns if c != "{}"]]'.format(column)] + global_state.set_history(data_id, curr_history) dtypes = global_state.get_dtypes(data_id) dtypes = [dt for dt in dtypes if dt['name'] != column] curr_settings = global_state.get_settings(data_id) @@ -1084,6 +1108,30 @@ def delete_col(data_id, column): return jsonify_error(e) +@dtale.route('/rename-col//') +def rename_col(data_id, column): + try: + rename = get_str_arg(request, 'rename') + data = global_state.get_data(data_id) + if column != rename and rename in data.columns: + return jsonify(dict(error='Column name "{}" already exists!')) + + data = data.rename(columns={column: rename}) + curr_history = global_state.get_history(data_id) or [] + curr_history += ["df = df.rename(columns={'%s': '%s'})" % (column, rename)] + global_state.set_history(data_id, curr_history) + dtypes = global_state.get_dtypes(data_id) + dtypes = [dict_merge(dt, {'name': rename}) if dt['name'] == column else dt for dt in dtypes] + curr_settings = global_state.get_settings(data_id) + curr_settings['locked'] = [rename if c == column else c for c in curr_settings.get('locked', [])] + global_state.set_data(data_id, data) + global_state.set_dtypes(data_id, dtypes) + global_state.set_settings(data_id, curr_settings) + return jsonify(success=True) + except BaseException as e: + return jsonify_error(e) + + @dtale.route('/column-filter-data//') def get_column_filter_data(data_id, column): try: @@ -1153,7 +1201,7 @@ def get_data(data_id): return jsonify({}) col_types = global_state.get_dtypes(data_id) - f = grid_formatter(col_types) + f = grid_formatter(col_types, nan_display='nan') curr_settings = global_state.get_settings(data_id) or {} if curr_settings.get('sort') != params.get('sort'): data = sort_df_for_grid(data, params) diff --git a/package.json b/package.json index 5b4d49670..463585bb6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtale", - "version": "1.8.9", + "version": "1.8.10", "description": "Visualizer for Pandas Data Structures", "main": "main.js", "directories": { diff --git a/setup.py b/setup.py index b0b879e97..c0cb357f3 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ def run_tests(self): setup( name="dtale", - version="1.8.9", + version="1.8.10", author="MAN Alpha Technology", author_email="ManAlphaTech@man.com", description="Web Client for Visualizing Pandas Objects", diff --git a/static/__tests__/dtale/DataViewer-base-test.jsx b/static/__tests__/dtale/DataViewer-base-test.jsx index 5427c8acf..511796c0d 100644 --- a/static/__tests__/dtale/DataViewer-base-test.jsx +++ b/static/__tests__/dtale/DataViewer-base-test.jsx @@ -90,7 +90,8 @@ describe("DataViewer tests", () => { .map(s => s.text()), _.concat( ["Describe", "Custom Filter", "Build Column", "Summarize Data", "Correlations", "Charts", "Heat Map"], - ["Highlight Dtypes", "Instances 1", "Code Export", "Export", "Refresh Widths", "About", "Shutdown"] + ["Highlight Dtypes", "Highlight Missing", "Highlight Outliers", "Instances 1", "Code Export", "Export"], + ["Refresh Widths", "About", "Shutdown"] ), "Should render default menu options" ); diff --git a/static/__tests__/dtale/DataViewer-correlations-test.jsx b/static/__tests__/dtale/DataViewer-correlations-test.jsx index 8eb0d6adf..447d63787 100644 --- a/static/__tests__/dtale/DataViewer-correlations-test.jsx +++ b/static/__tests__/dtale/DataViewer-correlations-test.jsx @@ -4,7 +4,6 @@ import React from "react"; import mockPopsicle from "../MockPopsicle"; import * as t from "../jest-assertions"; -import reduxUtils from "../redux-test-utils"; import { buildInnerHTML, withGlobalJquery } from "../test-utils"; const originalOffsetHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetHeight"); @@ -71,8 +70,7 @@ describe("DataViewer tests", () => { test("DataViewer: correlations", done => { const Correlations = require("../../popups/Correlations").Correlations; const ChartsBody = require("../../popups/charts/ChartsBody").default; - const store = reduxUtils.createDtaleStore(); - buildInnerHTML({ settings: "" }, store); + buildInnerHTML({ settings: "" }); Object.defineProperty(global.document, "queryCommandSupported", { value: () => true, }); diff --git a/static/__tests__/dtale/DataViewer-describe-test.jsx b/static/__tests__/dtale/DataViewer-describe-test.jsx index 9cd8bc76e..88933159f 100644 --- a/static/__tests__/dtale/DataViewer-describe-test.jsx +++ b/static/__tests__/dtale/DataViewer-describe-test.jsx @@ -1,13 +1,10 @@ import { mount } from "enzyme"; import $ from "jquery"; import React from "react"; -import { ModalClose } from "react-modal-bootstrap"; -import { Provider } from "react-redux"; import mockPopsicle from "../MockPopsicle"; import * as t from "../jest-assertions"; -import reduxUtils from "../redux-test-utils"; -import { buildInnerHTML, clickMainMenuButton, withGlobalJquery } from "../test-utils"; +import { buildInnerHTML, withGlobalJquery } from "../test-utils"; const originalOffsetHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetHeight"); const originalOffsetWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetWidth"); @@ -16,6 +13,7 @@ const originalInnerHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototyp describe("DataViewer tests", () => { const { post } = $; + const { opener } = window; beforeAll(() => { Object.defineProperty(HTMLElement.prototype, "offsetHeight", { @@ -35,6 +33,9 @@ describe("DataViewer tests", () => { value: 775, }); + delete window.opener; + window.opener = { location: { reload: jest.fn() } }; + const mockBuildLibs = withGlobalJquery(() => mockPopsicle.mock(url => { const { urlFetcher } = require("../redux-test-utils").default; @@ -64,156 +65,138 @@ describe("DataViewer tests", () => { Object.defineProperty(window, "innerWidth", originalInnerWidth); Object.defineProperty(window, "innerHeight", originalInnerHeight); $.post = post; + window.opener = opener; }); test("DataViewer: describe", done => { - const { DataViewer } = require("../../dtale/DataViewer"); - const Describe = require("../../popups/Describe").ReactDescribe; + const { Describe } = require("../../popups/Describe"); const DtypesGrid = require("../../popups/describe/DtypesGrid").DtypesGrid; const Details = require("../../popups/describe/Details").Details; - - const store = reduxUtils.createDtaleStore(); - buildInnerHTML({ settings: "" }, store); - const result = mount( - - - , - { attachTo: document.getElementById("content") } - ); + const props = { dataId: "1", chartData: { visible: true } }; + buildInnerHTML({ settings: "" }); + const result = mount(, { + attachTo: document.getElementById("content"), + }); setTimeout(() => { result.update(); - clickMainMenuButton(result, "Describe"); + let details = result.find(Details).first(); + details + .find("div.row") + .at(2) + .find("button") + .last() + .simulate("click"); setTimeout(() => { result.update(); - t.equal(result.find(Describe).length, 1, "should show describe"); - result - .find(ModalClose) - .first() - .simulate("click"); - t.equal(result.find(Describe).length, 0, "should hide describe"); - clickMainMenuButton(result, "Describe"); - setTimeout(() => { - result.update(); - let details = result.find(Details).first(); + details = result.find(Details).first(); + t.equal( details .find("div.row") + .at(3) + .find("span.font-weight-bold") + .first() + .text(), + "3 Outliers Found (top 100):" + ); + details + .find("div.row") + .at(3) + .find("a") + .simulate("click"); + setTimeout(() => { + let dtypesGrid = result.find(DtypesGrid).first(); + t.equal(dtypesGrid.find("div[role='row']").length, 5, "should render dtypes"); + + dtypesGrid + .find("div[role='columnheader']") + .first() + .simulate("click"); + dtypesGrid = result.find(DtypesGrid).first(); + t.equal( + dtypesGrid + .find("div.headerCell") + .first() + .find("svg.ReactVirtualized__Table__sortableHeaderIcon--ASC").length, + 1, + "should sort col1 ASC" + ); + dtypesGrid + .find("div[role='columnheader']") + .first() + .simulate("click"); + dtypesGrid = result.find(DtypesGrid).first(); + t.equal( + dtypesGrid + .find("div.headerCell") + .first() + .find("svg.ReactVirtualized__Table__sortableHeaderIcon--DESC").length, + 1, + "should sort col1 DESC" + ); + dtypesGrid + .find("div[role='columnheader']") + .first() + .simulate("click"); + dtypesGrid = result.find(DtypesGrid).first(); + t.equal( + dtypesGrid + .find("div.headerCell") + .first() + .find("svg.ReactVirtualized__Table__sortableHeaderIcon").length, + 0, + "should remove col1 sort" + ); + dtypesGrid + .find("div.headerCell") .at(2) - .find("button") - .last() + .find("input") + .first() + .simulate("change", { target: { value: "1" } }); + dtypesGrid = result.find(DtypesGrid).first(); + t.equal(dtypesGrid.find("div[role='row']").length, 2, "should render filtered dtypes"); + dtypesGrid + .find("div[title='col1']") + .first() .simulate("click"); setTimeout(() => { result.update(); - details = result.find(Details).first(); t.equal( - details - .find("div.row") - .at(3) - .find("span.font-weight-bold") + result + .find(Describe) + .first() + .find("h1") .first() .text(), - "3 Outliers Found (top 100):" + "col1", + "should describe col1" ); - details - .find("div.row") - .at(3) - .find("a") + dtypesGrid = result.find(DtypesGrid).first(); + dtypesGrid + .find("div.headerCell") + .at(1) + .find("i.ico-check-box") .simulate("click"); - setTimeout(() => { - let dtypesGrid = result.find(DtypesGrid).first(); - t.equal(dtypesGrid.find("div[role='row']").length, 5, "should render dtypes"); - - dtypesGrid - .find("div[role='columnheader']") - .first() - .simulate("click"); - dtypesGrid = result.find(DtypesGrid).first(); - t.equal( - dtypesGrid - .find("div.headerCell") - .first() - .find("svg.ReactVirtualized__Table__sortableHeaderIcon--ASC").length, - 1, - "should sort col1 ASC" - ); - dtypesGrid - .find("div[role='columnheader']") - .first() - .simulate("click"); - dtypesGrid = result.find(DtypesGrid).first(); - t.equal( - dtypesGrid - .find("div.headerCell") - .first() - .find("svg.ReactVirtualized__Table__sortableHeaderIcon--DESC").length, - 1, - "should sort col1 DESC" - ); - dtypesGrid - .find("div[role='columnheader']") - .first() - .simulate("click"); - dtypesGrid = result.find(DtypesGrid).first(); - t.equal( - dtypesGrid - .find("div.headerCell") - .first() - .find("svg.ReactVirtualized__Table__sortableHeaderIcon").length, - 0, - "should remove col1 sort" - ); - dtypesGrid - .find("div.headerCell") - .at(2) - .find("input") - .first() - .simulate("change", { target: { value: "1" } }); - dtypesGrid = result.find(DtypesGrid).first(); - t.equal(dtypesGrid.find("div[role='row']").length, 2, "should render filtered dtypes"); - dtypesGrid - .find("div[title='col1']") - .first() - .simulate("click"); - setTimeout(() => { - result.update(); - t.equal( - result - .find(Describe) - .first() - .find("h1") - .first() - .text(), - "col1", - "should describe col1" - ); - dtypesGrid = result.find(DtypesGrid).first(); - dtypesGrid - .find("div.headerCell") - .at(1) - .find("i.ico-check-box") - .simulate("click"); - dtypesGrid - .find("div.headerCell") - .at(1) - .find("i.ico-check-box") - .simulate("click"); - dtypesGrid - .find("i.ico-check-box") - .last() - .simulate("click"); - result - .find("div.modal-footer") - .first() - .find("button") - .first() - .simulate("click"); - expect($.post.mock.calls[0][0]).toBe("/dtale/update-visibility/1"); - $.post.mock.calls[0][2](); // execute callback - result.update(); - expect($.post.mock.calls[0][1].visibility).toBe('{"col1":false,"col2":true,"col3":true,"col4":true}'); - done(); - }, 400); - }, 400); + dtypesGrid + .find("div.headerCell") + .at(1) + .find("i.ico-check-box") + .simulate("click"); + dtypesGrid + .find("i.ico-check-box") + .last() + .simulate("click"); + result + .find("div.modal-footer") + .first() + .find("button") + .first() + .simulate("click"); + expect($.post.mock.calls[0][0]).toBe("/dtale/update-visibility/1"); + $.post.mock.calls[0][2](); // execute callback + result.update(); + expect($.post.mock.calls[0][1].visibility).toBe('{"col1":false,"col2":true,"col3":true,"col4":true}'); + done(); }, 400); }, 400); }, 400); diff --git a/static/__tests__/dtale/DataViewer-dtype-highlighting-test.jsx b/static/__tests__/dtale/DataViewer-dtype-highlighting-test.jsx index 5cd0918f8..09af2efa5 100644 --- a/static/__tests__/dtale/DataViewer-dtype-highlighting-test.jsx +++ b/static/__tests__/dtale/DataViewer-dtype-highlighting-test.jsx @@ -1,5 +1,4 @@ import { mount } from "enzyme"; -import _ from "lodash"; import React from "react"; import { Provider } from "react-redux"; @@ -11,7 +10,7 @@ import { buildInnerHTML, clickMainMenuButton, withGlobalJquery } from "../test-u const originalOffsetHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetHeight"); const originalOffsetWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetWidth"); -describe("DataViewer heatmap tests", () => { +describe("DataViewer dtypes tests", () => { beforeAll(() => { Object.defineProperty(HTMLElement.prototype, "offsetHeight", { configurable: true, @@ -53,17 +52,11 @@ describe("DataViewer heatmap tests", () => { clickMainMenuButton(result, "Highlight Dtypes"); result.update(); let dv = result.find(ReactDataViewer).instance().state; - t.deepEqual(_.pick(dv, ["dtypeHighlighting", "heatMapMode"]), { - dtypeHighlighting: true, - heatMapMode: null, - }); + t.equal(dv.backgroundMode, "dtypes"); clickMainMenuButton(result, "Highlight Dtypes"); result.update(); dv = result.find(ReactDataViewer).instance().state; - t.deepEqual(_.pick(dv, ["dtypeHighlighting", "heatMapMode"]), { - dtypeHighlighting: false, - heatMapMode: null, - }); + t.equal(dv.backgroundMode, null); done(); }, 600); diff --git a/static/__tests__/dtale/DataViewer-missing-highlighting-test.jsx b/static/__tests__/dtale/DataViewer-missing-highlighting-test.jsx new file mode 100644 index 000000000..53043bf45 --- /dev/null +++ b/static/__tests__/dtale/DataViewer-missing-highlighting-test.jsx @@ -0,0 +1,64 @@ +import { mount } from "enzyme"; +import React from "react"; +import { Provider } from "react-redux"; + +import mockPopsicle from "../MockPopsicle"; +import * as t from "../jest-assertions"; +import reduxUtils from "../redux-test-utils"; +import { buildInnerHTML, clickMainMenuButton, withGlobalJquery } from "../test-utils"; + +const originalOffsetHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetHeight"); +const originalOffsetWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetWidth"); + +describe("DataViewer missing tests", () => { + beforeAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", { + configurable: true, + value: 500, + }); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", { + configurable: true, + value: 500, + }); + + const mockBuildLibs = withGlobalJquery(() => + mockPopsicle.mock(url => { + const { urlFetcher } = require("../redux-test-utils").default; + return urlFetcher(url); + }) + ); + jest.mock("popsicle", () => mockBuildLibs); + }); + + afterAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", originalOffsetHeight); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", originalOffsetWidth); + }); + + test("DataViewer: missing highlighting", done => { + const { DataViewer, ReactDataViewer } = require("../../dtale/DataViewer"); + + const store = reduxUtils.createDtaleStore(); + buildInnerHTML({ settings: "", hideShutdown: "True", processes: 2 }, store); + const result = mount( + + + , + { attachTo: document.getElementById("content") } + ); + + setTimeout(() => { + result.update(); + clickMainMenuButton(result, "Highlight Missing"); + result.update(); + let dv = result.find(ReactDataViewer).instance().state; + t.equal(dv.backgroundMode, "missing"); + clickMainMenuButton(result, "Highlight Missing"); + result.update(); + dv = result.find(ReactDataViewer).instance().state; + t.equal(dv.backgroundMode, null); + + done(); + }, 600); + }); +}); diff --git a/static/__tests__/dtale/DataViewer-outliers-highlighting-test.jsx b/static/__tests__/dtale/DataViewer-outliers-highlighting-test.jsx new file mode 100644 index 000000000..eff474551 --- /dev/null +++ b/static/__tests__/dtale/DataViewer-outliers-highlighting-test.jsx @@ -0,0 +1,64 @@ +import { mount } from "enzyme"; +import React from "react"; +import { Provider } from "react-redux"; + +import mockPopsicle from "../MockPopsicle"; +import * as t from "../jest-assertions"; +import reduxUtils from "../redux-test-utils"; +import { buildInnerHTML, clickMainMenuButton, withGlobalJquery } from "../test-utils"; + +const originalOffsetHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetHeight"); +const originalOffsetWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetWidth"); + +describe("DataViewer outlier tests", () => { + beforeAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", { + configurable: true, + value: 500, + }); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", { + configurable: true, + value: 500, + }); + + const mockBuildLibs = withGlobalJquery(() => + mockPopsicle.mock(url => { + const { urlFetcher } = require("../redux-test-utils").default; + return urlFetcher(url); + }) + ); + jest.mock("popsicle", () => mockBuildLibs); + }); + + afterAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", originalOffsetHeight); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", originalOffsetWidth); + }); + + test("DataViewer: dtype highlighting", done => { + const { DataViewer, ReactDataViewer } = require("../../dtale/DataViewer"); + + const store = reduxUtils.createDtaleStore(); + buildInnerHTML({ settings: "", hideShutdown: "True", processes: 2 }, store); + const result = mount( + + + , + { attachTo: document.getElementById("content") } + ); + + setTimeout(() => { + result.update(); + clickMainMenuButton(result, "Highlight Outliers"); + result.update(); + let dv = result.find(ReactDataViewer).instance().state; + t.equal(dv.backgroundMode, "outliers"); + clickMainMenuButton(result, "Highlight Outliers"); + result.update(); + dv = result.find(ReactDataViewer).instance().state; + t.equal(dv.backgroundMode, null); + + done(); + }, 600); + }); +}); diff --git a/static/__tests__/dtale/create/type-conversion-test.jsx b/static/__tests__/dtale/create/type-conversion-test.jsx index 9afe3c4bf..eb1872c24 100644 --- a/static/__tests__/dtale/create/type-conversion-test.jsx +++ b/static/__tests__/dtale/create/type-conversion-test.jsx @@ -297,7 +297,7 @@ describe("DataViewer tests", () => { t.deepEqual(result.find(CreateColumn).instance().state.cfg, { col: "col4", to: "int", - from: "datetime[ns]", + from: "datetime64[ns]", unit: "ms", fmt: null, }); diff --git a/static/__tests__/iframe/DataViewer-base-test.jsx b/static/__tests__/iframe/DataViewer-base-test.jsx index d6d7d764b..5d4d366fc 100644 --- a/static/__tests__/iframe/DataViewer-base-test.jsx +++ b/static/__tests__/iframe/DataViewer-base-test.jsx @@ -119,8 +119,8 @@ describe("DataViewer iframe tests", () => { .map(s => s.text()), _.concat( ["Describe", "Custom Filter", "Build Column", "Summarize Data", "Correlations", "Charts", "Heat Map"], - ["Highlight Dtypes", "Instances 1", "Code Export", "Export", "Refresh Widths", "About", "Reload Data"], - ["Open In New Tab", "Shutdown"] + ["Highlight Dtypes", "Highlight Missing", "Highlight Outliers", "Instances 1", "Code Export", "Export"], + ["Refresh Widths", "About", "Reload Data", "Open In New Tab", "Shutdown"] ), "Should render default menu options" ); @@ -152,7 +152,7 @@ describe("DataViewer iframe tests", () => { ); t.deepEqual( colMenu.find("ul li span.font-weight-bold").map(s => s.text()), - ["Lock", "Hide", "Delete", "Describe", "Column Analysis", "Formats"], + ["Lock", "Hide", "Delete", "Rename", "Describe", "Column Analysis", "Formats"], "Should render column menu options" ); clickColMenuSubButton(result, "Asc"); diff --git a/static/__tests__/iframe/DataViewer-delete-test.jsx b/static/__tests__/iframe/DataViewer-delete-test.jsx index 723983959..b90f2c47a 100644 --- a/static/__tests__/iframe/DataViewer-delete-test.jsx +++ b/static/__tests__/iframe/DataViewer-delete-test.jsx @@ -51,8 +51,9 @@ describe("DataViewer iframe tests", () => { $.post = post; }); - test("DataViewer: hiding a column", done => { + test("DataViewer: deleting a column", done => { const { DataViewer } = require("../../dtale/DataViewer"); + const { ReactConfirmation } = require("../../popups/Confirmation"); const store = reduxUtils.createDtaleStore(); buildInnerHTML({ settings: "", iframe: "True" }, store); @@ -72,6 +73,12 @@ describe("DataViewer iframe tests", () => { .last() .simulate("click"); clickColMenuButton(result, "Delete"); + result + .find(ReactConfirmation) + .find("div.modal-footer") + .find("button") + .first() + .simulate("click"); setTimeout(() => { result.update(); t.deepEqual( diff --git a/static/__tests__/iframe/DataViewer-rename-test.jsx b/static/__tests__/iframe/DataViewer-rename-test.jsx new file mode 100644 index 000000000..392b53028 --- /dev/null +++ b/static/__tests__/iframe/DataViewer-rename-test.jsx @@ -0,0 +1,109 @@ +import { mount } from "enzyme"; +import $ from "jquery"; +import React from "react"; +import { Provider } from "react-redux"; + +import { RemovableError } from "../../RemovableError"; +import mockPopsicle from "../MockPopsicle"; +import * as t from "../jest-assertions"; +import reduxUtils from "../redux-test-utils"; +import { buildInnerHTML, withGlobalJquery } from "../test-utils"; +import { clickColMenuButton } from "./iframe-utils"; + +const originalOffsetHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetHeight"); +const originalOffsetWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetWidth"); + +describe("DataViewer iframe tests", () => { + const { post } = $; + + beforeAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", { + configurable: true, + value: 500, + }); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", { + configurable: true, + value: 500, + }); + + const mockBuildLibs = withGlobalJquery(() => + mockPopsicle.mock(url => { + const { urlFetcher } = require("../redux-test-utils").default; + return urlFetcher(url); + }) + ); + + const mockChartUtils = withGlobalJquery(() => (ctx, cfg) => { + const chartCfg = { ctx, cfg, data: cfg.data, destroyed: false }; + chartCfg.destroy = () => (chartCfg.destroyed = true); + chartCfg.getElementsAtXAxis = _evt => [{ _index: 0 }]; + return chartCfg; + }); + + jest.mock("popsicle", () => mockBuildLibs); + jest.mock("chart.js", () => mockChartUtils); + jest.mock("chartjs-plugin-zoom", () => ({})); + jest.mock("chartjs-chart-box-and-violin-plot/build/Chart.BoxPlot.js", () => ({})); + }); + + afterAll(() => { + Object.defineProperty(HTMLElement.prototype, "offsetHeight", originalOffsetHeight); + Object.defineProperty(HTMLElement.prototype, "offsetWidth", originalOffsetWidth); + $.post = post; + }); + + test("DataViewer: renaming a column", done => { + const { DataViewer } = require("../../dtale/DataViewer"); + const { ReactRename } = require("../../popups/Rename"); + + const store = reduxUtils.createDtaleStore(); + buildInnerHTML({ settings: "", iframe: "True" }, store); + const result = mount( + + + , + { + attachTo: document.getElementById("content"), + } + ); + + setTimeout(() => { + result.update(); + result + .find(".main-grid div.headerCell div") + .last() + .simulate("click"); + clickColMenuButton(result, "Rename"); + result + .find(ReactRename) + .find("div.modal-body") + .find("input") + .first() + .simulate("change", { target: { value: "col2" } }); + result.update(); + t.ok(result.find(ReactRename).find(RemovableError).length > 0); + result + .find(ReactRename) + .find("div.modal-body") + .find("input") + .first() + .simulate("change", { target: { value: "col5" } }); + result.update(); + result + .find(ReactRename) + .find("div.modal-footer") + .find("button") + .first() + .simulate("click"); + setTimeout(() => { + result.update(); + t.deepEqual( + result.find(".main-grid div.headerCell").map(hc => hc.text()), + ["col1", "col2", "col3", "col5"], + "should render column headers" + ); + done(); + }, 400); + }, 600); + }); +}); diff --git a/static/__tests__/iframe/DataViewer-within-iframe-test.jsx b/static/__tests__/iframe/DataViewer-within-iframe-test.jsx index 7df6cd4c4..0ba48fd5a 100644 --- a/static/__tests__/iframe/DataViewer-within-iframe-test.jsx +++ b/static/__tests__/iframe/DataViewer-within-iframe-test.jsx @@ -89,8 +89,8 @@ describe("DataViewer within iframe tests", () => { .map(s => s.text()), _.concat( ["Describe", "Custom Filter", "Build Column", "Summarize Data", "Correlations", "Charts", "Heat Map"], - ["Highlight Dtypes", "Instances 1", "Code Export", "Export", "Refresh Widths", "About", "Reload Data"], - ["Open In New Tab", "Shutdown"] + ["Highlight Dtypes", "Highlight Missing", "Highlight Outliers", "Instances 1", "Code Export", "Export"], + ["Refresh Widths", "About", "Reload Data", "Open In New Tab", "Shutdown"] ), "Should render default iframe menu options" ); diff --git a/static/__tests__/popups/Correlations-test.jsx b/static/__tests__/popups/Correlations-test.jsx index 723200396..5af97efa8 100644 --- a/static/__tests__/popups/Correlations-test.jsx +++ b/static/__tests__/popups/Correlations-test.jsx @@ -23,6 +23,7 @@ const originalOffsetHeight = Object.getOwnPropertyDescriptor(HTMLElement.prototy const originalOffsetWidth = Object.getOwnPropertyDescriptor(HTMLElement.prototype, "offsetWidth"); describe("Correlations tests", () => { + const { opener } = window; beforeAll(() => { Object.defineProperty(HTMLElement.prototype, "offsetHeight", { configurable: true, @@ -33,6 +34,9 @@ describe("Correlations tests", () => { value: 500, }); + delete window.opener; + window.opener = { location: { reload: jest.fn() } }; + const mockBuildLibs = withGlobalJquery(() => mockPopsicle.mock(url => { if (_.startsWith(url, "/dtale/correlations/")) { @@ -82,6 +86,7 @@ describe("Correlations tests", () => { afterAll(() => { Object.defineProperty(HTMLElement.prototype, "offsetHeight", originalOffsetHeight); Object.defineProperty(HTMLElement.prototype, "offsetWidth", originalOffsetWidth); + window.opener = opener; }); test("Correlations rendering data", done => { diff --git a/static/__tests__/popups/window/Describe-test.jsx b/static/__tests__/popups/window/Describe-test.jsx index 0ccf3b87c..6394f440f 100644 --- a/static/__tests__/popups/window/Describe-test.jsx +++ b/static/__tests__/popups/window/Describe-test.jsx @@ -58,7 +58,7 @@ describe("Describe tests", () => { }); test("Describe: dtypes error", done => { - const Describe = require("../../../popups/Describe").ReactDescribe; + const { Describe } = require("../../../popups/Describe"); buildInnerHTML({ settings: "" }); const result = mount(, { attachTo: document.getElementById("content"), @@ -72,7 +72,7 @@ describe("Describe tests", () => { }); test("Describe: describe error", done => { - const Describe = require("../../../popups/Describe").ReactDescribe; + const { Describe } = require("../../../popups/Describe"); buildInnerHTML({ settings: "" }); const result = mount(, { attachTo: document.getElementById("content"), diff --git a/static/__tests__/redux-test-utils.jsx b/static/__tests__/redux-test-utils.jsx index 5261efb6a..66729db6a 100644 --- a/static/__tests__/redux-test-utils.jsx +++ b/static/__tests__/redux-test-utils.jsx @@ -13,31 +13,45 @@ import scatterData from "./data/scatter"; const pjson = require("../../package.json"); +const DTYPES = { + dtypes: [ + { + name: "col1", + index: 0, + dtype: "int64", + min: 2, + max: 5, + visible: true, + hasMissing: true, + hasOutliers: false, + }, + { + name: "col2", + index: 1, + dtype: "float64", + min: 2.5, + max: 5.5, + visible: true, + hasMissing: false, + hasOutliers: false, + outlierRange: { lower: 3.5, upper: 4.5 }, + }, + { name: "col3", index: 2, dtype: "object", visible: true }, + { name: "col4", index: 3, dtype: "datetime64[ns]", visible: true }, + ], + success: true, +}; + const DATA = { results: [ { dtale_index: 0, col1: 1, col2: 2.5, col3: "foo", col4: "2000-01-01" }, { dtale_index: 1, col1: 2, col2: 3.5, col3: "foo", col4: "2000-01-01" }, { dtale_index: 2, col1: 3, col2: 4.5, col3: "foo", col4: "2000-01-01" }, { dtale_index: 3, col1: 4, col2: 5.5, col3: "foo" }, + { dtale_index: 4, col1: "nan", col2: 5.5, col3: "foo" }, ], - columns: [ - { name: "dtale_index", dtype: "int64", visible: true }, - { name: "col1", dtype: "int64", min: 2, max: 5, visible: true }, - { name: "col2", dtype: "float64", min: 2.5, max: 5.5, visible: true }, - { name: "col3", dtype: "object", visible: true }, - { name: "col4", dtype: "datetime64[ns]", visible: true }, - ], - total: 4, - success: true, -}; - -const DTYPES = { - dtypes: [ - { index: 0, name: "col1", dtype: "int64", visible: true }, - { index: 1, name: "col2", dtype: "float64", visible: true }, - { index: 2, name: "col3", dtype: "object", visible: true }, - { index: 3, name: "col4", dtype: "datetime[ns]", visible: true }, - ], + columns: _.concat([{ name: "dtale_index", dtype: "int64", visible: true }], DTYPES.dtypes), + total: 5, success: true, }; diff --git a/static/dtale/DataViewer.jsx b/static/dtale/DataViewer.jsx index fc7b5c775..c16026316 100644 --- a/static/dtale/DataViewer.jsx +++ b/static/dtale/DataViewer.jsx @@ -17,6 +17,7 @@ import { DataViewerInfo } from "./DataViewerInfo"; import { DataViewerMenu } from "./DataViewerMenu"; import { Header } from "./Header"; import { MeasureText } from "./MeasureText"; +import bu from "./backgroundUtils"; import { exports as gu } from "./gridUtils"; import { ColumnMenu } from "./iframe/ColumnMenu"; @@ -28,6 +29,7 @@ class ReactDataViewer extends React.Component { super(props); this.state = gu.buildState(props); this._cellRenderer = this._cellRenderer.bind(this); + this._cellRenderer = this._cellRenderer.bind(this); this._onSectionRendered = this._onSectionRendered.bind(this); this.propagateState = this.propagateState.bind(this); this.getData = this.getData.bind(this); @@ -35,17 +37,22 @@ class ReactDataViewer extends React.Component { propagateState(state, callback = _.noop) { if (_.has(state, "columns") && !_.get(state, "formattingUpdate", false)) { - state.columns = _.map(state.columns, c => _.assignIn(c, { width: gu.calcColWidth(c, this.state) })); - const totalRange = gu.getTotalRange(state.columns); - state.min = totalRange.min; - state.max = totalRange.max; + state.columns = gu.updateColWidths(this.state, state); + state = _.assignIn(state, gu.getTotalRange(state.columns)); + } + if (_.has(state, "renameUpdate")) { + state.data = state.renameUpdate(this.state.data); + } + if (_.has(state, "triggerBgResize")) { + state.columns = gu.updateColWidths(this.state, state); + state.triggerResize = true; } if (_.get(state, "refresh", false)) { this.getData(this.state.ids, true); callback(); return; } - this.setState(_.omit(state, "formattingUpdate"), callback); + this.setState(_.omit(state, ["formattingUpdate", "renameUpdate", "triggerBgResize"]), callback); } componentDidMount() { @@ -86,7 +93,7 @@ class ReactDataViewer extends React.Component { } getData(ids, refresh = false) { - const { loading, loadQueue, heatMapMode, dtypeHighlighting } = this.state; + const { loading, loadQueue, backgroundMode } = this.state; const data = this.state.data || {}; if (loading) { this.setState({ loadQueue: _.concat(loadQueue, [ids]) }); @@ -136,8 +143,7 @@ class ReactDataViewer extends React.Component { error: null, traceback: null, loading: false, - heatMapMode, - dtypeHighlighting, + backgroundMode, }; const { columns } = this.state; if (_.isEmpty(columns)) { @@ -195,15 +201,7 @@ class ReactDataViewer extends React.Component { const rec = _.get(this.state, ["data", rowIndex - 1, colCfg.name], {}); value = rec.view; valueStyle = _.get(rec, "style", {}); - if (this.state.heatMapMode === "col") { - valueStyle = _.assignIn(gu.heatMapBackground(rec, colCfg), valueStyle); - } - if (this.state.heatMapMode === "all" && colCfg.name !== gu.IDX) { - valueStyle = _.assignIn(gu.heatMapBackground(rec, this.state), valueStyle); - } - if (this.state.dtypeHighlighting) { - valueStyle = _.assignIn(gu.dtypeHighlighting(colCfg), valueStyle); - } + valueStyle = bu.updateBackgroundStyles(this.state, valueStyle, colCfg, rec); if (_.includes(["string", "date"], gu.findColType(colCfg.dtype)) && rec.raw !== rec.view) { divProps.title = rec.raw; } diff --git a/static/dtale/DataViewerMenu.jsx b/static/dtale/DataViewerMenu.jsx index 9d57a6c30..52b012ed1 100644 --- a/static/dtale/DataViewerMenu.jsx +++ b/static/dtale/DataViewerMenu.jsx @@ -5,6 +5,7 @@ import { connect } from "react-redux"; import ConditionalRender from "../ConditionalRender"; import { openChart } from "../actions/charts"; +import bu from "./backgroundUtils"; import menuFuncs from "./dataViewerMenuUtils"; import Descriptions from "./menu-descriptions.json"; @@ -20,22 +21,26 @@ class ReactDataViewerMenu extends React.Component { this.props.openChart(_.assignIn({ type, title: _.capitalize(type) }, this.props)); } }; + const openTab = type => () => window.open(menuFuncs.fullPath(`/dtale/popup/${type}`, dataId), "_blank"); const openCodeExport = () => menuFuncs.open("/dtale/popup/code-export", dataId, 450, 700); - const openCorrelations = () => window.open(menuFuncs.fullPath("/dtale/popup/correlations", dataId), "_blank"); const refreshWidths = () => this.props.propagateState({ columns: _.map(this.props.columns, c => _.assignIn({}, c)), }); - const toggleHeatMap = mode => () => - this.props.propagateState({ - heatMapMode: this.props.heatMapMode == mode ? null : mode, - dtypeHighlighting: false, - }); - const toggleDtypeHighlighting = () => - this.props.propagateState({ - dtypeHighlighting: !this.props.dtypeHighlighting, - heatMapMode: null, - }); + const resizeBgs = ["outliers", "missing"]; + const bgState = bgType => ({ + backgroundMode: this.props.backgroundMode === bgType ? null : bgType, + triggerBgResize: _.includes(resizeBgs, this.props.backgroundMode) || _.includes(resizeBgs, bgType), + }); + const toggleBackground = bgType => () => this.props.propagateState(bgState(bgType)); + const toggleOutlierBackground = () => { + const updatedState = bgState("outliers"); + if (updatedState.backgroundMode === "outliers") { + updatedState.columns = _.map(this.props.columns, bu.buildOutlierScales); + } + this.props.propagateState(updatedState); + }; + const heatmapActive = _.startsWith(this.props.backgroundMode, "heatmap"); const exportFile = tsv => () => window.open(`/dtale/data-export/${dataId}?tsv=${tsv}&_id=${new Date().getTime()}`, "_blank"); return ( @@ -47,7 +52,7 @@ class ReactDataViewerMenu extends React.Component {
  • - @@ -83,7 +88,7 @@ class ReactDataViewerMenu extends React.Component {
  • - @@ -101,27 +106,25 @@ class ReactDataViewerMenu extends React.Component {
    {Descriptions.charts}
    • -
  • +
  • - - - - {"Heat Map"} + + {"Heat Map"}
    {_.map( [ - ["By Col", "col"], - ["Overall", "all"], + ["By Col", "heatmap-col"], + ["Overall", "heatmap-all"], ], ([label, mode]) => ( ) )} @@ -130,15 +133,37 @@ class ReactDataViewerMenu extends React.Component {
  • -
    {Descriptions.highlight_dtypes}
    • +
  • + + + +
    {Descriptions.highlight_missings}
    +
    • +
  • + + + +
    {Descriptions.highlight_outliers}
    +
    • -
  • +
  • @@ -254,8 +279,7 @@ ReactDataViewerMenu.propTypes = { menuOpen: PropTypes.bool, propagateState: PropTypes.func, openChart: PropTypes.func, - heatMapMode: PropTypes.string, - dtypeHighlighting: PropTypes.bool, + backgroundMode: PropTypes.string, hideShutdown: PropTypes.bool, dataId: PropTypes.string.isRequired, }; diff --git a/static/dtale/Header.jsx b/static/dtale/Header.jsx index f8784f426..6f56614bb 100644 --- a/static/dtale/Header.jsx +++ b/static/dtale/Header.jsx @@ -5,6 +5,7 @@ import { connect } from "react-redux"; import actions from "../actions/dtale"; import menuUtils from "../menuUtils"; +import bu from "./backgroundUtils"; import { exports as gu } from "./gridUtils"; import { ignoreMenuClicks } from "./iframe/ColumnMenu"; @@ -64,13 +65,19 @@ class ReactHeader extends React.Component { const sortDir = (_.find(sortInfo, ([col, _dir]) => col === colName) || [null, null])[1]; let headerStyle = _.assignIn({}, style); let colNameMarkup = colName; - if (this.props.dtypeHighlighting) { - headerStyle = _.assignIn(gu.dtypeHighlighting(colCfg), headerStyle); + if (this.props.backgroundMode === "dtypes") { + headerStyle = _.assignIn(bu.dtypeHighlighting(colCfg), headerStyle); colNameMarkup =
    {colName}
    ; } + if (this.props.backgroundMode === "missing" && colCfg.hasMissing) { + colNameMarkup = `${bu.missingIcon}${colName}`; + } + if (this.props.backgroundMode === "outliers" && colCfg.hasOutliers) { + colNameMarkup = `${bu.outlierIcon} ${colName}`; + } return (
    -
    +
    {_.get(SORT_CHARS, sortDir, "")} {colNameMarkup}
    @@ -89,7 +96,7 @@ ReactHeader.propTypes = { rowCount: PropTypes.number, toggleColumnMenu: PropTypes.func, hideColumnMenu: PropTypes.func, - dtypeHighlighting: PropTypes.bool, + backgroundMode: PropTypes.string, }; const ReduxHeader = connect( diff --git a/static/dtale/backgroundUtils.jsx b/static/dtale/backgroundUtils.jsx new file mode 100644 index 000000000..be0b45001 --- /dev/null +++ b/static/dtale/backgroundUtils.jsx @@ -0,0 +1,114 @@ +import chroma from "chroma-js"; +import _ from "lodash"; + +import { exports as gu } from "./gridUtils"; + +const heatMap = chroma.scale(["red", "yellow", "green"]).domain([0, 0.5, 1]); + +const heatMapBackground = ({ raw, view }, { min, max }) => { + if (view === "") { + return {}; + } + const factor = min * -1; + return { background: heatMap((raw + factor) / (max + factor)) }; +}; + +const dtypeHighlighting = ({ name, dtype }) => { + if (name === gu.IDX) { + return {}; + } + const lowerDtype = (dtype || "").toLowerCase(); + const colType = gu.findColType(lowerDtype); + if (_.startsWith(lowerDtype, "category")) { + return { background: "#E1BEE7" }; + } else if (_.startsWith(lowerDtype, "timedelta")) { + return { background: "#FFCC80" }; + } else if (colType === "float") { + return { background: "#B2DFDB" }; + } else if (colType === "int") { + return { background: "#BBDEFB" }; + } else if (colType === "date") { + return { background: "#F8BBD0" }; + } else if (colType === "string") { + return {}; + } else if (_.startsWith(lowerDtype, "bool")) { + return { background: "#FFF59D" }; + } + return {}; +}; + +const missingHighlighting = ({ name, dtype, hasMissing }, value) => { + if (name === gu.IDX || !hasMissing) { + return {}; + } + if (value === "nan") { + return { background: "#FFF59D" }; + } + const lowerDtype = (dtype || "").toLowerCase(); + const colType = gu.findColType(lowerDtype); + if (colType === "string") { + if ((value || "") === "") { + return { background: "#FFCC80" }; + } + if (_.trim(value) === "") { + return { background: "#FFCC80" }; + } + } + return {}; +}; + +const buildOutlierScales = colCfg => { + const { name, min, max, hasOutliers, outlierRange } = colCfg; + const updatedColCfg = _.assignIn({}, colCfg); + if (name === gu.IDX || !hasOutliers) { + return updatedColCfg; + } + if (!_.has(outlierRange, "lowerScale")) { + updatedColCfg.outlierRange.lowerScale = chroma.scale(["dodgerblue", "white"]).domain([min, outlierRange.lower]); + } + if (!_.has(outlierRange, "upperScale")) { + updatedColCfg.outlierRange.upperScale = chroma.scale(["white", "red"]).domain([outlierRange.upper, max]); + } + return updatedColCfg; +}; + +const outlierHighlighting = ({ name, dtype, hasOutliers, outlierRange }, { raw }) => { + if (name === gu.IDX || !hasOutliers) { + return {}; + } + const lowerDtype = (dtype || "").toLowerCase(); + const colType = gu.findColType(lowerDtype); + if (_.includes(["float", "int"], colType)) { + if (raw < outlierRange.lower) { + return { background: outlierRange.lowerScale(raw) }; + } else if (raw > outlierRange.upper) { + return { background: outlierRange.upperScale(raw) }; + } + } + return {}; +}; + +const updateBackgroundStyles = (state, valueStyle, colCfg, rec) => { + switch (state.backgroundMode) { + case "heatmap-col": + return _.assignIn(heatMapBackground(rec, colCfg), valueStyle); + case "heatmap-all": + return colCfg.name === gu.IDX ? valueStyle : _.assignIn(heatMapBackground(rec, state), valueStyle); + case "dtypes": + return _.assignIn(dtypeHighlighting(colCfg), valueStyle); + case "missing": + return _.assignIn(missingHighlighting(colCfg, rec.view), valueStyle); + case "outliers": + return _.assignIn(outlierHighlighting(colCfg, rec), valueStyle); + default: + return valueStyle; + } +}; + +export default { + missingIcon: String.fromCodePoint(10071), // "!" emoji + outlierIcon: String.fromCodePoint(11088), // star emoji + dtypeHighlighting, + updateBackgroundStyles, + buildOutlierScales, +}; diff --git a/static/dtale/gridUtils.jsx b/static/dtale/gridUtils.jsx index 04d4e2de6..9be20e844 100644 --- a/static/dtale/gridUtils.jsx +++ b/static/dtale/gridUtils.jsx @@ -1,4 +1,3 @@ -import chroma from "chroma-js"; import _ from "lodash"; import moment from "moment"; import numeral from "numeral"; @@ -35,7 +34,7 @@ EXPORTS.findColType = dtype => { }; function buildNumeral(val, fmt) { - return numeral(val).format(fmt); + return _.includes(["nan", "inf"], val) ? val : numeral(val).format(fmt); } function buildValue({ name, dtype }, rawValue, { columnFormats }) { @@ -70,13 +69,16 @@ function getHeatActive(column) { return (_.has(column, "min") || column.name === EXPORTS.IDX) && column.visible; } -EXPORTS.getActiveCols = ({ columns, heatMapMode }) => - _.filter(columns || [], c => (heatMapMode ? getHeatActive(c) : c.visible)); +EXPORTS.getActiveCols = ({ columns, backgroundMode }) => { + const heatmapActive = _.startsWith(backgroundMode, "heatmap"); + return _.filter(columns || [], c => (heatmapActive ? getHeatActive(c) : c.visible)); +}; -EXPORTS.getCol = (index, { columns, heatMapMode }) => _.get(EXPORTS.getActiveCols({ columns, heatMapMode }), index, {}); +EXPORTS.getCol = (index, { columns, backgroundMode }) => + _.get(EXPORTS.getActiveCols({ columns, backgroundMode }), index, {}); -EXPORTS.getColWidth = (index, { columns, heatMapMode }) => - _.get(EXPORTS.getCol(index, { columns, heatMapMode }), "width", DEFAULT_COL_WIDTH); +EXPORTS.getColWidth = (index, { columns, backgroundMode }) => + _.get(EXPORTS.getCol(index, { columns, backgroundMode }), "width", DEFAULT_COL_WIDTH); EXPORTS.getRanges = array => { const ranges = []; @@ -93,14 +95,19 @@ EXPORTS.getRanges = array => { return ranges; }; -EXPORTS.calcColWidth = ({ name, dtype }, { data, rowCount, sortInfo }) => { - let w = DEFAULT_COL_WIDTH; +EXPORTS.calcColWidth = ({ name, dtype, hasMissing, hasOutliers }, { data, rowCount, sortInfo, backgroundMode }) => { + let w; if (name === EXPORTS.IDX) { w = measureText(rowCount - 1 + ""); w = w < DEFAULT_COL_WIDTH ? DEFAULT_COL_WIDTH : w; } else { const sortDir = (_.find(sortInfo, ([col, _dir]) => col === name) || [null, null])[1]; - const headerWidth = measureText(name) + (_.includes(["ASC", "DESC"], sortDir) ? 10 : 0); + let headerWidth = measureText(name) + (_.includes(["ASC", "DESC"], sortDir) ? 10 : 0); + if (backgroundMode === "missing" && hasMissing) { + headerWidth += 10; // "!" emoji + } else if (backgroundMode === "outliers" && hasOutliers) { + headerWidth += 15; // star emoji + } switch (EXPORTS.findColType((dtype || "").toLowerCase())) { case "date": { let maxText = _.last(_.sortBy(data, d => _.get(d, [name, "view", "length"], 0))); @@ -154,8 +161,6 @@ EXPORTS.buildGridStyles = (headerHeight = EXPORTS.HEADER_HEIGHT) => ({ hideBottomLeftGridScrollbar: true, }); -const heatMap = chroma.scale(["red", "yellow", "green"]).domain([0, 0.5, 1]); - EXPORTS.getTotalRange = columns => { const activeCols = EXPORTS.getActiveCols({ columns }); return { @@ -164,38 +169,6 @@ EXPORTS.getTotalRange = columns => { }; }; -EXPORTS.heatMapBackground = ({ raw, view }, { min, max }) => { - if (view === "") { - return {}; - } - const factor = min * -1; - return { background: heatMap((raw + factor) / (max + factor)) }; -}; - -EXPORTS.dtypeHighlighting = ({ name, dtype }) => { - if (name === EXPORTS.IDX) { - return {}; - } - const lowerDtype = (dtype || "").toLowerCase(); - const colType = EXPORTS.findColType(lowerDtype); - if (_.startsWith(lowerDtype, "category")) { - return { background: "#E1BEE7" }; - } else if (_.startsWith(lowerDtype, "timedelta")) { - return { background: "#FFCC80" }; - } else if (colType === "float") { - return { background: "#B2DFDB" }; - } else if (colType === "int") { - return { background: "#BBDEFB" }; - } else if (colType === "date") { - return { background: "#F8BBD0" }; - } else if (colType === "string") { - return {}; - } else if (_.startsWith(lowerDtype, "bool")) { - return { background: "#FFF59D" }; - } - return {}; -}; - EXPORTS.SORT_PROPS = [ { dir: "ASC", @@ -238,8 +211,7 @@ EXPORTS.buildState = props => ({ menuOpen: false, formattingOpen: false, triggerResize: false, - heatMapMode: null, - dtypeHighlighting: false, + backgroundMode: null, }); EXPORTS.noHidden = columns => !_.some(columns, { visible: false }); @@ -258,4 +230,11 @@ EXPORTS.hasNoInfo = ({ sortInfo, query, columns, columnFilters, outlierFilters } return hideSort && hideFilter && hideHidden; }; +EXPORTS.updateColWidths = (currState, newState) => + _.map(_.get(newState, "columns", currState.columns), c => + _.assignIn(c, { + width: EXPORTS.calcColWidth(c, _.assignIn(currState, newState)), + }) + ); + export { EXPORTS as exports }; diff --git a/static/dtale/iframe/ColumnMenu.jsx b/static/dtale/iframe/ColumnMenu.jsx index 29ff465c2..3b3ad997d 100644 --- a/static/dtale/iframe/ColumnMenu.jsx +++ b/static/dtale/iframe/ColumnMenu.jsx @@ -112,6 +112,13 @@ class ReactColumnMenu extends React.Component { ); } }; + const openDescribe = () => + window.open( + buildURLString(menuFuncs.fullPath("/dtale/popup/describe", dataId), { + selectedCol, + }), + "_blank" + ); const openFormatting = () => this.props.propagateState({ formattingOpen: true, @@ -126,11 +133,23 @@ class ReactColumnMenu extends React.Component { }; serverState.toggleVisibility(dataId, selectedCol, hideCallback); }; - const deleteCol = () => - this.props.propagateState( - { columns: _.reject(this.props.columns, { name: selectedCol }) }, - serverState.deleteColumn(dataId, selectedCol) - ); + const deleteCol = () => { + const yesAction = () => + this.props.propagateState( + { columns: _.reject(this.props.columns, { name: selectedCol }) }, + serverState.deleteColumn(dataId, selectedCol) + ); + const msg = `Are you sure you want to delete the column "${selectedCol}"?`; + const title = `Delete column - ${selectedCol}`; + openChart({ type: "confirm", title, msg, yesAction, size: "modal-sm" }); + }; + const renameCol = () => + openChart({ + type: "rename", + selectedCol, + columns: this.props.columns, + size: "modal-sm", + }); return (
  • - + +
    • +
  • + + diff --git a/static/dtale/menu-descriptions.json b/static/dtale/menu-descriptions.json index 58d6bec14..2ccd1f6c6 100644 --- a/static/dtale/menu-descriptions.json +++ b/static/dtale/menu-descriptions.json @@ -2,13 +2,15 @@ "describe": "Describe column's values (Top unique values, Min, Max, Sum, STD, Var,...)", "filter": "Defining custom Filtering for the main spreadsheet, Charts, ...", "build": "Creating a/multi new column based on the existing columns or random values", - "reshape": "Create a new grid using either PIVOT table, Aggregations w/ GroupBy or Transpose", + "reshape": "Create a new grid using either PIVOT table, Aggregations w/ GroupBy or Transpose.", "corr": "Presenting correlations matrix and chart", "charts": "Plot chart (line, bar, scatter, pie, wordcloud,...)", "heatmap": "Presenting heatmap for numerical values in the main table. 'Overall' will calculate over the min/max of every column whereas 'By Col' will calculate each column individually by its min/max.", - "highlight_dtypes": "Highlight columns based on pandas dtypes. For dtype's name hover the mouse over column name", - "instances": "Show existing datasets loaded to D-Tale", - "code": "Code snippet of commands performed upon your data to produce this grid", + "highlight_dtypes": "Highlight columns based on pandas dtypes. For dtype's name hover the mouse over column name.", + "highlight_missings": "Highlight any cells which contain \"nan\" values (or in the case of strings, empty strings or strings consisting only of spaces). Also, a \"!\" will be prepended to any column header which contains missing values.", + "highlight_outliers": "Highlight any cells for numeric columns which surpass the upper or lower bounds of a custom outlier computation. Lower bounds outliers will be on a red scale and upper bounds outliers will be on a blue scale. Also, a \"★\" will be prepended to any column header which contains outliers.", + "instances": "Show existing datasets loaded to D-Tale.", + "code": "Code snippet of commands performed upon your data to produce this grid.", "export": "Download DataFrame as CSV or TSV", "widths": "Auto-Resize the column widths", "about": "Link to the source code repository, documentation", diff --git a/static/dtale/serverStateManagement.jsx b/static/dtale/serverStateManagement.jsx index 7bc0df5e0..c41639d3d 100644 --- a/static/dtale/serverStateManagement.jsx +++ b/static/dtale/serverStateManagement.jsx @@ -126,6 +126,10 @@ function updateSettings(settings, dataId, callback = _.noop) { }); } +function renameColumn(dataId, col, rename, callback) { + fetchJson(buildURLString(`/dtale/rename-col/${dataId}/${col}`, { rename }), callback); +} + export default { moveToFront: (selectedCol, props) => moveTo(selectedCol, props, "front"), moveToBack: (selectedCol, props) => moveTo(selectedCol, props, "back"), @@ -138,4 +142,5 @@ export default { toggleVisibility: (dataId, toggle, callback) => persistVisibility(dataId, { toggle }, callback), updateSettings, deleteColumn: (dataId, col) => () => fetchJson(buildURLString(`/dtale/delete-col/${dataId}/${col}`), _.noop), + renameColumn, }; diff --git a/static/main.jsx b/static/main.jsx index ad371ebe3..97483e967 100644 --- a/static/main.jsx +++ b/static/main.jsx @@ -9,7 +9,7 @@ import { DataViewer } from "./dtale/DataViewer"; import { CodeExport } from "./popups/CodeExport"; import { CodePopup } from "./popups/CodePopup"; import { Correlations } from "./popups/Correlations"; -import { ReactDescribe as Describe } from "./popups/Describe"; +import { Describe } from "./popups/Describe"; import { ReactFilter as Filter } from "./popups/Filter"; import Instances from "./popups/Instances"; import { ReactColumnAnalysis as ColumnAnalysis } from "./popups/analysis/ColumnAnalysis"; diff --git a/static/popups/Confirmation.css b/static/popups/Confirmation.css new file mode 100644 index 000000000..b17392d3d --- /dev/null +++ b/static/popups/Confirmation.css @@ -0,0 +1,12 @@ +.modal-footer.confirmation { + justify-content: center; +} + +.modal-footer.confirmation > .btn { + min-width: 8em; + border: 1px solid transparent; +} + +.modal-footer.confirmation > :not(:last-child) { + margin-right: unset; +} diff --git a/static/popups/Confirmation.jsx b/static/popups/Confirmation.jsx new file mode 100644 index 000000000..797d12d6e --- /dev/null +++ b/static/popups/Confirmation.jsx @@ -0,0 +1,54 @@ +import _ from "lodash"; +import PropTypes from "prop-types"; +import React from "react"; +import { connect } from "react-redux"; + +import { closeChart } from "../actions/charts"; + +require("./Confirmation.css"); + +class ReactConfirmation extends React.Component { + constructor(props) { + super(props); + } + + render() { + const { msg, yesAction } = this.props.chartData; + const fullYesAction = () => { + yesAction(); + this.props.onClose(); + }; + return [ +
    +
    +
    {msg}
    +
    +
    , +
    + + +
    , + ]; + } +} +ReactConfirmation.displayName = "Confirmation"; +ReactConfirmation.propTypes = { + chartData: PropTypes.shape({ + visible: PropTypes.bool.isRequired, + title: PropTypes.string, + msg: PropTypes.string, + yesAction: PropTypes.func, + }), + onClose: PropTypes.func, +}; + +const ReduxConfirmation = connect( + state => _.pick(state, ["chartData"]), + dispatch => ({ onClose: chartData => dispatch(closeChart(chartData || {})) }) +)(ReactConfirmation); + +export { ReactConfirmation, ReduxConfirmation as Confirmation }; diff --git a/static/popups/Describe.jsx b/static/popups/Describe.jsx index 475665eed..daeb31221 100644 --- a/static/popups/Describe.jsx +++ b/static/popups/Describe.jsx @@ -1,18 +1,16 @@ import _ from "lodash"; import PropTypes from "prop-types"; import React from "react"; -import { connect } from "react-redux"; import { BouncerWrapper } from "../BouncerWrapper"; import { RemovableError } from "../RemovableError"; -import { closeChart } from "../actions/charts"; import { dtypesUrl } from "../actions/url-utils"; import serverState from "../dtale/serverStateManagement"; import { fetchJson } from "../fetcher"; import { Details } from "./describe/Details"; import { DtypesGrid } from "./describe/DtypesGrid"; -class ReactDescribe extends React.Component { +class Describe extends React.Component { constructor(props) { super(props); this.state = { @@ -58,17 +56,7 @@ class ReactDescribe extends React.Component { } const save = () => { const visibility = _.reduce(this._grid.state.dtypes, (ret, d) => _.assignIn(ret, { [d.name]: d.visible }), {}); - const vizzCallback = () => { - if (_.startsWith(window.location.pathname, "/dtale/popup/describe")) { - window.opener.location.reload(); - } else { - const updatedColumns = _.map(this.props.chartData.columns, c => - _.assignIn({}, c, { visible: _.get(visibility, c.name, true) }) - ); - this.props.chartData.propagateState({ columns: updatedColumns }, this.props.onClose); - } - }; - serverState.updateVisibility(this.props.dataId, visibility, vizzCallback); + serverState.updateVisibility(this.props.dataId, visibility, window.opener.location.reload); }; const propagateState = state => this.setState(state); return [ @@ -80,11 +68,7 @@ class ReactDescribe extends React.Component {
    -
    +
    , @@ -96,22 +80,13 @@ class ReactDescribe extends React.Component { ]; } } -ReactDescribe.displayName = "ReactDescribe"; -ReactDescribe.propTypes = { +Describe.displayName = "Describe"; +Describe.propTypes = { dataId: PropTypes.string.isRequired, chartData: PropTypes.shape({ visible: PropTypes.bool.isRequired, selectedCol: PropTypes.string, - columns: PropTypes.arrayOf(PropTypes.object), - propagateState: PropTypes.func, }), - onClose: PropTypes.func, }; -ReactDescribe.defaultProps = { onClose: _.noop }; - -const ReduxDescribe = connect( - state => _.pick(state, ["dataId", "chartData"]), - dispatch => ({ onClose: chartData => dispatch(closeChart(chartData || {})) }) -)(ReactDescribe); -export { ReactDescribe, ReduxDescribe as Describe }; +export { Describe }; diff --git a/static/popups/Popup.jsx b/static/popups/Popup.jsx index f6170a208..210cdc951 100644 --- a/static/popups/Popup.jsx +++ b/static/popups/Popup.jsx @@ -8,10 +8,12 @@ import ConditionalRender from "../ConditionalRender"; import { closeChart } from "../actions/charts"; import About from "./About"; import { CodeExport } from "./CodeExport"; +import { Confirmation } from "./Confirmation"; import { Correlations } from "./Correlations"; import { Describe } from "./Describe"; import { Filter } from "./Filter"; import Instances from "./Instances"; +import { Rename } from "./Rename"; import { ColumnAnalysis } from "./analysis/ColumnAnalysis"; import { Charts } from "./charts/Charts"; import { CreateColumn } from "./create/CreateColumn"; @@ -103,6 +105,25 @@ class ReactPopup extends React.Component { ); body = ; break; + case "confirm": + modalTitle = ( + + + Yes/No + ({chartData.title}) + + ); + body = ; + break; + case "rename": + modalTitle = ( + + + Rename + + ); + body = ; + break; case "instances": modalTitle = ( diff --git a/static/popups/Rename.jsx b/static/popups/Rename.jsx new file mode 100644 index 000000000..1aa490874 --- /dev/null +++ b/static/popups/Rename.jsx @@ -0,0 +1,102 @@ +import _ from "lodash"; +import PropTypes from "prop-types"; +import React from "react"; +import { connect } from "react-redux"; + +import { RemovableError } from "../RemovableError"; +import { closeChart } from "../actions/charts"; +import serverState from "../dtale/serverStateManagement"; + +require("./Confirmation.css"); + +class ReactRename extends React.Component { + constructor(props) { + super(props); + this.state = { + name: _.get(props, "chartData.selectedCol", ""), + error: null, + }; + this.updateName = this.updateName.bind(this); + } + + updateName(name) { + const { selectedCol, columns } = this.props.chartData; + if (name !== selectedCol && _.find(columns, { name })) { + this.setState({ + error: , + }); + return; + } + this.setState({ name }); + } + + render() { + const { dataId, propagateState, onClose } = this.props; + const { selectedCol, columns } = this.props.chartData; + const rename = this.state.name; + const renameAction = () => { + const callback = data => { + if (data.error) { + this.setState({ error: }); + return; + } + const updatedColumns = _.map(columns, c => _.assignIn({}, c, c.name === selectedCol ? { name: rename } : {})); + const renameUpdate = data => + _.mapValues(data, d => { + const newRecord = _.assignIn(d, { [rename]: d[selectedCol] }); + delete newRecord[selectedCol]; + return newRecord; + }); + propagateState({ columns: updatedColumns, renameUpdate }); + onClose(); + }; + serverState.renameColumn(dataId, selectedCol, rename, callback); + }; + return [ +
    + {this.state.error} +
    + +
    {selectedCol}
    +
    +
    + +
    + this.updateName(e.target.value)} + /> +
    +
    +
    , +
    + + +
    , + ]; + } +} +ReactRename.displayName = "Rename"; +ReactRename.propTypes = { + chartData: PropTypes.shape({ + visible: PropTypes.bool.isRequired, + selectedCol: PropTypes.string, + columns: PropTypes.arrayOf(PropTypes.object), + }), + propagateState: PropTypes.func, + dataId: PropTypes.string.isRequired, + onClose: PropTypes.func, +}; + +const ReduxRename = connect( + state => _.pick(state, ["chartData"]), + dispatch => ({ onClose: chartData => dispatch(closeChart(chartData || {})) }) +)(ReactRename); + +export { ReactRename, ReduxRename as Rename }; diff --git a/static/popups/describe/Details.jsx b/static/popups/describe/Details.jsx index 4c550d851..4e226bbf3 100644 --- a/static/popups/describe/Details.jsx +++ b/static/popups/describe/Details.jsx @@ -58,6 +58,7 @@ class Details extends React.Component { } newState.details = _.pick(detailData, ["describe", "uniques"]); newState.details.name = this.props.selected.name; + newState.details.dtype = this.props.selected.dtype; newState.code = detailData.code; this.setState(newState, this.createBoxplot); }); @@ -195,7 +196,7 @@ class Details extends React.Component { queryApplied: !outliers.queryApplied, }), }, - fetchJson(url, data => this.props.propagateState({ outlierFilters: data.currFilters || {} })) + fetchJson(url, () => window.opener.location.reload()) ); }; return [ @@ -244,7 +245,7 @@ class Details extends React.Component {

    {details.name}

    - ({this.props.selected.dtype}) + ({details.dtype})
    {renderCodePopupAnchor(this.state.code, "Describe")}
    , @@ -277,7 +278,6 @@ Details.displayName = "Details"; Details.propTypes = { selected: PropTypes.object, dataId: PropTypes.string, - propagateState: PropTypes.func, }; export { Details }; diff --git a/static/popups/reshape/Aggregate.jsx b/static/popups/reshape/Aggregate.jsx index c43421849..1043c0339 100644 --- a/static/popups/reshape/Aggregate.jsx +++ b/static/popups/reshape/Aggregate.jsx @@ -144,15 +144,15 @@ class Aggregate extends React.Component { input = _.concat( [
    -
    +
    - Col: + Col: {this.renderSelect(null, "index", false, "_curr_agg_col")}
    -
    +
    - Agg: + Agg: