1.8.10

* #172: allowing groups to be specified in 3D scatter * #181: percentage sum/count charts * #179: confirmation for column deletion * #175: rename columns * #173: wider column input box for GroupBy in "Summarize Data" popup * #174, moved "Describe" popup to new browser tab * #170: filter "Value" dropdown for maps to only int or float columns * #164: show information about missing data in "Describe" popup * #184: "nan" not showing up for numeric columns * #176: highlight background of outliers/missing values
man-group · Apr 25, 2020 · bbe720a · bbe720a
1 parent b7a478d
commit bbe720a
Show file tree

Hide file tree

Showing 49 changed files with 1,120 additions and 414 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -5,7 +5,7 @@ defaults: &defaults
       CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
       CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
       CODECOV_TOKEN: b0d35139-0a75-427a-907b-2c78a762f8f0
-      VERSION: 1.8.9
+      VERSION: 1.8.10
       PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
     steps:
     - checkout

diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,7 @@
 # test output files
 .cache
 .coverage
+.DS_store
 .eggs
 .nyc_output
 js_junit.xml

diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,17 @@
 ## Changelog
 
+### 1.8.10 (2020-4-26)
+ * [#184](https://github.com/man-group/dtale/issues/184): "nan" not showing up for numeric columns
+ * [#181](https://github.com/man-group/dtale/issues/181): percentage sum/count charts
+ * [#179](https://github.com/man-group/dtale/issues/179): confirmation for column deletion
+ * [#176](https://github.com/man-group/dtale/issues/176): highlight background of outliers/missing values
+ * [#175](https://github.com/man-group/dtale/issues/175): column renaming
+ * [#174](https://github.com/man-group/dtale/issues/174): moved "Describe" popup to new browser tab
+ * [#173](https://github.com/man-group/dtale/issues/173): wider column input box for GroupBy in "Summarize Data" popup
+ * [#172](https://github.com/man-group/dtale/issues/172): allowing groups to be specified in 3D scatter
+ * [#170](https://github.com/man-group/dtale/issues/170): filter "Value" dropdown for maps to only int or float columns
+ * [#164](https://github.com/man-group/dtale/issues/164): show information about missing data in "Describe" popup
+
 ### 1.8.9 (2020-4-18)
  * updated correlations & "Open Popup" to create new tabs instead
  * test fixes for dash 1.11.0

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -3,21 +3,29 @@ version: '3.4'
 services:
   dtale_2_7:
     image: dtale_2_7:latest
+    env_file:
+      - ./docker/dtale.env
     build:
       context: .
       dockerfile: docker/2_7/Dockerfile
   dtale_3_6:
     image: dtale_3_6:latest
+    env_file:
+      - ./docker/dtale.env
     build:
       context: .
       dockerfile: docker/3_6/Dockerfile
   dtale_3_7:
     image: dtale_3_7:latest
+    env_file:
+      - ./docker/dtale.env
     build:
       context: .
       dockerfile: docker/3_7/Dockerfile
   dtale_3_8:
     image: dtale_3_8:latest
+    env_file:
+      - ./docker/dtale.env
     build:
       context: .
       dockerfile: docker/3_8/Dockerfile
diff --git a/.env → docker/dtale.env b/.env → docker/dtale.env
@@ -1,2 +1,2 @@
-VERSION=1.8.9
+VERSION=1.8.10
 TZ=America/New_York
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -64,9 +64,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'1.8.9'
+version = u'1.8.10'
 # The full version, including alpha/beta/rc tags.
-release = u'1.8.9'
+release = u'1.8.10'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

diff --git a/dtale/charts/utils.py b/dtale/charts/utils.py
@@ -239,7 +239,7 @@ def check_exceptions(df, allow_duplicates, unlimited_data=False, data_limit=1500
         raise ChartBuildingError(limit_msg.format(data_limit))
 
 
-def build_agg_data(df, x, y, inputs, agg, z=None, animate_by=None):
+def build_agg_data(df, x, y, inputs, agg, z=None, group_col=None, animate_by=None):
     """
     Builds aggregated data when an aggregation (sum, mean, max, min...) is selected from the front-end.
 
@@ -282,29 +282,46 @@ def build_agg_data(df, x, y, inputs, agg, z=None, animate_by=None):
         ]
         return agg_df, code
 
+    idx_cols = make_list(animate_by) + make_list(group_col) + [x]
+    agg_cols = y
     if z_exists:
-        idx_cols = make_list(animate_by) + [x] + make_list(y)
-        groups = df.groupby(idx_cols)
-        groups = getattr(groups[make_list(z)], agg)()
-        if animate_by is not None:
-            full_idx = pd.MultiIndex.from_product([df[c].unique() for c in idx_cols], names=idx_cols)
-            groups = groups.reindex(full_idx).fillna(0)
-        return groups.reset_index(), [
-            "chart_data = chart_data.groupby(['{cols}'])[['{z}']].{agg}().reset_index()".format(
-                cols="', '".join([x] + make_list(y)), z=z, agg=agg
+        idx_cols += make_list(y)
+        agg_cols = make_list(z)
+
+    groups = df.groupby(idx_cols)
+    if agg in ['pctsum', 'pctct']:
+        func = 'sum' if agg == 'pctsum' else 'size'
+        subidx_cols = [c for c in idx_cols if c not in make_list(group_col)]
+        groups = getattr(groups[agg_cols], func)()
+        groups = groups / getattr(df.groupby(subidx_cols)[agg_cols], func)() * 100
+        if len(agg_cols) > 1:
+            groups.columns = agg_cols
+        elif len(agg_cols) == 1:
+            groups.name = agg_cols[0]
+        code = (
+            "chart_data = chart_data.groupby(['{cols}'])[['{agg_cols}']].{agg}()\n"
+            "chart_data = chart_data / chart_data.groupby(['{subidx_cols}']).{agg}()\n"
+            "chart_data = chart_data.reset_index()"
+        )
+        code = code.format(cols="', '".join(idx_cols), subidx_cols="', '".join(subidx_cols),
+                           agg_cols="', '".join(agg_cols), agg=func)
+        code = [code]
+    else:
+        groups = getattr(groups[agg_cols], agg)()
+        code = [
+            "chart_data = chart_data.groupby(['{cols}'])[['{agg_cols}']].{agg}().reset_index()".format(
+                cols="', '".join(idx_cols), agg_cols="', '".join(agg_cols), agg=agg
             )
         ]
-    idx_cols = make_list(animate_by) + [x]
-    groups = df.groupby(idx_cols)
-    groups = getattr(groups[y], agg)()
     if animate_by is not None:
         full_idx = pd.MultiIndex.from_product([df[c].unique() for c in idx_cols], names=idx_cols)
         groups = groups.reindex(full_idx).fillna(0)
-    return groups.reset_index(), [
-        "chart_data = chart_data.groupby('{x}')[['{y}']].{agg}().reset_index()".format(
-            x=x, y=make_list(y)[0], agg=agg
-        )
-    ]
+        code += [
+            "idx_cols = ['{cols}']".format(cols="', '".join(idx_cols)),
+            'full_idx = pd.MultiIndex.from_product([df[c].unique() for c in idx_cols], names=idx_cols)'
+            'chart_data = chart_data.reindex(full_idx).fillna(0)'
+        ]
+    return groups.reset_index(), code
 
 
 def build_base_chart(raw_data, x, y, group_col=None, group_val=None, agg=None, allow_duplicates=False, return_raw=False,
@@ -337,22 +354,21 @@ def build_base_chart(raw_data, x, y, group_col=None, group_val=None, agg=None, a
     y_cols = make_list(y)
     z_col = kwargs.get('z')
     z_cols = make_list(z_col)
+    sort_cols = (y_cols if len(z_cols) else [])
     if group_col is not None and len(group_col):
         main_group = group_col
         if animate_by is not None:
             main_group = [animate_by] + main_group
-        sort_cols = main_group + [x]
+        sort_cols = main_group + [x] + sort_cols
         data = data.sort_values(sort_cols)
         code.append("chart_data = chart_data.sort_values(['{cols}'])".format(cols="', '".join(sort_cols)))
-        check_all_nan(data, [x] + y_cols)
+        check_all_nan(data)
         data = data.rename(columns={x: x_col})
         code.append("chart_data = chart_data.rename(columns={'" + x + "': '" + x_col + "'})")
-        if agg is not None and agg != 'raw':
-            data = data.groupby(main_group + [x_col])
-            data = getattr(data, agg)().reset_index()
-            code.append("chart_data = chart_data.groupby(['{cols}']).{agg}().reset_index()".format(
-                cols="', '".join(main_group + [x]), agg=agg
-            ))
+        if agg is not None:
+            data, agg_code = build_agg_data(data, x_col, y_cols, kwargs, agg, z=z_col, group_col=group_col,
+                                            animate_by=animate_by)
+            code += agg_code
         MAX_GROUPS = 30
         group_vals = data[group_col].drop_duplicates()
         if len(group_vals) > MAX_GROUPS:
@@ -376,8 +392,8 @@ def build_base_chart(raw_data, x, y, group_col=None, group_val=None, agg=None, a
         data_f, range_f = build_formatters(data)
         ret_data = dict(
             data={},
-            min={col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols},
-            max={col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols},
+            min={col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols},
+            max={col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols},
         )
 
         dtypes = get_dtypes(data)
@@ -408,15 +424,17 @@ def _group_filter():
     main_group = [x]
     if animate_by is not None:
         main_group = [animate_by] + main_group
-    sort_cols = main_group + (y_cols if len(z_cols) else [])
+    sort_cols = main_group + sort_cols
     data = data.sort_values(sort_cols)
     code.append("chart_data = chart_data.sort_values(['{cols}'])".format(cols="', '".join(sort_cols)))
-    check_all_nan(data, main_group + y_cols + z_cols)
+    check_all_nan(data)
     y_cols = [str(y_col) for y_col in y_cols]
     data = data[main_group + y_cols + z_cols]
-    main_group[-1] = x_col
-    data.columns = main_group + y_cols + z_cols
-    code.append("chart_data.columns = ['{cols}']".format(cols="', '".join(main_group + y_cols + z_cols)))
+
+    data = data.rename(columns={x: x_col})
+    main_group = [x_col if c == x else c for c in main_group]
+    code.append("chart_data = chart_data.rename(columns={'" + x + "': '" + x_col + "'})")
+
     if agg is not None:
         data, agg_code = build_agg_data(data, x_col, y_cols, kwargs, agg, z=z_col, animate_by=animate_by)
         code += agg_code
@@ -427,7 +445,7 @@ def _group_filter():
 
     dupe_cols = main_group + (y_cols if len(z_cols) else [])
     check_exceptions(
-        data[dupe_cols].rename(columns={'x': x}),
+        data[dupe_cols].rename(columns={x_col: x}),
         allow_duplicates or agg == 'raw',
         unlimited_data=unlimited_data,
         data_limit=40000 if len(z_cols) or animate_by is not None else 15000

diff --git a/dtale/dash_application/charts.py b/dtale/dash_application/charts.py
@@ -385,7 +385,7 @@ def build_layout(cfg):
     :return: layout object
     :rtype: :plotly:`plotly.graph_objects.Layout <plotly.graph_objects.Layout>`
     """
-    return go.Layout(**dict_merge(dict(legend=dict(orientation='h', y=1.2)), cfg))
+    return go.Layout(**dict_merge(dict(legend=dict(orientation='h')), cfg))
 
 
 def cpg_chunker(charts, columns=2):
@@ -479,7 +479,6 @@ def build_frame(frame):
             id='scatter-{}-{}'.format(group or 'all', y_val),
             figure=figure_cfg
         ), group_filter=dict_merge(dict(y=y_val), {} if group is None else dict(group=group)))
-
     return [_build_final_scatter(y2) for y2 in y]
 
 
@@ -1147,7 +1146,6 @@ def build_figure_data(data_id, chart_type=None, query=None, x=None, y=None, z=No
         chart_kwargs['animate_by'] = animate_by
     if chart_type in ZAXIS_CHARTS:
         chart_kwargs['z'] = z
-        del chart_kwargs['group_col']
     data, chart_code = build_base_chart(data, x, y, unlimited_data=True, **chart_kwargs)
     return data, code + chart_code
 
@@ -1274,21 +1272,20 @@ def build_chart(data_id=None, **inputs):
             return pie_builder(data, x, y, chart_builder, **chart_inputs), range_data, code
 
         axes_builder = build_axes(data_id, x, axis_inputs, data['min'], data['max'], z=z, agg=agg)
-        if chart_type == 'scatter':
+        if chart_type in ['scatter', '3d_scatter']:
+            kwargs = dict(agg=agg)
+            if chart_type == '3d_scatter':
+                kwargs['z'] = z
+                kwargs['animate_by'] = animate_by
             if inputs['cpg']:
                 scatter_charts = flatten_lists([
-                    scatter_builder(data, x, y, axes_builder, chart_builder, group=subgroup, agg=agg)
+                    scatter_builder(data, x, y, axes_builder, chart_builder, group=subgroup, **kwargs)
                     for subgroup in data['data']
                 ])
             else:
-                scatter_charts = scatter_builder(data, x, y, axes_builder, chart_builder, agg=agg)
+                scatter_charts = scatter_builder(data, x, y, axes_builder, chart_builder, **kwargs)
             return cpg_chunker(scatter_charts), range_data, code
 
-        if chart_type == '3d_scatter':
-            chart = scatter_builder(data, x, y, axes_builder, chart_builder, z=z, agg=agg,
-                                    animate_by=animate_by)
-            return chart, range_data, code
-
         if chart_type == 'surface':
             return surface_builder(data, x, y, z, axes_builder, chart_builder, agg=agg), range_data, code
 

diff --git a/dtale/dash_application/layout.py b/dtale/dash_application/layout.py
@@ -172,9 +172,10 @@ def build_option(value, label=None):
                  map_group=dict(display=True)),
 }
 AGGS = dict(
-    count='Count', nunique='Unique Count', sum='Sum', mean='Mean', rolling='Rolling', corr='Correlation', first='First',
-    last='Last', median='Median', min='Minimum', max='Maximum', std='Standard Deviation', var='Variance',
-    mad='Mean Absolute Deviation', prod='Product of All Items', raw='No Aggregation'
+    raw='No Aggregation', count='Count', nunique='Unique Count', sum='Sum', mean='Mean', rolling='Rolling',
+    corr='Correlation', first='First', last='Last', median='Median', min='Minimum', max='Maximum',
+    std='Standard Deviation', var='Variance', mad='Mean Absolute Deviation', prod='Product of All Items',
+    pctct='Percentage Count', pctsum='Percentage Sum'
 )
 FREQS = ['H', 'H2', 'WD', 'D', 'W', 'M', 'Q', 'Y']
 FREQ_LABELS = dict(H='Hourly', H2='Hour', WD='Weekday', W='Weekly', M='Monthly', Q='Quarterly', Y='Yearly')
@@ -268,8 +269,7 @@ def build_loc_mode_hover(loc_mode):
                 id='loc-mode-hover'
             )
         ],
-        className='input-group-addon pt-1 pb-0',
-        style=dict(height='35.5px')
+        className='input-group-addon pt-1 pb-0'
     )
 
 
@@ -313,7 +313,7 @@ def build_error(error, tb):
 
     :param error: execption message
     :type error: str
-    :param tb: traceback
+    :param tb: tracebackF
     :type tb: str
     :return: error component
     :rtype: :dash:`dash_html_components.Div <dash-html-components/div>`
@@ -380,23 +380,26 @@ def build_input_options(df, **inputs):
 def build_map_options(df, type='choropleth', loc=None, lat=None, lon=None, map_val=None):
     dtypes = get_dtypes(df)
     cols = sorted(dtypes.keys())
-    float_cols, str_cols = [], []
+    float_cols, str_cols, num_cols = [], [], []
     for c in cols:
         dtype = dtypes[c]
-        if classify_type(dtype) == 'F':
-            float_cols.append(c)
-            continue
-        if classify_type(dtype) == 'S':
+        classification = classify_type(dtype)
+        if classification == 'S':
             str_cols.append(c)
+            continue
+        if classification in ['F', 'I']:
+            num_cols.append(c)
+            if classification == 'F':
+                float_cols.append(c)
 
     lat_options = [build_option(c) for c in float_cols if c not in build_selections(lon, map_val)]
     lon_options = [build_option(c) for c in float_cols if c not in build_selections(lat, map_val)]
     loc_options = [build_option(c) for c in str_cols if c not in build_selections(map_val)]
 
     if type == 'choropleth':
-        val_options = [build_option(c) for c in cols if c not in build_selections(loc)]
+        val_options = [build_option(c) for c in num_cols if c not in build_selections(loc)]
     else:
-        val_options = [build_option(c) for c in cols if c not in build_selections(lon, lat)]
+        val_options = [build_option(c) for c in num_cols if c not in build_selections(lon, lat)]
     return loc_options, lat_options, lon_options, val_options
 
 
@@ -636,7 +639,7 @@ def show_map_style(show):
                         )
                     ],
                     id='non-map-inputs', style={} if not show_map else {'display': 'none'},
-                    className='row pt-3 pb-3 charts-filters'
+                    className='row p-0 charts-filters'
                 ),
                 html.Div(
                     [
@@ -748,9 +751,9 @@ def show_map_style(show):
                 html.Div([
                     build_input('Aggregation', dcc.Dropdown(
                         id='agg-dropdown',
-                        options=[build_option(v, AGGS[v]) for v in ['count', 'nunique', 'sum', 'mean', 'rolling',
+                        options=[build_option(v, AGGS[v]) for v in ['raw', 'count', 'nunique', 'sum', 'mean', 'rolling',
                                                                     'corr', 'first', 'last', 'median', 'min', 'max',
-                                                                    'std', 'var', 'mad', 'prod', 'raw']],
+                                                                    'std', 'var', 'mad', 'prod', 'pctsum', 'pctct']],
                         placeholder='Select an aggregation',
                         style=dict(width='inherit'),
                         value=agg or 'raw',
@@ -849,7 +852,7 @@ def show_map_style(show):
                             value=inputs.get('animate_by')
                         ), className='col-auto addon-min-width', style=animate_style, id='animate-by-input'),
                     ],
-                    className='row pt-3 pb-5 charts-filters'
+                    className='row pt-3 pb-5 charts-filters', id='chart-inputs'
                 )],
                 id='main-inputs', className=main_input_class
             ), build_input('Group(s)', dcc.Dropdown(

diff --git a/dtale/global_state.py b/dtale/global_state.py
@@ -1,6 +1,5 @@
-from collections import MutableMapping
-
 from six import PY3
+from six.moves.collections_abc import MutableMapping
 
 DATA = {}
 DTYPES = {}

diff --git a/dtale/static/css/dash.css b/dtale/static/css/dash.css
@@ -530,3 +530,9 @@ div.modebar > div.modebar-group:first-child /* hide plotly "export to png" */
     padding-left: 0;
 }
 
+div#non-map-inputs > div,
+div#chart-inputs > div {
+  padding-top: 0.25rem !important;
+  padding-bottom: 0.25rem !important;
+}
+