From e5f07dbfb39b6380a0b8c3bcc1c6d40d6144ce60 Mon Sep 17 00:00:00 2001 From: Andrew Schonfeld Date: Wed, 5 Aug 2020 22:02:40 -0400 Subject: [PATCH] 1.12.1 * better axis display on heatmaps * handling for column filter data on "mixed" type columns * "title" parameter added for offline charts * heatmap drilldowns on animations * bugfix for refreshing custom geojson charts --- .circleci/config.yml | 2 +- CHANGES.md | 7 + README.md | 5 +- docker/dtale.env | 2 +- docs/KDNuggets/DTale.html | 167 ++++++++++++++++++++++ docs/source/conf.py | 4 +- dtale/app.py | 4 + dtale/dash_application/charts.py | 126 +++++++++++----- dtale/dash_application/custom_geojson.py | 4 +- dtale/dash_application/drilldown_modal.py | 15 ++ dtale/views.py | 32 +++-- package.json | 2 +- setup.py | 2 +- tests/dtale/dash/test_drilldown.py | 43 ++++++ tests/dtale/test_views.py | 40 +++++- 15 files changed, 399 insertions(+), 56 deletions(-) create mode 100644 docs/KDNuggets/DTale.html diff --git a/.circleci/config.yml b/.circleci/config.yml index d973df22..7d020293 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,7 +63,7 @@ python_variables: &python_variables CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_TEST_REPORTS: /tmp/circleci-test-results CODECOV_TOKEN: b0d35139-0a75-427a-907b-2c78a762f8f0 - VERSION: 1.12.0 + VERSION: 1.12.1 PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases RUN_BLACK: true python: &python diff --git a/CHANGES.md b/CHANGES.md index 397caf11..7725d98a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,12 @@ ## Changelog +### 1.12.1 (2020-8-5) + * better axis display on heatmaps + * handling for column filter data on "mixed" type columns + * "title" parameter added for offline charts + * heatmap drilldowns on animations + * bugfix for refreshing custom geojson charts + ### 1.12.0 (2020-8-1) * added better notification for when users view Category breakdowns in "Column Analysis" & "Describe" * fixed code snippets in "Numeric" column builder when no operation is selected diff --git a/README.md b/README.md index fe75be12..da179667 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,7 @@ package index](https://pypi.org/project/dtale) and on conda using [conda-forge]( ```sh # conda -conda config --add channels conda-forge -conda install dtale +conda install dtale -c conda-forge ``` ```sh @@ -623,7 +622,7 @@ You can now export your dash charts (with the exception of Wordclouds) to static I've been asked about being able to export the data that is contained within your chart to a CSV for further analysis in tools like Excel. This button makes that possible. -**OFFLINE CHARTS** +### OFFLINE CHARTS Want to run D-Tale in a jupyter notebook and build a chart that will still be displayed even after your D-Tale process has shutdown? Now you can! Here's an example code snippet show how to use it: diff --git a/docker/dtale.env b/docker/dtale.env index 41ad878e..a9a5504f 100644 --- a/docker/dtale.env +++ b/docker/dtale.env @@ -1,2 +1,2 @@ -VERSION=1.12.0 +VERSION=1.12.1 TZ=America/New_York \ No newline at end of file diff --git a/docs/KDNuggets/DTale.html b/docs/KDNuggets/DTale.html new file mode 100644 index 00000000..c7429776 --- /dev/null +++ b/docs/KDNuggets/DTale.html @@ -0,0 +1,167 @@ + + + + + + + + + + + + +

Bring your Pandas Dataframes to life with D-Tale

+ By Andrew Schonfeld +

+ Tired of running 

df.head()
on your dataframes? In this tutorial, we will explore the open-source visualizer for Pandas dataframes, D-Tale. Some of the features we'll touch on are installation, startup, navigating the grid, viewing column statistics, building a chart & code exports. +

+

What is it?

+

D-Tale is the combination of a Flask back-end and a React front-end to bring you an easy way to view & analyze Pandas data structures. It integrates seamlessly with ipython notebooks & python/ipython terminals. Currently this tool supports such Pandas objects as DataFrame, Series, MultiIndex, DatetimeIndex & RangeIndex.

+

Step 1: Installation

+ Installation is available using pip or conda + 
+            # conda
+            conda install dtale -c conda-forge
+
+            # pip
+            pip install -U dtale
+
+ Source code is available here. +

Step 2: Opening the Grid

+ Execute the following code within your Python console or jupyter notebook + 
+            import pandas as pd
+            import dtale
+
+            df = pd.DataFrame(dict(a=[1,1,2,2,3,3], b=[1,2,3,4,5,6]))
+            dtale.show(df)
+
+ You will be presented with one of the following: + + Examples + + + + + + + + + + + + + +
PyCharmjupyter
+

Step 3: Navigating the Grid

+

Once inside the grid you have all of the standard grid functionality at your fingertips by clicking column headers. If your still in the output cell of your jupyter notebook feel free to click the triangle in the upper lefthand corner to open the main menu and then click "Open in New Tab" to give you a larger workspace.

+ + + + + +
+
    +
  • Sorting
    • +
  • Renaming
    • +
  • Filtering
    • +
  • Lock Columns to the Left side (this is handy if you have a very wide dataframe)
    • +
+ 		+ +
+

Step 4: Building Columns

+

If you open the main menu by clicking on the triangle in the upper lefthand corner you'll be presented with many options, one of which is "Build Columns". Click that and you see many options for different ways to build new columns based on your existing data. Here are some examples of a few of them:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BuilderMain MenuColumn Builder MenuOutput
Transform (Groupby Mean) + + + + + +
Mean Adjust (Subtract Columns) + + + +
Winsorize + + + +
+

Step 5: View Column Statistics

+

Many times you'll want to be able to view a quick overview of the contents of your dataframe. One way to do this is by running 

df.describe()
. We've brought that function to life with the "Describe" menu option. By either opening the main menu or clicking a column header and then clicking the "Describe" button (clicking from a column header will preselect that column for you).

+ +

If you take a look you'll notice a listing of different statistics (which will vary based on data type of the column selected). These statistics are the output of calling 

df.describe()
on that column as well as some other helpful statistics like percentage of missings & kurtosis. You also have the ability to view other helpful information:

+ +

Step 6: Building a Chart With Your Data

+

By opening the main menu once more and clicking the "Charts" button you will be brought to a new tab with the ability to build the following charts using Plotly Dash:

+ + Here's an example of building a bar chart comparing the raw values (a) to its grouped mean (b_mean). + + Now you'll also notice some links at the top of your chart: + +

Step 7: Code Export

+

Let's take a look at the output of clicking the "Code Export" link of you chart that we built in Step 6.

+ +

Now the goal of code export is to help users learn a little bit about what code was run to get them what their looking at, but it is by no means gospel. So feel free to submit suggestions or bugs on the Issues page page of the repo.

+
+ Here are some other competitors to D-Tale: + +

Thank you for reading this tutorial and I hope it helps you with your data exploration. There's many other features that I haven't touched on here so I urge you to check it out the README, particularly the different UI functions. If you liked this please support open-source and star the repo. :)

+ + \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index c79ee79a..416cac2d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -64,9 +64,9 @@ # built documents. # # The short X.Y version. -version = u'1.12.0' +version = u'1.12.1' # The full version, including alpha/beta/rc tags. -release = u'1.12.0' +release = u'1.12.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/dtale/app.py b/dtale/app.py index 1907939a..493a2715 100644 --- a/dtale/app.py +++ b/dtale/app.py @@ -740,6 +740,7 @@ def offline_chart( barsort=None, yaxis=None, filepath=None, + title=None, **kwargs ): """ @@ -773,6 +774,8 @@ def offline_chart( :type barsort: str, optional :param filepath: location to save HTML output :type filepath: str, optional + :param title: Title of your chart + :type title: str, optional :param kwargs: optional keyword arguments, here in case invalid arguments are passed to this function :type kwargs: dict :return: possible outcomes are: @@ -796,6 +799,7 @@ def offline_chart( barsort=barsort, yaxis=yaxis, filepath=filepath, + title=title, **kwargs ) global_state.cleanup() diff --git a/dtale/dash_application/charts.py b/dtale/dash_application/charts.py index 7bb7818a..c055babb 100644 --- a/dtale/dash_application/charts.py +++ b/dtale/dash_application/charts.py @@ -603,7 +603,7 @@ def build_frame(frame, frame_name): mode="markers", opacity=0.7, name=series_key, - marker=build_scatter_marker(series, z), + marker=build_scatter_marker(series, z, colorscale), customdata=[frame_name] * len(series["x"]), ) ) @@ -818,7 +818,11 @@ def build_grouped_bars_with_multi_yaxis(series_cfgs, y): def update_cfg_w_frames(cfg, frame_col, frames, slider_steps): - cfg["data"][0]["customdata"] = frames[-1]["data"][0]["customdata"] + data = frames[-1]["data"][0] + if "customdata" in data: + cfg["data"][0]["customdata"] = data["customdata"] + elif "text" in data: + cfg["data"][0]["text"] = data["text"] cfg["frames"] = frames cfg["layout"]["updatemenus"] = [ { @@ -1423,7 +1427,7 @@ def heatmap_builder(data_id, export=False, **inputs): hm_kwargs = dict( colorscale=build_colorscale(inputs.get("colorscale") or "Greens"), showscale=True, - hoverinfo="x+y+z", + hoverinfo="text", ) animate_by, x, y, z, agg = ( inputs.get(p) for p in ["animate_by", "x", "y", "z", "agg"] @@ -1505,57 +1509,110 @@ def heatmap_builder(data_id, export=False, **inputs): heat_data = heat_data.set_index([x, y]) heat_data = heat_data.unstack(0)[z] heat_data = heat_data.values + + def _build_text(z_vals, animate_str=""): + return [ + [ + "{}{}: {}
{}: {}
{}: {}".format( + animate_str, + x, + str(x_data[x_idx]), + y, + str(y_data[y_idx]), + z, + str(z2), + ) + for x_idx, z2 in enumerate(z1) + ] + for y_idx, z1 in enumerate(z_vals) + ] + + text = _build_text(heat_data) + formatter = "{}: {}
{}: {}
{}: {}" code.append( ( "chart_data = data.sort_values(['{x}', '{y}'])\n" "chart_data = chart_data.set_index(['{x}', '{y}'])\n" "chart_data = unstack(0)['{z}']" - ).format(x=x, y=y, z=z) + "text = [\n" + "\t[\n" + "\t\t'{formatter}'.format(\n" + "\t\t\tx, str(chart_data.columns[x_idx]), y, str(chart_data.index.values[y_idx]), z, str(z2)\n" + "\t\t)\n" + "\t\tfor x_idx, z2 in enumerate(z1)\n" + "\t]\n" + "\tfor y_idx, z1 in enumerate(chart_data.values)\n" + "]" + ).format(x=x, y=y, z=z, formatter=formatter) ) - x_axis = dict_merge( - {"title": x_title, "tickangle": -20}, build_spaced_ticks(x_data) - ) - if dtypes.get(x) == "I": - x_axis["tickformat"] = ".0f" + def _build_heatmap_axis(col, data, title): + axis_cfg = { + "title": title, + "tickangle": -20, + "showticklabels": True, + "visible": True, + "domain": [0, 1], + } + if dtypes.get(col) in ["I", "F"]: + rng = [data[0], data[-1]] + axis_cfg = dict_merge( + axis_cfg, + { + "autorange": True, + "rangemode": "normal", + "tickmode": "auto", + "range": rng, + "type": "linear", + }, + ) + if dtypes.get(col) == "I": + axis_cfg["tickformat"] = ".0f" if dtypes.get(col) == "I" else ".3f" + return axis_cfg + return dict_merge(axis_cfg, {"type": "category", "tickmode": "auto"}) - y_axis = dict_merge( - {"title": y_title, "tickangle": -20}, build_spaced_ticks(y_data) - ) - if dtypes.get(y) == "I": - y_axis["tickformat"] = ".0f" + x_axis = _build_heatmap_axis(x, x_data, x_title) + y_axis = _build_heatmap_axis(y, y_data, y_title) - hm_kwargs = dict_merge( - hm_kwargs, dict(colorbar={"title": z_title}, hoverinfo="x+y+z",), + hm_kwargs = dict_merge(hm_kwargs, dict(colorbar={"title": z_title}, text=text),) + + hm_kwargs = dict_merge(hm_kwargs, {"z": heat_data}) + layout_cfg = build_layout( + dict_merge( + dict(xaxis_zeroline=False, yaxis_zeroline=False), + build_title(x, y, z=z, agg=agg), + ) ) + + heatmap_func = go.Heatmapgl + heatmap_func_str = "go.Heatmapgl(z=chart_data.values, text=text, **hm_kwargs)" + if len(x_data) * len(y_data) < 10000: + heatmap_func = go.Heatmap + heatmap_func_str = ( + "go.Heatmap(\n" + "\tx=chart_data.columns, y=chart_data.index.values, z=chart_data.values, text=text, **hm_kwargs\n" + ")" + ) + layout_cfg["xaxis"] = x_axis + layout_cfg["yaxis"] = y_axis + hm_kwargs["x"] = x_data + hm_kwargs["y"] = y_data + pp = pprint.PrettyPrinter(indent=4) code.append( ( "\nimport plotly.graph_objects as go\n\n" "hm_kwargs = {hm_kwargs_str}\n" - "chart = go.Heatmapgl(x=chart_data.columns, y=chart_data.index.values, z=chart_data.values, **hm_kwargs" - ")" - ).format(hm_kwargs_str=pp.pformat(hm_kwargs)) + "chart = {chart}" + ).format(chart=heatmap_func_str, hm_kwargs_str=pp.pformat(hm_kwargs)) ) - hm_kwargs = dict_merge(hm_kwargs, dict(x=x_data, y=y_data, z=heat_data)) - layout_cfg = build_layout( - dict_merge( - dict( - xaxis=x_axis, - yaxis=y_axis, - xaxis_zeroline=False, - yaxis_zeroline=False, - ), - build_title(x, y, z=z, agg=agg), - ) - ) code.append( "figure = go.Figure(data=[chart], layout=go.{layout})".format( layout=pp.pformat(layout_cfg) ) ) - figure_cfg = {"data": [go.Heatmapgl(**hm_kwargs)], "layout": layout_cfg} + figure_cfg = {"data": [heatmap_func(**hm_kwargs)], "layout": layout_cfg} if animate_by is not None: @@ -1564,7 +1621,8 @@ def build_frame(df, name): df = df.set_index([x, y]) df = df.unstack(0)[z] return dict( - x=x_data, y=y_data, z=df.values, customdata=[name] * len(df), + z=df.values, + text=_build_text(df.values, "{}: {}
".format(animate_by, name)), ) update_cfg_w_frames( @@ -2332,6 +2390,8 @@ def clean_output(output): output = output[0] if isinstance(output, dcc.Graph): output = output.figure + if inputs.get("title"): + output["layout"]["title"] = dict(text=inputs.get("title")) return output def _raw_chart_builder(): diff --git a/dtale/dash_application/custom_geojson.py b/dtale/dash_application/custom_geojson.py index 7173387b..b4f0295a 100644 --- a/dtale/dash_application/custom_geojson.py +++ b/dtale/dash_application/custom_geojson.py @@ -51,7 +51,7 @@ def load_geojson(contents, filename): if data["type"] == "FeatureCollection": data["properties"] = sorted(geojson["features"][0]["properties"].keys()) - geojson_key = add_custom_geojson(geojson_key, data,) + geojson_key = add_custom_geojson(geojson_key, data) return geojson_key @@ -62,7 +62,7 @@ def build_geojson_upload(loc_mode, geojson_key=None, featureidkey=None): featureidkey_value = featureidkey featureidkey_placeholder = "Select uploaded data" disabled = False - if curr_geojson: + if curr_geojson and not isinstance(curr_geojson, list): if curr_geojson.get("type") == "FeatureCollection": featureidkey_options = [ build_option(fik) for fik in curr_geojson["properties"] diff --git a/dtale/dash_application/drilldown_modal.py b/dtale/dash_application/drilldown_modal.py index 46525e14..1d400f72 100644 --- a/dtale/dash_application/drilldown_modal.py +++ b/dtale/dash_application/drilldown_modal.py @@ -92,6 +92,13 @@ def _build_val(col, val): return json_date(convert_date_val_to_date(val)) return val + if "text" in click_point: # Heatmaps + strs = [] + for dim in click_point["text"].split("
"): + prop, val = dim.split(": ") + strs.append("{} ({})".format(prop, val)) + return "Drilldown for: {}".format(", ".join(strs)) + strs = [] frame_col = all_inputs.get("animate_by") if frame_col: @@ -223,6 +230,14 @@ def load_drilldown_content( x, y, z, frame = ( click_point.get(p) for p in ["x", "y", "z", "customdata"] ) + if chart_type == "heatmap": + click_point_vals = {} + for dim in click_point["text"].split("
"): + prop, val = dim.split(": ") + click_point_vals[prop] = val + x, y, frame = ( + click_point_vals.get(p) for p in [x_col, y_col, frame_col] + ) point_filter = {x_col: x, y_col: y} if frame_col: point_filter[frame_col] = frame diff --git a/dtale/views.py b/dtale/views.py index bbfde882..56d341cf 100644 --- a/dtale/views.py +++ b/dtale/views.py @@ -410,6 +410,7 @@ def offline_chart( barsort=None, yaxis=None, filepath=None, + title=None, **kwargs ): """ @@ -444,6 +445,8 @@ def offline_chart( :type yaxis: dict, optional :param filepath: location to save HTML output :type filepath: str, optional + :param title: Title of your chart + :type title: str, optional :param kwargs: optional keyword arguments, here in case invalid arguments are passed to this function :type kwargs: dict :return: possible outcomes are: @@ -465,6 +468,7 @@ def offline_chart( barmode=barmode, barsort=barsort, yaxis=yaxis, + title=title, ) params = dict_merge(params, kwargs) @@ -1188,7 +1192,7 @@ def build_column(data_id): curr_dtypes.append(dtype_f(len(curr_dtypes), name)) global_state.set_dtypes(data_id, curr_dtypes) curr_history = global_state.get_history(data_id) or [] - curr_history += [builder.build_code()] + curr_history += make_list(builder.build_code()) global_state.set_history(data_id, curr_history) return jsonify(success=True) @@ -1552,7 +1556,7 @@ def delete_col(data_id, column): data = global_state.get_data(data_id) data = data[[c for c in data.columns if c != column]] curr_history = global_state.get_history(data_id) or [] - curr_history += ['df = df[[c for c in df.columns if c != "{}"]]'.format(column)] + curr_history += ["df = df[[c for c in df.columns if c != '{}']]".format(column)] global_state.set_history(data_id, curr_history) dtypes = global_state.get_dtypes(data_id) dtypes = [dt for dt in dtypes if dt["name"] != column] @@ -1667,6 +1671,21 @@ def edit_cell(data_id, column): return jsonify(success=True) +def build_filter_vals(series, data_id, column, fmt): + dtype_info = get_dtype_info(data_id, column) + vals = list(series.dropna().unique()) + try: + vals = sorted(vals) + except BaseException: + pass # if there are mixed values (EX: strings with ints) this fails + if dtype_info["unique_ct"] > 500: + # columns with too many unique values will need to use asynchronous loading, so for now we'll give the + # first 5 values + vals = vals[:5] + vals = [fmt(v) for v in vals] + return vals + + @dtale.route("/column-filter-data//") @exception_decorator def get_column_filter_data(data_id, column): @@ -1680,14 +1699,7 @@ def get_column_filter_data(data_id, column): data_range = {k: fmt(v) for k, v in data_range.items()} ret = dict_merge(ret, data_range) if classification in ["S", "I", "B"]: - dtype_info = get_dtype_info(data_id, column) - vals = sorted(s.dropna().unique()) - if dtype_info["unique_ct"] > 500: - # columns with too many unique values will need to use asynchronous loading, so for now we'll give the - # first 5 values - vals = vals[:5] - vals = [fmt(v) for v in vals] - ret["uniques"] = vals + ret["uniques"] = build_filter_vals(s, data_id, column, fmt) return jsonify(ret) diff --git a/package.json b/package.json index aee9c05d..55d5943f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dtale", - "version": "1.12.0", + "version": "1.12.1", "description": "Visualizer for Pandas Data Structures", "main": "main.js", "directories": { diff --git a/setup.py b/setup.py index dd8e8584..2502868a 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ def run_tests(self): setup( name="dtale", - version="1.12.0", + version="1.12.1", author="MAN Alpha Technology", author_email="ManAlphaTech@man.com", description="Web Client for Visualizing Pandas Objects", diff --git a/tests/dtale/dash/test_drilldown.py b/tests/dtale/dash/test_drilldown.py index 1efe9b94..41082fea 100644 --- a/tests/dtale/dash/test_drilldown.py +++ b/tests/dtale/dash/test_drilldown.py @@ -241,6 +241,24 @@ def test_update_click_data(): header == "Drilldown for: date (customdata), a (x), b (y), Mean c (z)" ) + # Heatmap Animation + params["inputs"][0]["value"] = { + "points": [ + { + "x": "x", + "y": "y", + "z": "z", + "text": "date: date
x: x
y: y
z: z", + } + ] + } + params["state"][1]["value"]["chart_type"] = "heatmap" + response = c.post("/charts/_dash-update-component", json=params) + header = response.get_json()["response"]["drilldown-modal-header-1"][ + "children" + ] + assert header == "Drilldown for: date (date), x (x), y (y), z (z)" + # Choropleth params["inputs"][0]["value"] = { "points": [{"location": "x", "z": "z", "customdata": "customdata"}] @@ -385,6 +403,7 @@ def _chart_title(resp, histogram=False): params["inputs"][-2]["value"] = "bar" response = c.post("/charts/_dash-update-component", json=params) assert _chart_title(response) == "Col0 by security_id (No Aggregation)" + params["inputs"][-2]["value"] = "histogram" params["state"][1]["value"]["chart_type"] = "3d_scatter" params["state"][1]["value"]["y"] = "Col4" @@ -396,6 +415,30 @@ def _chart_title(resp, histogram=False): response = c.post("/charts/_dash-update-component", json=params) assert _chart_title(response) == "Col0 by security_id (No Aggregation)" + params["inputs"][-2]["value"] = "histogram" + params["state"][1]["value"]["chart_type"] = "heatmap" + date_val = pd.Timestamp( + df[(df.security_id == 100000) & (df.Col4 == 4)].date.values[0] + ).strftime("%Y%m%d") + params["state"][-2]["value"] = { + "points": [ + { + "x": 100000, + "y": 4, + "z": 1, + "text": "date: {}
security_id: 100000
Col4: 4
Col0: 1".format( + date_val + ), + "customdata": date_val, + } + ] + } + response = c.post("/charts/_dash-update-component", json=params) + assert _chart_title(response, True) == "Histogram of Col0 (1 data points)" + params["inputs"][-2]["value"] = "bar" + response = c.post("/charts/_dash-update-component", json=params) + assert _chart_title(response) == "Col0 by security_id (No Aggregation)" + params["inputs"][-2]["value"] = "histogram" params["state"][1]["value"]["chart_type"] = "maps" params["state"][4]["value"] = { diff --git a/tests/dtale/test_views.py b/tests/dtale/test_views.py index 5f7fbdba..ddc7d98c 100644 --- a/tests/dtale/test_views.py +++ b/tests/dtale/test_views.py @@ -2517,6 +2517,20 @@ def test_main(): assert "D-Tale (test_name) - Test (col: foo)" in str( response.data ) + response = c.get( + "/dtale/popup/reshape/{}".format(c.port), query_string=dict(col="foo") + ) + assert ( + "D-Tale (test_name) - Summarize Data (col: foo)" + in str(response.data) + ) + response = c.get( + "/dtale/popup/filter/{}".format(c.port), query_string=dict(col="foo") + ) + assert ( + "D-Tale (test_name) - Custom Filter (col: foo)" + in str(response.data) + ) with app.test_client() as c: with ExitStack() as stack: @@ -2779,7 +2793,7 @@ def test_get_column_filter_data(unittest, custom_data): "dtale.global_state.DTYPES", {c.port: views.build_dtypes_state(df)} ) ) - stack.enter_context(mock.patch("dtale.global_state.DATA", {c.port: df})) + response = c.get( "/dtale/column-filter-data/{}/{}".format(c.port, "bool_val") ) @@ -2828,6 +2842,29 @@ def test_get_column_filter_data(unittest, custom_data): response_data = json.loads(response.data) assert not response_data["success"] + # mixed data test + df = pd.DataFrame.from_dict( + { + "a": ["a", "UNknown", "b"], + "b": ["", " ", " - "], + "c": [1, "", 3], + "d": [1.1, np.nan, 3], + "e": ["a", np.nan, "b"], + } + ) + df, _ = views.format_data(df) + with build_app(url=URL).test_client() as c: + with ExitStack() as stack: + stack.enter_context(mock.patch("dtale.global_state.DATA", {c.port: df})) + stack.enter_context( + mock.patch( + "dtale.global_state.DTYPES", {c.port: views.build_dtypes_state(df)} + ) + ) + response = c.get("/dtale/column-filter-data/{}/{}".format(c.port, "c")) + response_data = json.loads(response.data) + unittest.assertEqual(sorted(response_data["uniques"]), ["", "1", "3"]) + @pytest.mark.unit @pytest.mark.parametrize("custom_data", [dict(rows=1000, cols=3)], indirect=True) @@ -2843,7 +2880,6 @@ def test_get_async_column_filter_data(unittest, custom_data): "dtale.global_state.DTYPES", {c.port: views.build_dtypes_state(df)} ) ) - stack.enter_context(mock.patch("dtale.global_state.DATA", {c.port: df})) str_val = df.str_val.values[0] response = c.get(