Merge pull request #424 from man-group/bugfixes_20210205

Bugfixes 20210205

.circleci/config.yml

@@ -45,6 +45,7 @@ javascript: &javascript
             bash <(curl -s https://codecov.io/bash) -c -F javascript -f ./JS_coverage/lcov.info
             yarn run report-duplicate-code
             cp -r ./JS_coverage /tmp/circleci-test-results
+          no_output_timeout: 1200
       - run:
           name: Build JS
           command: |
@@ -63,7 +64,7 @@ python_variables: &python_variables
   CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
   CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
   CODECOV_TOKEN: b0d35139-0a75-427a-907b-2c78a762f8f0
-  VERSION: 1.33.1
+  VERSION: 1.34.0
   PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases
   RUN_BLACK: true
 python: &python
@@ -214,7 +215,7 @@ python: &python
 version: 2
 jobs:
   build_JS:
-    working_directory: ~/man-group/dtale_JS
+    working_directory: /mnt/ramdisk
     docker:
       - image: circleci/node:latest
     <<: *javascript

CHANGES.md

@@ -1,5 +1,10 @@
 ## Changelog
 
+### 1.34.0 (2021-2-7)
+* [#423](https://github.com/man-group/dtale/issues/423): y-axis scale toggle
+* [#422](https://github.com/man-group/dtale/issues/422): sheet selection on excel uploads
+* [#421](https://github.com/man-group/dtale/issues/421): replacements not replacing zeroes
+
 ### 1.33.1 (2021-2-1)
 * [#420](https://github.com/man-group/dtale/issues/420): Added python2.7 support back
 * [#416](https://github.com/man-group/dtale/issues/416): aggregating charts by "first" or "last"

docker/dtale.env

@@ -1,2 +1,2 @@
-VERSION=1.33.1
+VERSION=1.34.0
 TZ=America/New_York

docs/source/conf.py

@@ -64,9 +64,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u"1.33.1"
+version = u"1.34.0"
 # The full version, including alpha/beta/rc tags.
-release = u"1.33.1"
+release = u"1.34.0"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

dtale/cli/clickutils.py

@@ -214,8 +214,12 @@ def handle_str_content(resp):
     return BytesIO(resp.content) if PY3 else StringIO(resp.content.decode("utf-8"))
 
 
+def is_url(path):
+    return path.startswith("http://") or path.startswith("https://")
+
+
 def handle_path(path, kwargs, resp_handler=handle_str_content):
-    if path.startswith("http://") or path.startswith("https://"):
+    if is_url(path):
         proxy = kwargs.pop("proxy", None)
         req_kwargs = {}
         if proxy is not None:

dtale/cli/loaders/excel_loader.py

@@ -25,10 +25,9 @@ def show_loader(**kwargs):
     return show(data_loader=lambda: loader_func(**kwargs), **kwargs)
 
 
-def loader_func(**kwargs):
+def load_file(sheet_name=None, **kwargs):
     path = kwargs.pop("path")
     engine = "xlrd" if path.endswith("xls") else "openpyxl"
-    sheet_name = kwargs.pop("sheet", None)
     path = handle_path(path, kwargs)
     dfs = pd.read_excel(
         path,
@@ -38,6 +37,12 @@ def loader_func(**kwargs):
     )
     if dfs is None or not len(dfs):
         raise Exception("Failed to load Excel file. Returned no data.")
+    return dfs
+
+
+def loader_func(**kwargs):
+    sheet_name = kwargs.pop("sheet", None)
+    dfs = load_file(sheet_name=sheet_name, **kwargs)
     if sheet_name:
         if sheet_name not in dfs:
             raise Exception(

dtale/column_replacements.py

@@ -36,7 +36,8 @@ def get_inner_replacement_value(val):
 
 
 def get_replacement_value(cfg, prop):
-    value = (cfg or {}).get(prop) or "nan"
+    value = (cfg or {}).get(prop)
+    value = "nan" if value is None else value
     return get_inner_replacement_value(value)
 
 

dtale/dash_application/charts.py

@@ -186,6 +186,7 @@ def chart_url_querystring(params, data=None, group_filter=None):
         "group_type",
         "bins_val",
         "bin_type",
+        "scale",
     ]
     chart_type = params.get("chart_type")
     if chart_type == "bar":
@@ -247,7 +248,9 @@ def build_colorscale(colorscale):
     return [[i / (len(colorscale) - 1), rgb] for i, rgb in enumerate(colorscale)]
 
 
-def build_axes(data_id, x, axis_inputs, mins, maxs, z=None, agg=None, data=None):
+def build_axes(
+    data_id, x, axis_inputs, mins, maxs, z=None, agg=None, data=None, scale="linear"
+):
     """
     Returns helper function for building axis configurations against a specific y-axis.
 
@@ -286,7 +289,9 @@ def _build_axes(y):
             for i, y2 in enumerate(y, 0):
                 right = i % 2 == 1
                 axis_ct = int(i / 2)
-                value = dict(title=_add_agg_label(update_label_for_freq(y2)))
+                value = dict(
+                    title=_add_agg_label(update_label_for_freq(y2)), type=scale
+                )
                 if i == 0:
                     key = "yaxis"
                 else:
@@ -313,7 +318,7 @@ def _build_axes(y):
                     value["tickformat"] = ".0f"
                 axes[key] = value
         elif axis_type == "single":
-            yaxis_cfg = dict(title=_add_agg_label(update_label_for_freq(y)))
+            yaxis_cfg = dict(title=_add_agg_label(update_label_for_freq(y)), type=scale)
             all_range = axis_data.get("all") or {}
             all_range = [
                 all_range.get(p) for p in ["min", "max"] if all_range.get(p) is not None
@@ -326,7 +331,7 @@ def _build_axes(y):
                 yaxis_cfg["tickformat"] = ".0f"
             axes["yaxis"] = yaxis_cfg
         else:
-            yaxis_cfg = dict(title=_add_agg_label(update_label_for_freq(y)))
+            yaxis_cfg = dict(title=_add_agg_label(update_label_for_freq(y)), type=scale)
             if classify_type(dtypes.get(y[0])) == "I":
                 yaxis_cfg["tickformat"] = ".0f"
             axes["yaxis"] = yaxis_cfg
@@ -2800,9 +2805,9 @@ def build_chart(data_id=None, data=None, **inputs):
         range_data = dict(min=data["min"], max=data["max"])
         axis_inputs = inputs.get("yaxis") or {}
         chart_builder = chart_wrapper(data_id, data, inputs)
-        x, y, z, agg, group, animate_by, trendline = (
+        x, y, z, agg, group, animate_by, trendline, scale = (
             inputs.get(p)
-            for p in ["x", "y", "z", "agg", "group", "animate_by", "trendline"]
+            for p in ["x", "y", "z", "agg", "group", "animate_by", "trendline", "scale"]
         )
         x = str("x") if x is None else x
         z = z if chart_type in ZAXIS_CHARTS else None
@@ -2832,7 +2837,7 @@ def build_chart(data_id=None, data=None, **inputs):
             )
 
         axes_builder = build_axes(
-            data_id, x, axis_inputs, data["min"], data["max"], z=z, agg=agg
+            data_id, x, axis_inputs, data["min"], data["max"], z=z, agg=agg, scale=scale
         )
         if chart_type in ["scatter", "3d_scatter"]:
             kwargs = dict(agg=agg)

dtale/dash_application/layout/layout.py

@@ -781,6 +781,13 @@ def get_yaxis_type_tabs(y):
     return tabs + [build_tab("Multi", "multi", {"padding": "2px", "minWidth": "4em"})]
 
 
+def get_yaxis_scale_tabs():
+    return [
+        build_tab("Linear", "linear", {"padding": "2px", "minWidth": "4em"}),
+        build_tab("Log", "log", {"padding": "2px", "minWidth": "4em"}),
+    ]
+
+
 def build_group_val_options(df, group_cols):
     group_vals = find_group_vals(df, group_cols)
     return [
@@ -1763,11 +1770,19 @@ def show_map_style(show):
                                 className="input-group-addon",
                             ),
                             html.Div(
-                                dcc.Tabs(
-                                    id="yaxis-type",
-                                    value=yaxis_type,
-                                    children=get_yaxis_type_tabs(y),
-                                ),
+                                [
+                                    dcc.Tabs(
+                                        id="yaxis-scale",
+                                        value=inputs.get("scale") or "linear",
+                                        children=get_yaxis_scale_tabs(),
+                                        className="pr-5",
+                                    ),
+                                    dcc.Tabs(
+                                        id="yaxis-type",
+                                        value=yaxis_type,
+                                        children=get_yaxis_type_tabs(y),
+                                    ),
+                                ],
                                 id="yaxis-type-div",
                                 className="form-control col-auto pt-3",
                                 style=yaxis_type_style,

dtale/dash_application/views.py

@@ -594,10 +594,19 @@ def input_toggles(_ts, inputs, pathname):
             Input("animate-toggle", "on"),
             Input("animate-by-dropdown", "value"),
             Input("trendline-dropdown", "value"),
+            Input("yaxis-scale", "value"),
         ],
     )
     def chart_input_data(
-        cpg, barmode, barsort, top_bars, colorscale, animate, animate_by, trendline
+        cpg,
+        barmode,
+        barsort,
+        top_bars,
+        colorscale,
+        animate,
+        animate_by,
+        trendline,
+        scale,
     ):
         """
         dash callback for maintaining selections in chart-formatting inputs
@@ -614,6 +623,7 @@ def chart_input_data(
             animate=animate,
             animate_by=animate_by,
             trendline=trendline,
+            scale=scale,
         )
 
     @dash_app.callback(

dtale/views.py

@@ -3119,23 +3119,40 @@ def chart_csv_export(data_id):
     return send_file(csv_buffer.getvalue(), filename, "text/csv")
 
 
-@dtale.route("/cleanup/<data_id>")
+@dtale.route("/cleanup-datasets")
 @exception_decorator
-def run_cleanup(data_id):
-    global_state.cleanup(data_id)
+def cleanup_datasets():
+    data_ids = get_str_arg(request, "dataIds")
+    data_ids = (data_ids or "").split(",")
+    for data_id in data_ids:
+        global_state.cleanup(data_id)
     return jsonify(success=True)
 
 
-def load_new_data(df, startup_code):
-    instance = startup(data=df, ignore_duplicate=True)
+def load_new_data(df, startup_code, name=None, return_id=False):
+    instance = startup(data=df, name=name, ignore_duplicate=True)
     curr_settings = global_state.get_settings(instance._data_id)
     global_state.set_settings(
         instance._data_id,
         dict_merge(curr_settings, dict(startup_code=startup_code)),
     )
+    if return_id:
+        return instance._data_id
     return jsonify(success=True, data_id=instance._data_id)
 
 
+def handle_excel_upload(dfs):
+    sheet_names = list(dfs.keys())
+    data_ids = []
+    for sheet_name in sheet_names:
+        df, code = dfs[sheet_name]
+        if len(sheet_names) == 1:
+            return load_new_data(df, code, name=sheet_name)
+        data_id = load_new_data(df, code, name=sheet_name, return_id=True)
+        data_ids.append(dict(name=sheet_name, dataId=data_id))
+    return jsonify(dict(sheets=data_ids, success=True))
+
+
 @dtale.route("/upload", methods=["POST"])
 @exception_decorator
 def upload():
@@ -3155,18 +3172,17 @@ def upload():
         if ext in [".xls", ".xlsx"]:
             engine = "xlrd" if ext == ".xls" else "openpyxl"
             dfs = pd.read_excel(contents, sheet_name=None, engine=engine)
-            if not dfs:
-                raise Exception("Failed to load Excel file. Returned no data.")
-            sheet_names = list(dfs.keys())
-            for sheet_name in sheet_names:
-                df = dfs[sheet_name]
-                code = "df = pd.read_excel('{}', sheet_name='{}', engine='{}')".format(
+
+            def build_xls_code(sheet_name):
+                return "df = pd.read_excel('{}', sheet_name='{}', engine='{}')".format(
                     filename, sheet_name, engine
                 )
-                if sheet_name == sheet_names[-1]:
-                    return load_new_data(df, code)
-                else:
-                    load_new_data(df, code)
+
+            dfs = {
+                sheet_name: (df, build_xls_code(sheet_name))
+                for sheet_name, df in dfs.items()
+            }
+            return handle_excel_upload(dfs)
         raise Exception("File type of {} is not supported!".format(ext))
 
 
@@ -3175,7 +3191,7 @@ def upload():
 def web_upload():
     from dtale.cli.loaders.csv_loader import loader_func as load_csv
     from dtale.cli.loaders.json_loader import loader_func as load_json
-    from dtale.cli.loaders.excel_loader import loader_func as load_excel
+    from dtale.cli.loaders.excel_loader import load_file as load_excel
 
     data_type = get_str_arg(request, "type")
     url = get_str_arg(request, "url")
@@ -3199,11 +3215,23 @@ def web_upload():
             "df = load_csv(path='{url}'{proxy})"
         ).format(url=url, proxy=", '{}'".format(proxy) if proxy else "")
     elif data_type == "excel":
-        df = load_excel(path=url, proxy=proxy)
-        startup_code = (
-            "from dtale.cli.loaders.excel_loader import loader_func as load_excel\n\n"
-            "df = load_excel(path='{url}'{proxy})"
-        ).format(url=url, proxy=", '{}'".format(proxy) if proxy else "")
+        dfs = load_excel(path=url, proxy=proxy)
+
+        def build_xls_code(sheet_name):
+            return (
+                "from dtale.cli.loaders.excel_loader import load_file as load_excel\n\n"
+                "df = load_excel(sheet_name='{sheet_name}', path='{url}'{proxy})"
+            ).format(
+                sheet_name=sheet_name,
+                url=url,
+                proxy=", '{}'".format(proxy) if proxy else "",
+            )
+
+        dfs = {
+            sheet_name: (df, build_xls_code(sheet_name))
+            for sheet_name, df in dfs.items()
+        }
+        return handle_excel_upload(dfs)
 
     return load_new_data(df, startup_code)
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dtale",
-  "version": "1.33.1",
+  "version": "1.34.0",
   "description": "Visualizer for Pandas Data Structures",
   "main": "main.js",
   "directories": {

setup.py

@@ -64,7 +64,7 @@ def run_tests(self):
 
 setup(
     name="dtale",
-    version="1.33.1",
+    version="1.34.0",
     author="MAN Alpha Technology",
     author_email="ManAlphaTech@man.com",
     description="Web Client for Visualizing Pandas Objects",

static/__tests__/dtale/DataViewer-upload-test.jsx

@@ -48,7 +48,7 @@ describe("DataViewer tests", () => {
     mockChartJS();
 
     DataViewer = require("../../dtale/DataViewer").DataViewer;
-    Upload = require("../../popups/Upload").ReactUpload;
+    Upload = require("../../popups/upload/Upload").ReactUpload;
   });
 
   beforeEach(async () => {