fixes for [#50](#50)

CHANGES.md

@@ -1,5 +1,9 @@
 ## Changelog
 
+### 1.6.7 (2020-1-3)
+
+ * [#50](https://github.com/man-group/dtale/issues/50): updates to rolling correlation functionality
+
 ### 1.6.6 (2020-1-2)
 
  * [#47](https://github.com/man-group/dtale/issues/47): selection of multiple columns for y-axis

docker/2_7/Dockerfile

@@ -44,4 +44,4 @@ WORKDIR /app
 
 RUN set -eux \
   ; . /root/.bashrc \
-  ; easy_install dtale-1.6.6-py2.7.egg
+  ; easy_install dtale-1.6.7-py2.7.egg

docker/3_6/Dockerfile

@@ -44,4 +44,4 @@ WORKDIR /app
 
 RUN set -eux \
   ; . /root/.bashrc \
-  ; easy_install dtale-1.6.6-py3.7.egg
+  ; easy_install dtale-1.6.7-py3.7.egg

docs/source/conf.py

@@ -63,9 +63,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'1.6.6'
+version = u'1.6.7'
 # The full version, including alpha/beta/rc tags.
-release = u'1.6.6'
+release = u'1.6.7'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

dtale/static/css/main.css

@@ -10337,3 +10337,47 @@ table tbody tr.highlight-row {
     0%{background-position:0% 50%;}
     0%{background-position:100% -50%;}
 }
+
+[data-tip] {
+	position:relative;
+
+}
+[data-tip]:before {
+	content:'';
+	/* hides the tooltip when not hovered */
+	display:none;
+	content:'';
+	border-left: 5px solid transparent;
+	border-right: 5px solid transparent;
+	border-bottom: 5px solid #1a1a1a;
+	position:absolute;
+	top:30px;
+	left:35px;
+	z-index:8;
+	font-size:0;
+	line-height:0;
+	width:0;
+	height:0;
+}
+[data-tip]:after {
+	display:none;
+	content:attr(data-tip);
+	position:absolute;
+	top:35px;
+	left:0px;
+	padding:5px 8px;
+	background:#1a1a1a;
+	color:#fff;
+	z-index:9;
+	font-size: 0.75em;
+	line-height:18px;
+	-webkit-border-radius: 3px;
+	-moz-border-radius: 3px;
+	border-radius: 3px;
+	white-space:nowrap;
+	word-wrap:normal;
+}
+[data-tip]:hover:before,
+[data-tip]:hover:after {
+	display:block;
+}

dtale/utils.py

@@ -111,6 +111,23 @@ def get_str_arg(r, name, default=None):
             return default
 
 
+def get_json_arg(r, name, default=None):
+    """
+    Retrieve argument from :attr:`flask:flask.request` and parse JSON to python data structure
+
+    :param r: :attr:`flask:flask.request`
+    :param name: argument name
+    :type: str
+    :param default: default value if parameter is non-existent, defaults to None
+    :return: parsed JSON
+    """
+    val = r.args.get(name)
+    if val is None or val == '':
+        return default
+    else:
+        return json.loads(val)
+
+
 def get_int_arg(r, name, default=None):
     """
     Retrieve argument from :attr:`flask:flask.request` and convert to integer

dtale/views.py

@@ -17,7 +17,7 @@
 from dtale.utils import (build_shutdown_url, classify_type, dict_merge,
                          filter_df_for_grid, find_dtype_formatter,
                          find_selected_column, get_bool_arg, get_dtypes,
-                         get_int_arg, get_str_arg, grid_columns,
+                         get_int_arg, get_json_arg, get_str_arg, grid_columns,
                          grid_formatter, json_date, json_float, json_int,
                          json_timestamp, jsonify, make_list,
                          retrieve_grid_params, running_with_flask_debug,
@@ -539,7 +539,7 @@ def update_settings(data_id):
         global SETTINGS
 
         curr_settings = SETTINGS.get(data_id, {})
-        updated_settings = dict_merge(curr_settings, json.loads(get_str_arg(request, 'settings', '{}')))
+        updated_settings = dict_merge(curr_settings, get_json_arg(request, 'settings', {}))
         SETTINGS[data_id] = updated_settings
         return jsonify(dict(success=True))
     except BaseException as e:
@@ -718,10 +718,8 @@ def get_data(data_id):
             DTYPES[data_id] = build_dtypes_state(data)
 
         params = retrieve_grid_params(request)
-        ids = get_str_arg(request, 'ids')
-        if ids:
-            ids = json.loads(ids)
-        else:
+        ids = get_json_arg(request, 'ids')
+        if ids is None:
             return jsonify({})
 
         col_types = DTYPES[data_id]
@@ -861,7 +859,7 @@ def build_chart(data, x, y, group_col=None, agg=None, allow_duplicates=False, **
     :param x: column to be used as x-axis of chart
     :type x: str
     :param y: column to be used as y-axis of chart
-    :type y: str
+    :type y: list of strings
     :param group: comma-separated string of columns to group chart data by
     :type group: str, optional
     :param aggregation: points to a specific function that can be applied to
@@ -880,7 +878,7 @@ def build_formatters(df):
         return data_f, range_f
 
     x_col = str('x')
-    y_cols = y.split(',')
+    y_cols = make_list(y)
     if group_col is not None:
         data = data[group_col + [x] + y_cols].sort_values(group_col + [x])
         y_cols = [str(y_col) for y_col in y_cols]
@@ -973,10 +971,8 @@ def get_chart_data(data_id):
             if not len(data):
                 return jsonify(dict(error='query "{}" found no data, please alter'.format(query)))
         x = get_str_arg(request, 'x')
-        y = get_str_arg(request, 'y')
-        group_col = get_str_arg(request, 'group')
-        if group_col is not None:
-            group_col = group_col.split(',')
+        y = get_json_arg(request, 'y')
+        group_col = get_json_arg(request, 'group')
         agg = get_str_arg(request, 'agg')
         allow_duplicates = get_bool_arg(request, 'allowDupes')
         window = get_int_arg(request, 'rollingWin')
@@ -1009,13 +1005,15 @@ def get_correlations_ts(data_id):
         data = DATA[data_id]
         data = data.query(query) if query is not None else data
         cols = get_str_arg(request, 'cols')
-        cols = cols.split(',')
+        cols = json.loads(cols)
         date_col = get_str_arg(request, 'dateCol')
         rolling_window = get_int_arg(request, 'rollingWindow')
         if rolling_window:
             [col1, col2] = list(set(cols))
-            data = data[['date', col1, col2]].set_index('date')
-            data = data[col1].rolling(rolling_window).corr(data[col2]).reset_index()
+            data = data[[date_col, col1, col2]].set_index(date_col)
+            data = data[[col1, col2]].rolling(rolling_window).corr().reset_index()
+            data = data.dropna()
+            data = data[data['level_1'] == col1][[date_col, col2]]
         else:
             data = data.groupby(date_col)[list(set(cols))].corr(method='pearson')
             data.index.names = ['date', 'column']
@@ -1054,19 +1052,29 @@ def get_scatter(data_id):
         y: col2
     } or {error: 'Exception message', traceback: 'Exception stacktrace'}
     """
-    cols = get_str_arg(request, 'cols')
-    cols = cols.split(',')
+    cols = get_json_arg(request, 'cols')
     query = get_str_arg(request, 'query')
     date = get_str_arg(request, 'date')
     date_col = get_str_arg(request, 'dateCol')
+    rolling = get_bool_arg(request, 'rolling')
     try:
         data = DATA[data_id]
-        data = data[data[date_col] == date] if date else data
         if query:
             data = data.query(query)
 
-        data = data[list(set(cols))].dropna(how='any')
-        data[str('index')] = data.index
+        idx_col = str('index')
+        y_cols = [cols[1], idx_col]
+        if rolling:
+            window = get_int_arg(request, 'window')
+            idx = min(data[data[date_col] == date].index) + 1
+            data = data.iloc[(idx - window):idx]
+            data = data[list(set(cols)) + [date_col]].dropna(how='any')
+            y_cols.append(date_col)
+        else:
+            data = data[data[date_col] == date] if date else data
+            data = data[list(set(cols))].dropna(how='any')
+
+        data[idx_col] = data.index
         s0 = data[cols[0]]
         s1 = data[cols[1]]
         pearson = s0.corr(s1, method='pearson')
@@ -1084,7 +1092,7 @@ def get_scatter(data_id):
                 stats=stats,
                 error='Dataset exceeds 15,000 records, cannot render scatter. Please apply filter...'
             )
-        data = build_chart(data, cols[0], str('{},index'.format(cols[1])), allow_duplicates=True)
+        data = build_chart(data, cols[0], y_cols, allow_duplicates=True)
         data['x'] = cols[0]
         data['y'] = cols[1]
         data['stats'] = stats

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dtale",
-  "version": "1.6.6",
+  "version": "1.6.7",
   "description": "Visualizer for Pandas Data Structures",
   "main": "main.js",
   "directories": {

setup.py

@@ -50,7 +50,7 @@ def run_tests(self):
 
 setup(
     name="dtale",
-    version="1.6.6",
+    version="1.6.7",
     author="MAN Alpha Technology",
     author_email="ManAlphaTech@man.com",
     description="Web Client for Visualizing Pandas Objects",

static/__tests__/actions/url-utils-test.jsx

@@ -14,7 +14,7 @@ describe("url-utils tests", () => {
     t.deepEqual(
       urlParams,
       {
-        cols: "col1,col2",
+        cols: '["col1","col2"]',
         query: "col == 3",
         sort: '[["col1","ASC"]]',
         ids: "[0,5]",

static/__tests__/popups/Correlations-test.jsx

@@ -199,7 +199,7 @@ describe("Correlations tests", () => {
           [{ datasetIndex: 0, index: 0 }],
           scatterChart.data
         );
-        t.deepEqual(title, [["Index: 0"]], "should render title");
+        t.deepEqual(title, ["index: 0"], "should render title");
         const label = scatterChart.cfg.options.tooltips.callbacks.label(
           { datasetIndex: 0, index: 0 },
           scatterChart.data
@@ -219,6 +219,7 @@ describe("Correlations tests", () => {
   test("Correlations rendering rolling data", done => {
     const Correlations = require("../../popups/Correlations").ReactCorrelations;
     const ChartsBody = require("../../popups/charts/ChartsBody").default;
+    const CorrelationScatterStats = require("../../popups/correlations/CorrelationScatterStats").default;
     buildInnerHTML({ settings: "" });
     const result = mount(<Correlations chartData={_.assign({}, chartData, { query: "rolling" })} dataId="1" />, {
       attachTo: document.getElementById("content"),
@@ -234,26 +235,44 @@ describe("Correlations tests", () => {
       setTimeout(() => {
         result.update();
         t.equal(result.find(ChartsBody).length, 1, "should show correlation timeseries");
-        t.equal(result.find("#rawScatterChart").length, 1, "should show scatter chart");
-        t.deepEqual(
-          result
-            .find("select.custom-select")
-            .first()
-            .find("option")
-            .map(o => o.text()),
-          ["col4", "col5"],
-          "should render date options for timeseries"
-        );
-        t.ok((result.state().selectedDate = "col4"), "should select timeseries date");
         result
-          .find(CorrelationsTsOptions)
-          .find("input")
-          .findWhere(i => i.prop("type") === "text")
-          .simulate("change", { target: { value: "5" } });
+          .find(ChartsBody)
+          .instance()
+          .state.charts[0].cfg.options.onClick({ foo: 1 });
         setTimeout(() => {
           result.update();
-          done();
-        }, 200);
+          t.ok(result.find(Correlations).instance().state.chart, "should show scatter chart");
+          t.ok(_.startsWith(result.find(CorrelationScatterStats).text(), "col1 vs. col2 for 20181215-20181219"));
+          t.deepEqual(
+            result
+              .find(Correlations)
+              .instance()
+              .state.chart.cfg.options.tooltips.callbacks.title(
+                [{ datasetIndex: 0, index: 0 }],
+                result.find(Correlations).instance().state.chart.data
+              ),
+            ["index: 0", "date: 2018-04-30 12:36:44 pm"]
+          );
+          t.deepEqual(
+            result
+              .find("select.custom-select")
+              .first()
+              .find("option")
+              .map(o => o.text()),
+            ["col4", "col5"],
+            "should render date options for timeseries"
+          );
+          t.ok((result.state().selectedDate = "col4"), "should select timeseries date");
+          result
+            .find(CorrelationsTsOptions)
+            .find("input")
+            .findWhere(i => i.prop("type") === "text")
+            .simulate("change", { target: { value: "5" } });
+          setTimeout(() => {
+            result.update();
+            done();
+          }, 200);
+        }, 400);
       }, 200);
     }, 200);
   });

static/__tests__/popups/Histogram-test.jsx

@@ -106,8 +106,13 @@ describe("Histogram tests", () => {
       const xlabel = _.get(chart, "cfg.options.scales.xAxes[0].scaleLabel.labelString");
       t.equal(xlabel, "Bin", "should display correct x-axis label");
 
-      const event = { target: { value: 50 } };
-      result.find("input").simulate("change", event);
+      result.find("input").simulate("change", { target: { value: "" } });
+      result.find("input").simulate("keyPress", { key: "Shift" });
+      result.find("input").simulate("keyPress", { key: "Enter" });
+      result.find("input").simulate("change", { target: { value: "a" } });
+      result.find("input").simulate("keyPress", { key: "Enter" });
+      result.find("input").simulate("change", { target: { value: 50 } });
+      result.find("input").simulate("keyPress", { key: "Enter" });
 
       setTimeout(() => {
         result.update();

static/__tests__/popups/window/Charts-bar-test.jsx

@@ -37,7 +37,7 @@ describe("Charts bar tests", () => {
     const mockBuildLibs = withGlobalJquery(() =>
       mockPopsicle.mock(url => {
         const urlParams = qs.parse(url.split("?")[1]);
-        if (urlParams.x === "error" && urlParams.y === "error2") {
+        if (urlParams.x === "error" && _.includes(JSON.parse(urlParams.y), "error2")) {
           return { data: {} };
         }
         const { urlFetcher } = require("../../redux-test-utils").default;

static/__tests__/popups/window/Charts-multi-column-test.jsx

@@ -38,7 +38,7 @@ describe("Charts tests", () => {
     const mockBuildLibs = withGlobalJquery(() =>
       mockPopsicle.mock(url => {
         const urlParams = qs.parse(url.split("?")[1]);
-        if (urlParams.x === "error" && urlParams.y === "error2") {
+        if (urlParams.x === "error" && _.includes(JSON.parse(urlParams.y), "error2")) {
           return { data: {} };
         }
         const { urlFetcher } = require("../../redux-test-utils").default;
@@ -139,10 +139,14 @@ describe("Charts tests", () => {
       setTimeout(() => {
         result.update();
         t.ok(result.find(ChartsBody).instance().state.charts.length == 1, "should render charts");
-        t.ok(
-          _.endsWith(
-            result.find(Charts).instance().state.url,
-            "x=col4&y=col1%2Ccol2&query=col4%20%3D%3D%20'20181201'&agg=rolling&rollingWin=10&rollingComp=corr"
+        t.equal(
+          _.last(_.split(result.find(Charts).instance().state.url, "?")),
+          _.join(
+            [
+              "x=col4&y=%5B%22col1%22%2C%22col2%22%5D&query=col4%20%3D%3D%20'20181201'",
+              "agg=rolling&rollingWin=10&rollingComp=corr",
+            ],
+            "&"
           ),
           "should update chart URL"
         );

static/__tests__/popups/window/Charts-rolling-test.jsx

@@ -27,7 +27,7 @@ describe("Charts rolling tests", () => {
     const mockBuildLibs = withGlobalJquery(() =>
       mockPopsicle.mock(url => {
         const urlParams = qs.parse(url.split("?")[1]);
-        if (urlParams.x === "error" && urlParams.y === "error2") {
+        if (urlParams.x === "error" && _.includes(JSON.parse(urlParams.y), "error2")) {
           return { data: {} };
         }
         const { urlFetcher } = require("../../redux-test-utils").default;