Correlation Scatter Updates:

* #486: make 15K point limitation correlations scatter an editable setting
* #487: fix for non-unique column exception in correlation scatter

dtale/charts/utils.py

@@ -3,6 +3,7 @@
 import numpy as np
 import pandas as pd
 
+import dtale.global_state as global_state
 from dtale.column_analysis import handle_cleaners
 from dtale.query import build_col_key, run_query
 from dtale.utils import (
@@ -48,6 +49,18 @@
     pctsum="Percentage Sum",
 )
 
+CHART_POINTS_LIMIT = (
+    "In order to adjust the limitation on the amount of points in charts please update your startup code that "
+    "loads the dataframe to D-Tale and add these lines of code before calling 'dtale.show':\n\n"
+    "import dtale.global_state as global_state\n\n"
+    "global_state.set_chart_settings({'scatter_points': 15000, '3d_points': 40000})\n\n"
+    "You could also add the following properties to your global configuration file:\n\n"
+    "[charts]\n"
+    "scatter_points = 20000\n\n"
+    "Documentation on global configurations can be found here:\n"
+    "https://github.com/man-group/dtale/blob/master/docs/CONFIGURATION.md"
+)
+
 
 def get_mapbox_token():
     global MAPBOX_TOKEN
@@ -898,11 +911,14 @@ def _group_filter():
     code.append("chart_data = chart_data.dropna()")
 
     dupe_cols = main_group + y_group_cols
+    data_limit = global_state.get_chart_settings()[
+        "3d_points" if is_z or animate_by is not None else "scatter_points"
+    ]
     check_exceptions(
         data[dupe_cols].rename(columns={x_col: x}),
         allow_duplicates or agg in ["raw", "drop_duplicates"],
         unlimited_data=unlimited_data,
-        data_limit=40000 if is_z or animate_by is not None else 15000,
+        data_limit=data_limit,
     )
     data_f, range_f = build_formatters(data)
 

dtale/config.py

@@ -87,6 +87,22 @@ def load_app_settings(config):
     )
 
 
+def load_chart_settings(config):
+    if config is None:
+        return
+    curr_chart_settings = global_state.get_chart_settings()
+
+    scatter_points = get_config_val(
+        config, curr_chart_settings, "scatter_points", section="charts", getter="getint"
+    )
+    three_dimensional_points = get_config_val(
+        config, curr_chart_settings, "3d_points", section="charts", getter="getint"
+    )
+    global_state.set_chart_settings(
+        {"scatter_points": scatter_points, "3d_points": three_dimensional_points}
+    )
+
+
 def load_auth_settings(config):
     if config is None:
         return
@@ -182,3 +198,4 @@ def build_show_options(options=None):
 LOADED_CONFIG = get_config()
 load_app_settings(LOADED_CONFIG)
 load_auth_settings(LOADED_CONFIG)
+load_chart_settings(LOADED_CONFIG)

dtale/global_state.py

@@ -17,6 +17,8 @@
     "github_fork": False,
     "hide_shutdown": False,
     "max_column_width": None,
+    "main_title": None,
+    "main_title_font": None,
 }
 
 AUTH_SETTINGS = {
@@ -25,6 +27,11 @@
     "password": None,
 }
 
+CHART_SETTINGS = {
+    "scatter_points": 15000,
+    "3d_points": 40000,
+}
+
 
 class DtaleInstance(object):
 
@@ -347,6 +354,18 @@ def set_auth_settings(settings):
         AUTH_SETTINGS[prop] = val
 
 
+def get_chart_settings():
+    global CHART_SETTINGS
+    return CHART_SETTINGS
+
+
+def set_chart_settings(settings):
+    global CHART_SETTINGS
+
+    for prop, val in settings.items():
+        CHART_SETTINGS[prop] = val
+
+
 def cleanup(data_id=None):
     if data_id is None:
         _default_store.clear_store()

dtale/static/css/main.css

@@ -10700,6 +10700,11 @@ table tbody tr.highlight-row {
   padding-top: 5px;
 }
 
+div.traceback pre {
+  white-space: pre-wrap;
+  word-break: break-word;
+}
+
 .dtale-alert.alert .ico-cancel {
   cursor: pointer;
 }

dtale/views.py

@@ -33,7 +33,7 @@
 import dtale.global_state as global_state
 import dtale.predefined_filters as predefined_filters
 from dtale import dtale
-from dtale.charts.utils import build_base_chart, build_formatters
+from dtale.charts.utils import build_base_chart, build_formatters, CHART_POINTS_LIMIT
 from dtale.cli.clickutils import retrieve_meta_info_and_version
 from dtale.column_analysis import ColumnAnalysis
 from dtale.column_builders import ColumnBuilder, printable
@@ -2708,7 +2708,7 @@ def get_scatter(data_id):
         build_query(data_id, curr_settings.get("query")),
         global_state.get_context_variables(data_id),
     )
-    idx_col = str("index")
+    idx_col = str("_corr_index")
     y_cols = [cols[1], idx_col]
     code = build_code_export(data_id)
     selected_date = None
@@ -2786,11 +2786,15 @@ def get_scatter(data_id):
         ).format(col1=col1, col2=col2, idx_col=idx_col)
     )
 
-    if len(data) > 15000:
+    max_points = global_state.get_chart_settings()["scatter_points"]
+    if len(data) > max_points:
         return jsonify(
             stats=stats,
             code="\n".join(code),
-            error="Dataset exceeds 15,000 records, cannot render scatter. Please apply filter...",
+            error="Dataset exceeds {:,} records, cannot render scatter. Please apply filter...".format(
+                max_points
+            ),
+            traceback=CHART_POINTS_LIMIT,
         )
     data, _code = build_base_chart(data, cols[0], y_cols, allow_duplicates=True)
     data["x"] = cols[0]

static/popups/correlations/CorrelationsGrid.css

@@ -25,6 +25,7 @@
 .CorrDragHandleIcon {
   justify-content: center;
   align-items: center;
-  line-height: 0.5;
+  line-height: 0.25;
   width: 100%;
+  text-align: center;
 }

tests/dtale/config/dtale.ini

@@ -8,6 +8,10 @@ max_column_width = 100
 main_title = My App
 main_title_font = Arial
 
+[charts]
+scatter_points = 15000
+3d_points = 40000
+
 [show]
 host = localhost
 port = 8080

tests/dtale/test_views.py

@@ -14,6 +14,7 @@
 from six import PY3
 
 from dtale.app import build_app
+from dtale.charts.utils import CHART_POINTS_LIMIT
 from dtale.utils import DuplicateDataError
 from tests import ExitStack
 from tests.dtale import build_data_inst, build_settings, build_dtypes
@@ -1994,6 +1995,7 @@ def test_get_scatter(unittest, rolling_data):
                 "spearman": 1.0,
             },
             error="Dataset exceeds 15,000 records, cannot render scatter. Please apply filter...",
+            traceback=CHART_POINTS_LIMIT,
         )
         unittest.assertEqual(
             {k: v for k, v in response_data.items() if k != "code"},