#592: highlight rows of a dataframe filter

dtale/app.py

@@ -679,7 +679,7 @@ def show(data=None, data_loader=None, name=None, context_vars=None, **options):
     :param github_fork: If true, this will display a "Fork me on GitHub" ribbon in the upper right-hand corner of the
                         app
     :type github_fork: bool, optional
-    :param hide_drop_rows: If true, this will hide the "Drop Rows" buton from users
+    :param hide_drop_rows: If true, this will hide the "Drop Rows" button from users
     :type hide_drop_rows: bool, optional
     :param hide_shutdown: If true, this will hide the "Shutdown" buton from users
     :type hide_shutdown: bool, optional
@@ -688,6 +688,9 @@ def show(data=None, data_loader=None, name=None, context_vars=None, **options):
     :param auto_hide_empty_columns: if True, then auto-hide any columns on the front-end that are comprised entirely of
                                     NaN values
     :type auto_hide_empty_columns: boolean, optional
+    :param highlight_filter: if True, then highlight rows on the frontend which will be filtered when applying a filter
+                             rather than hiding them from the dataframe
+    :type highlight_filter: boolean, optional
 
     :Example:
 
@@ -766,6 +769,7 @@ def show(data=None, data_loader=None, name=None, context_vars=None, **options):
             hide_shutdown=final_options.get("hide_shutdown"),
             column_edit_options=final_options.get("column_edit_options"),
             auto_hide_empty_columns=final_options.get("auto_hide_empty_columns"),
+            highlight_filter=final_options.get("highlight_filter"),
         )
         instance.started_with_open_browser = final_options["open_browser"]
         is_active = not running_with_flask_debug() and is_up(app_url)

dtale/column_builders.py

@@ -956,7 +956,7 @@ def build_column(self, data):
             n = int(self.cfg.get("n"))
             features = (
                 FeatureHasher(n_features=n, input_type="string")
-                .transform(data[col].astype("str"))
+                .transform(data[[col]].astype("str").values)
                 .toarray()
             )
             features = pd.DataFrame(features, index=data.index)

dtale/config.py

@@ -184,6 +184,7 @@ def build_show_options(options=None):
         hide_shutdown=False,
         column_edit_options=None,
         auto_hide_empty_columns=False,
+        highlight_filter=False,
     )
     config_options = {}
     config = get_config()
@@ -273,6 +274,9 @@ def build_show_options(options=None):
         config_options["auto_hide_empty_columns"] = get_config_val(
             config, defaults, "auto_hide_empty_columns", "getboolean"
         )
+        config_options["highlight_filter"] = get_config_val(
+            config, defaults, "highlight_filter", "getboolean"
+        )
 
     return dict_merge(defaults, config_options, options)
 

dtale/query.py

@@ -31,7 +31,13 @@ def inner_build_query(settings, query=None):
 
 
 def run_query(
-    df, query, context_vars=None, ignore_empty=False, pct=100, pct_type="random"
+    df,
+    query,
+    context_vars=None,
+    ignore_empty=False,
+    pct=100,
+    pct_type="random",
+    highlight_filter=False,
 ):
     """
     Utility function for running :func:`pandas:pandas.DataFrame.query` . This function contains extra logic to
@@ -48,6 +54,8 @@ def run_query(
     :type context_vars: dict, optional
     :param pct: random percentage of dataframe to load
     :type pct: int, optional
+    :param highlight_filter: if true, then highlight which rows will be filtered rather than drop them
+    :type highlight_filter: boolean, optional
     :return: filtered dataframe
     """
 
@@ -61,20 +69,34 @@ def _load_pct(df):
         return df
 
     if (query or "") == "":
+        if highlight_filter:
+            return _load_pct(df), []
         return _load_pct(df)
 
     is_pandas25 = parse_version(pd.__version__) >= parse_version("0.25.0")
     curr_app_settings = global_state.get_app_settings()
     engine = curr_app_settings.get("query_engine", "python")
-    df = df.query(
-        query if is_pandas25 else query.replace("`", ""),
-        local_dict=context_vars or {},
-        engine=engine,
-    )
+    filtered_indexes = []
+    if highlight_filter:
+        filtered_indexes = set(
+            df.query(
+                query if is_pandas25 else query.replace("`", ""),
+                local_dict=context_vars or {},
+                engine=engine,
+            ).index
+        )
+    else:
+        df = df.query(
+            query if is_pandas25 else query.replace("`", ""),
+            local_dict=context_vars or {},
+            engine=engine,
+        )
 
     if not len(df) and not ignore_empty:
         raise Exception('query "{}" found no data, please alter'.format(query))
 
+    if highlight_filter:
+        return _load_pct(df), filtered_indexes
     return _load_pct(df)
 
 

dtale/views.py

@@ -319,16 +319,25 @@ def update_settings(self, **updates):
         Helper function for updating instance-specific settings. For example:
         * allow_cell_edits - whether cells can be edited
         * locked - which columns are locked to the left of the grid
+        * sort - The sort to apply to the data on startup (EX: [("col1", "ASC"), ("col2", "DESC"),...])
         * custom_formats - display formatting for specific columns
         * background_mode - different background displays in grid
         * range_highlights - specify background colors for ranges of values in the grid
         * vertical_headers - if True, then rotate column headers vertically
+        * column_edit_options - the options to allow on the front-end when editing a cell for the columns specified
+        * highlight_filter - if True, then highlight rows on the frontend which will be filtered when applying a filter
+                             rather than hiding them from the dataframe
+        * hide_shutdown - if true, this will hide the "Shutdown" buton from users
+        * nan_display - if value in dataframe is :attr:`numpy:numpy.nan` then return this value on the frontend
+
+        After applying please refresh any open browsers!
         """
         name_updates = dict(
             range_highlights="rangeHighlight",
             column_formats="columnFormats",
             background_mode="backgroundMode",
             vertical_headers="verticalHeaders",
+            highlight_filter="highlightFilter",
         )
         settings = {name_updates.get(k, k): v for k, v in updates.items()}
         _update_settings(self._data_id, settings)
@@ -942,6 +951,7 @@ def startup(
     hide_shutdown=False,
     column_edit_options=None,
     auto_hide_empty_columns=False,
+    highlight_filter=False,
 ):
     """
     Loads and stores data globally
@@ -1004,6 +1014,9 @@ def startup(
     :param auto_hide_empty_columns: if True, then auto-hide any columns on the front-end that are comprised entirely of
                                     NaN values
     :type auto_hide_empty_columns: boolean, optional
+    :param highlight_filter: if True, then highlight rows on the frontend which will be filtered when applying a filter
+                             rather than hiding them from the dataframe
+    :type highlight_filter: boolean, optional
     """
 
     if (
@@ -1059,6 +1072,7 @@ def startup(
                 hide_shutdown=hide_shutdown,
                 column_edit_options=column_edit_options,
                 auto_hide_empty_columns=auto_hide_empty_columns,
+                highlight_filter=highlight_filter,
             )
 
             global_state.set_dataset(instance._data_id, data)
@@ -1103,6 +1117,7 @@ def startup(
             backgroundMode=background_mode,
             rangeHighlight=range_highlights,
             verticalHeaders=vertical_headers,
+            highlightFilter=highlight_filter,
         )
         base_predefined = predefined_filters.init_filters()
         if base_predefined:
@@ -2537,8 +2552,8 @@ def get_data(data_id):
     if not export and ids is None:
         return jsonify({})
 
-    col_types = global_state.get_dtypes(data_id)
     curr_settings = global_state.get_settings(data_id) or {}
+    col_types = global_state.get_dtypes(data_id)
     f = grid_formatter(col_types, nan_display=curr_settings.get("nanDisplay", "nan"))
     if curr_settings.get("sortInfo") != params.get("sort"):
         data = sort_df_for_grid(data, params)
@@ -2548,12 +2563,17 @@ def get_data(data_id):
     else:
         curr_settings = {k: v for k, v in curr_settings.items() if k != "sortInfo"}
     final_query = build_query(data_id, curr_settings.get("query"))
+    highlight_filter = curr_settings.get("highlightFilter") or False
+    filtered_indexes = []
     data = run_query(
         handle_predefined(data_id),
         final_query,
         global_state.get_context_variables(data_id),
         ignore_empty=True,
+        highlight_filter=highlight_filter,
     )
+    if highlight_filter:
+        data, filtered_indexes = data
     global_state.set_settings(data_id, curr_settings)
 
     total = len(data)
@@ -2574,6 +2594,8 @@ def get_data(data_id):
                     results[sub_range[0]] = dict_merge(
                         {IDX_COL: sub_range[0]}, sub_df[0]
                     )
+                    if highlight_filter and sub_range[0] in filtered_indexes:
+                        results[sub_range[0]]["__filtered"] = True
                 else:
                     [start, end] = sub_range
                     sub_df = (
@@ -2584,11 +2606,16 @@ def get_data(data_id):
                     sub_df = f.format_dicts(sub_df.itertuples())
                     for i, d in zip(range(start, end + 1), sub_df):
                         results[i] = dict_merge({IDX_COL: i}, d)
+                        if highlight_filter and i in filtered_indexes:
+                            results[i]["__filtered"] = True
     columns = [
         dict(name=IDX_COL, dtype="int64", visible=True)
     ] + global_state.get_dtypes(data_id)
     return_data = dict(
-        results=results, columns=columns, total=total, final_query=final_query
+        results=results,
+        columns=columns,
+        total=total,
+        final_query=None if highlight_filter else final_query,
     )
 
     if export:
@@ -3354,6 +3381,7 @@ def value_as_str(value):
             "outlierFilters",
             "predefinedFilters",
             "invertFilter",
+            "highlightFilter",
         ]
     }
     return jsonify(contextVars=ctxt_vars, success=True, **curr_settings)

frontend/static/__tests__/dtale/DataViewer-filter-test.tsx

@@ -81,6 +81,10 @@ describe('FilterPanel', () => {
     await toggleFilterMenu();
     await clickFilterBtn('numexpr');
     expect(store.getState().queryEngine).toBe('numexpr');
+    await act(async () => {
+      await fireEvent.click(screen.getByText('Highlight Filtered Rows').parentElement?.getElementsByTagName('i')[0]!);
+    });
+    expect(store.getState().settings.highlightFilter).toBe(true);
     await act(async () => {
       const textarea = screen.getByTestId('filter-panel').getElementsByTagName('textarea')[0];
       fireEvent.change(textarea, { target: { value: 'test' } });

frontend/static/__tests__/dtale/backgroundUtils-test.ts

@@ -35,36 +35,36 @@ describe('backgroundUtils tests', () => {
   });
 
   it('updateBackgroundStyles - heatmap-col', () => {
-    expect(bu.updateBackgroundStyles(colCfg, rec, state.settings)).toEqual({});
+    expect(bu.updateBackgroundStyles(colCfg, rec, {}, state.settings)).toEqual({});
     colCfg = { ...colCfg, min: 5, max: 10 };
     rec = { view: '7', raw: 7 };
-    const output: React.CSSProperties = bu.updateBackgroundStyles(colCfg, rec, state.settings);
+    const output: React.CSSProperties = bu.updateBackgroundStyles(colCfg, rec, {}, state.settings);
     expect(output.background).toBe('#ffcc00');
   });
 
   it('updateBackgroundStyles - dtypes', () => {
     state.settings.backgroundMode = 'dtypes';
     colCfg.dtype = 'bool';
-    let output: React.CSSProperties = bu.updateBackgroundStyles(colCfg, rec, state.settings);
+    let output: React.CSSProperties = bu.updateBackgroundStyles(colCfg, rec, {}, state.settings);
     expect(output.background).toBe('#FFF59D');
     colCfg.dtype = 'category';
-    output = bu.updateBackgroundStyles(colCfg, rec, state.settings);
+    output = bu.updateBackgroundStyles(colCfg, rec, {}, state.settings);
     expect(output.background).toBe('#E1BEE7');
     colCfg.dtype = 'timedelta';
-    output = bu.updateBackgroundStyles(colCfg, rec, state.settings);
+    output = bu.updateBackgroundStyles(colCfg, rec, {}, state.settings);
     expect(output.background).toBe('#FFCC80');
     colCfg.dtype = 'unknown';
-    output = bu.updateBackgroundStyles(colCfg, rec, state.settings);
+    output = bu.updateBackgroundStyles(colCfg, rec, {}, state.settings);
     expect(output).toEqual({});
   });
 
   it('updateBackgroundStyles - missing', () => {
     state.settings.backgroundMode = 'missing';
     colCfg = { ...colCfg, dtype: 'string', hasMissing: 1 };
-    let output: React.CSSProperties = bu.updateBackgroundStyles(colCfg, rec, state.settings);
+    let output: React.CSSProperties = bu.updateBackgroundStyles(colCfg, rec, {}, state.settings);
     expect(output.background).toBe('#FFCC80');
     rec.view = ' ';
-    output = bu.updateBackgroundStyles(colCfg, rec, state.settings);
+    output = bu.updateBackgroundStyles(colCfg, rec, {}, state.settings);
     expect(output.background).toBe('#FFCC80');
   });
 
@@ -73,13 +73,35 @@ describe('backgroundUtils tests', () => {
     colCfg = { ...colCfg, dtype: 'int', hasOutliers: 1, min: 1, max: 10, outlierRange: { lower: 3, upper: 5 } };
     colCfg = bu.buildOutlierScales(colCfg);
     rec.raw = 2;
-    let output: React.CSSProperties = bu.updateBackgroundStyles(colCfg, rec, state.settings);
+    let output: React.CSSProperties = bu.updateBackgroundStyles(colCfg, rec, {}, state.settings);
     expect(output.background).toBe('#8fc8ff');
     rec.raw = 6;
-    output = bu.updateBackgroundStyles(colCfg, rec, state.settings);
+    output = bu.updateBackgroundStyles(colCfg, rec, {}, state.settings);
     expect(output.background).toBe('#ffcccc');
     rec.raw = 4;
-    output = bu.updateBackgroundStyles(colCfg, rec, state.settings);
+    output = bu.updateBackgroundStyles(colCfg, rec, {}, state.settings);
+    expect(output).toEqual({});
+  });
+
+  it('updateBackgroundStyles - highlightFilter', () => {
+    state.settings.backgroundMode = 'outliers';
+    state.settings.highlightFilter = true;
+    colCfg = { ...colCfg, dtype: 'int', hasOutliers: 1, min: 1, max: 10, outlierRange: { lower: 3, upper: 5 } };
+    colCfg = bu.buildOutlierScales(colCfg);
+    rec.raw = 2;
+    let output: React.CSSProperties = bu.updateBackgroundStyles(
+      colCfg,
+      rec,
+      { __filtered: { view: 'true', raw: true } },
+      state.settings,
+    );
+    expect(output.background).toBe('#FFF59D');
+    rec.raw = 6;
+    output = bu.updateBackgroundStyles(colCfg, rec, {}, state.settings);
+    expect(output.background).toBe('#ffcccc');
+    rec.raw = 4;
+    state.settings.highlightFilter = false;
+    output = bu.updateBackgroundStyles(colCfg, rec, { __filtered: { view: 'true', raw: true } }, state.settings);
     expect(output).toEqual({});
   });
 });

frontend/static/__tests__/dtale/info/FilterDisplay-test.tsx

@@ -170,6 +170,14 @@ describe('FilterDisplay', () => {
     expect(updateSettingsSpy).toHaveBeenCalledWith({ invertFilter: true }, '1');
   });
 
+  it('highlight filter', async () => {
+    await buildMock();
+    await act(async () => {
+      await fireEvent.click(wrapper.querySelector('i.fa-highlighter')!);
+    });
+    expect(updateSettingsSpy).toHaveBeenCalledWith({ highlightFilter: true }, '1');
+  });
+
   describe('Queries', () => {
     let queries: Element;
 

frontend/static/__tests__/popups/filter/FilterPopup-test.tsx

@@ -30,6 +30,7 @@ describe('FilterPopup', () => {
       predefinedFilters: {},
       contextVars: [],
       invertFilter: false,
+      highlightFilter: false,
       success: true,
     });
     saveFilterSpy = jest.spyOn(CustomFilterRepository, 'save');
@@ -114,6 +115,14 @@ describe('FilterPopup', () => {
     expect(updateQueryEngineSpy.mock.calls[0][0]).toBe('numexpr');
   });
 
+  it('toggle highlight filter', async () => {
+    await act(async () => {
+      await fireEvent.click(screen.getByText('Highlight Filtered Rows').parentElement?.getElementsByTagName('i')[0]!);
+    });
+    expect(updateSettingsSpy).toHaveBeenLastCalledWith({ highlightFilter: true }, '1');
+    expect(store.getState().settings.highlightFilter).toBe(true);
+  });
+
   describe('new window', () => {
     const { location, close, opener } = window;
 
@@ -160,5 +169,13 @@ describe('FilterPopup', () => {
       expect(window.opener.location.reload).toHaveBeenCalledTimes(1);
       expect(window.close).toHaveBeenCalledTimes(1);
     });
+
+    it('toggle highlight filter', async () => {
+      await act(async () => {
+        await fireEvent.click(screen.getByText('Highlight Filtered Rows').parentElement?.getElementsByTagName('i')[0]!);
+      });
+      expect(updateSettingsSpy).toHaveBeenLastCalledWith({ highlightFilter: true }, '1');
+      expect(window.opener.location.reload).toHaveBeenCalledTimes(1);
+    });
   });
 });

frontend/static/dtale/DataViewer.tsx

@@ -142,7 +142,7 @@ export const DataViewer: React.FC = () => {
               (res2, col) => ({
                 ...res2,
                 [col]: gu.buildDataProps(
-                  response.columns.find(({ name }) => name === col)!,
+                  response.columns.find(({ name }) => name === col),
                   response.results[Number(rowIdx)][col],
                   settings,
                 ),