diff --git a/panel/models/tabulator.ts b/panel/models/tabulator.ts index 33ca68b32f..15153e41cc 100644 --- a/panel/models/tabulator.ts +++ b/panel/models/tabulator.ts @@ -15,6 +15,7 @@ import {debounce} from "debounce" import {comm_settings} from "./comm_manager" import {transform_cds_to_records} from "./data" import {HTMLBox, HTMLBoxView} from "./layout" +import {schedule_when} from "./util" export class TableEditEvent extends ModelEvent { constructor(readonly column: string, readonly row: number, readonly pre: boolean) { @@ -391,7 +392,7 @@ export class DataTabulatorView extends HTMLBoxView { this.tabulator.download(ftype, this.model.filename) }) - this.on_change(children, () => this.renderChildren()) + this.on_change(children, () => this.renderChildren(false)) this.on_change(expanded, () => { // The first cell is the cell of the frozen _index column. @@ -501,25 +502,27 @@ export class DataTabulatorView extends HTMLBoxView { } if (rows && (this.tabulator.rowManager.renderer != null)) { this.tabulator.rowManager.redraw(true) - this.renderChildren() this.setStyles() } this._redrawing = false this._restore_scroll = true } + get is_drawing(): boolean { + return this._building || this._redrawing || !this.root.has_finished() + } + override after_layout(): void { super.after_layout() - const finished = this.root.has_finished() - if (this.tabulator != null && this._initializing && !this._building && finished) { + if (this.tabulator != null && this._initializing && !this.is_drawing) { this._initializing = false - this.redraw() + this._resize_redraw() } } override after_resize(): void { super.after_resize() - if (!this._is_scrolling && !this._redrawing) { + if (!this._is_scrolling && !this._initializing && !this.is_drawing) { this._debounced_redraw() } } @@ -550,6 +553,7 @@ export class DataTabulatorView extends HTMLBoxView { } super.render() this._initializing = true + this._building = true const container = div({style: {display: "contents"}}) const el = div({style: {width: "100%", height: "100%", visibility: "hidden"}}) this.container = el @@ -602,7 +606,9 @@ export class DataTabulatorView extends HTMLBoxView { }, 50, false)) // Sync state with model - this.tabulator.on("rowSelectionChanged", (data: any, rows: any, selected: any, deselected: any) => this.rowSelectionChanged(data, rows, selected, deselected)) + this.tabulator.on("rowSelectionChanged", (data: any, rows: any, selected: any, deselected: any) => { + this.rowSelectionChanged(data, rows, selected, deselected) + }) this.tabulator.on("rowClick", (e: any, row: any) => this.rowClicked(e, row)) this.tabulator.on("cellEdited", (cell: any) => this.cellEdited(cell)) this.tabulator.on("dataFiltering", (filters: any) => { @@ -694,19 +700,13 @@ export class DataTabulatorView extends HTMLBoxView { this.tabulator.setPage(this.model.page) } this._building = false - if (this._initializing && this.root.has_finished()) { + schedule_when(() => { + const initializing = this._initializing this._initializing = false - this.redraw() - } else if (!this.root.has_finished()) { - const finalize = () => { - if (this.root.has_finished()) { - this._initializing = false - } else { - setTimeout(finalize, 10) - } + if (initializing) { + this._resize_redraw() } - setTimeout(finalize, 10) - } + }, () => this.root.has_finished()) } requestPage(page: number, sorters: any[]): Promise { @@ -772,21 +772,13 @@ export class DataTabulatorView extends HTMLBoxView { frozenRows: (row: any) => { return (this.model.frozen_rows.length > 0) ? this.model.frozen_rows.includes(row._row.data._index) : false }, + rowFormatter: (row: any) => this._render_row(row), } if (this.model.pagination === "remote") { configuration.ajaxURL = "http://panel.pyviz.org" configuration.sortMode = "remote" } - const cds: any = this.model.source - let data: any[] - if (cds === null || (cds.columns().length === 0)) { - data = [] - } else { - data = transform_cds_to_records(cds, true) - } - if (configuration.dataTree) { - data = group_data(data, this.model.columns, this.model.indexes, this.model.aggregators) - } + const data = this.getData() return { ...configuration, data, @@ -811,19 +803,27 @@ export class DataTabulatorView extends HTMLBoxView { return children } - renderChildren(): void { + get row_index(): Map { + const rows = this.tabulator.getRows() + const lookup = new Map() + for (const row of rows) { + const index = row._row?.data._index + if (index != null) { + lookup.set(index, row) + } + } + return lookup + } + + renderChildren(all: boolean = true): void { new Promise(async (resolve: any) => { - const new_children = await this.build_child_views() + let new_children = await this.build_child_views() + if (all) { + new_children = this.child_views + } resolve(new_children) }).then((new_children) => { - const rows = this.tabulator.getRows() - const lookup = new Map() - for (const row of rows) { - const index = row._row?.data._index - if (index != null) { - lookup.set(index, row) - } - } + const lookup = this.row_index for (const index of this.model.expanded) { const model = this.get_child(index) const row = lookup.get(index) @@ -859,14 +859,19 @@ export class DataTabulatorView extends HTMLBoxView { let viewEl if (prev_child != null && prev_child.className == "row-content") { viewEl = prev_child + if (viewEl.children.length && viewEl.children[0] === view.el) { + return + } } else { viewEl = div({class: "row-content", style: {background_color: bg, margin_left: neg_margin, max_width: "100%", overflow_x: "hidden"}}) + rowEl.appendChild(viewEl) } viewEl.appendChild(view.el) - rowEl.appendChild(viewEl) - if (!view.has_finished() && render) { - view.render() - view.after_render() + if (render) { + schedule_when(() => { + view.render() + view.after_render() + }, () => this.root.has_finished()) } if (resize) { this._update_children() @@ -914,7 +919,13 @@ export class DataTabulatorView extends HTMLBoxView { } getData(): any[] { - let data = transform_cds_to_records(this.model.source, true) + const cds = this.model.source + let data: any[] + if (cds === null || (cds.columns().length === 0)) { + data = [] + } else { + data = transform_cds_to_records(cds, true) + } if (this.model.configuration.dataTree) { data = group_data(data, this.model.columns, this.model.indexes, this.model.aggregators) } @@ -1261,7 +1272,6 @@ export class DataTabulatorView extends HTMLBoxView { setPage(): void { this.tabulator.setPage(Math.min(this.model.max_page, this.model.page)) if (this.model.pagination === "local") { - this.renderChildren() this.setStyles() } } @@ -1269,7 +1279,6 @@ export class DataTabulatorView extends HTMLBoxView { setPageSize(): void { this.tabulator.setPageSize(this.model.page_size) if (this.model.pagination === "local") { - this.renderChildren() this.setStyles() } } diff --git a/panel/models/util.ts b/panel/models/util.ts index 2697a747b2..50a70eba6e 100644 --- a/panel/models/util.ts +++ b/panel/models/util.ts @@ -147,3 +147,14 @@ export function find_attributes(text: string, obj: string, ignored: string[]) { return uniq(matches) } + +export function schedule_when(func: () => void, predicate: () => boolean, timeout: number = 10): void { + const scheduled = () => { + if (predicate()) { + func() + } else { + setTimeout(scheduled, timeout) + } + } + scheduled() +} diff --git a/panel/tests/conftest.py b/panel/tests/conftest.py index 3235a4ae96..0842e03aa3 100644 --- a/panel/tests/conftest.py +++ b/panel/tests/conftest.py @@ -144,7 +144,8 @@ def pytest_addoption(parser): for marker, info in optional_markers.items(): parser.addoption(f"--{marker}", action="store_true", default=False, help=info['help']) - + parser.addoption('--repeat', action='store', + help='Number of times to repeat each test') def pytest_configure(config): for marker, info in optional_markers.items(): @@ -155,6 +156,19 @@ def pytest_configure(config): config.addinivalue_line("markers", "internet: mark test as requiring an internet connection") +def pytest_generate_tests(metafunc): + if metafunc.config.option.repeat is not None: + count = int(metafunc.config.option.repeat) + + # We're going to duplicate these tests by parametrizing them, + # which requires that each test has a fixture to accept the parameter. + # We can add a new fixture like so: + metafunc.fixturenames.append('tmp_ct') + + # Now we parametrize. This is what happens when we do e.g., + # @pytest.mark.parametrize('tmp_ct', range(count)) + # def test_foo(): pass + metafunc.parametrize('tmp_ct', range(count)) def pytest_collection_modifyitems(config, items): skipped, selected = [], [] diff --git a/panel/tests/widgets/test_tables.py b/panel/tests/widgets/test_tables.py index 44d3baf3cd..661b7f382f 100644 --- a/panel/tests/widgets/test_tables.py +++ b/panel/tests/widgets/test_tables.py @@ -429,15 +429,14 @@ def test_tabulator_remote_sorted_paginated_expanded_content(document, comm): assert row0.text == "<pre>2.0</pre>" -@pytest.mark.parametrize('pagination', ['local', 'remote', None]) -def test_tabulator_filtered_expanded_content(document, comm, pagination): +def test_tabulator_filtered_expanded_content_remote_pagination(document, comm): df = makeMixedDataFrame() table = Tabulator( df, expanded=[0, 1, 2, 3], filters=[{'field': 'B', 'sorter': 'number', 'type': '=', 'value': '1.0'}], - pagination=pagination, + pagination='remote', row_content=lambda r: r.A, ) @@ -470,6 +469,59 @@ def test_tabulator_filtered_expanded_content(document, comm, pagination): assert row0.text == "<pre>0.0</pre>" +@pytest.mark.parametrize('pagination', ['local', None]) +def test_tabulator_filtered_expanded_content(document, comm, pagination): + df = makeMixedDataFrame() + + table = Tabulator( + df, + expanded=[0, 1, 2, 3], + filters=[{'field': 'B', 'sorter': 'number', 'type': '=', 'value': '1.0'}], + pagination=pagination, + row_content=lambda r: r.A, + ) + + model = table.get_root(document, comm) + + assert len(model.children) == 4 + + assert 0 in model.children + row0 = model.children[0] + assert row0.text == "<pre>0.0</pre>" + + assert 1 in model.children + row1 = model.children[1] + assert row1.text == "<pre>1.0</pre>" + + assert 2 in model.children + row2 = model.children[2] + assert row2.text == "<pre>2.0</pre>" + + assert 3 in model.children + row3 = model.children[3] + assert row3.text == "<pre>3.0</pre>" + + model.expanded = [1] + assert table.expanded == [1] + + table.filters = [{'field': 'B', 'sorter': 'number', 'type': '=', 'value': '0'}] + + assert model.expanded == [1] + assert table.expanded == [1] + + table.expanded = [0, 1] + + assert len(model.children) == 2 + + assert 0 in model.children + row0 = model.children[0] + assert row0.text == "<pre>0.0</pre>" + + assert 1 in model.children + row1 = model.children[1] + assert row1.text == "<pre>1.0</pre>" + + def test_tabulator_index_column(document, comm): df = pd.DataFrame({ 'int': [1, 2, 3], @@ -1428,11 +1480,15 @@ def test_tabulator_patch_with_filters(document, comm): dtype='datetime64[ns]') } expected_src = { - 'index': np.array([3]), - 'A': np.array([3]), - 'B': np.array([1]), - 'C': np.array(['foo4']), - 'D': np.array(['2009-01-06T00:00:00.000000000'], + 'index': np.array([0, 1, 2, 3, 4]), + 'A': np.array([2., 1., 2., 3., 1.]), + 'B': np.array([0., 1., 0., 1., 0.]), + 'C': np.array(['foo0', 'foo2', 'foo3', 'foo4', 'foo5']), + 'D': np.array(['2009-01-01T00:00:00.000000000', + '2009-01-02T00:00:00.000000000', + '2009-01-05T00:00:00.000000000', + '2009-01-06T00:00:00.000000000', + '2009-01-07T00:00:00.000000000'], dtype='datetime64[ns]').astype(np.int64) / 10e5 } for col, values in model.source.data.items(): @@ -1723,9 +1779,10 @@ def test_tabulator_stream_dataframe(document, comm): for col, values in model.source.data.items(): np.testing.assert_array_equal(values, expected[col]) -def test_tabulator_constant_scalar_filter_client_side(document, comm): +@pytest.mark.parametrize('pagination', ['local', 'remote', None]) +def test_tabulator_constant_scalar_filter_client_side(document, comm, pagination): df = makeMixedDataFrame() - table = Tabulator(df) + table = Tabulator(df, pagination=pagination) table.filters = [{'field': 'C', 'type': '=', 'value': 'foo3'}] @@ -1736,11 +1793,14 @@ def test_tabulator_constant_scalar_filter_client_side(document, comm): 'D': np.array(['2009-01-05T00:00:00.000000000'], dtype='datetime64[ns]') }, index=[2]) - pd.testing.assert_frame_equal(table._processed, expected) + pd.testing.assert_frame_equal( + table._processed, expected if pagination == 'remote' else df + ) -def test_tabulator_constant_scalar_filter_on_index_client_side(document, comm): +@pytest.mark.parametrize('pagination', ['local', 'remote', None]) +def test_tabulator_constant_scalar_filter_on_index_client_side(document, comm, pagination): df = makeMixedDataFrame() - table = Tabulator(df) + table = Tabulator(df, pagination=pagination) table.filters = [{'field': 'index', 'sorter': 'number', 'type': '=', 'value': 2}] @@ -1751,11 +1811,14 @@ def test_tabulator_constant_scalar_filter_on_index_client_side(document, comm): 'D': np.array(['2009-01-05T00:00:00.000000000'], dtype='datetime64[ns]') }, index=[2]) - pd.testing.assert_frame_equal(table._processed, expected) + pd.testing.assert_frame_equal( + table._processed, expected if pagination == 'remote' else df + ) -def test_tabulator_constant_scalar_filter_on_multi_index_client_side(document, comm): +@pytest.mark.parametrize('pagination', ['local', 'remote', None]) +def test_tabulator_constant_scalar_filter_on_multi_index_client_side(document, comm, pagination): df = makeMixedDataFrame() - table = Tabulator(df.set_index(['A', 'C'])) + table = Tabulator(df.set_index(['A', 'C']), pagination=pagination) table.filters = [ {'field': 'A', 'sorter': 'number', 'type': '=', 'value': 2}, @@ -1769,11 +1832,14 @@ def test_tabulator_constant_scalar_filter_on_multi_index_client_side(document, c 'D': np.array(['2009-01-05T00:00:00.000000000'], dtype='datetime64[ns]') }) - pd.testing.assert_frame_equal(table._processed, expected) + pd.testing.assert_frame_equal( + table._processed, expected if pagination == 'remote' else df[['A', 'C', 'B', 'D']] + ) -def test_tabulator_constant_list_filter_client_side(document, comm): +@pytest.mark.parametrize('pagination', ['local', 'remote', None]) +def test_tabulator_constant_list_filter_client_side(document, comm, pagination): df = makeMixedDataFrame() - table = Tabulator(df) + table = Tabulator(df, pagination=pagination) table.filters = [{'field': 'C', 'type': 'in', 'value': ['foo3', 'foo5']}] @@ -1785,11 +1851,14 @@ def test_tabulator_constant_list_filter_client_side(document, comm): '2009-01-07T00:00:00.000000000'], dtype='datetime64[ns]') }, index=[2, 4]) - pd.testing.assert_frame_equal(table._processed, expected) + pd.testing.assert_frame_equal( + table._processed, expected if pagination == 'remote' else df + ) -def test_tabulator_constant_single_element_list_filter_client_side(document, comm): +@pytest.mark.parametrize('pagination', ['local', 'remote', None]) +def test_tabulator_constant_single_element_list_filter_client_side(document, comm, pagination): df = makeMixedDataFrame() - table = Tabulator(df) + table = Tabulator(df, pagination=pagination) table.filters = [{'field': 'C', 'type': 'in', 'value': ['foo3']}] @@ -1800,11 +1869,14 @@ def test_tabulator_constant_single_element_list_filter_client_side(document, com 'D': np.array(['2009-01-05T00:00:00.000000000'], dtype='datetime64[ns]') }, index=[2]) - pd.testing.assert_frame_equal(table._processed, expected) + pd.testing.assert_frame_equal( + table._processed, expected if pagination == 'remote' else df + ) -def test_tabulator_keywords_filter_client_side(document, comm): +@pytest.mark.parametrize('pagination', ['local', 'remote', None]) +def test_tabulator_keywords_filter_client_side(document, comm, pagination): df = makeMixedDataFrame() - table = Tabulator(df) + table = Tabulator(df, pagination=pagination) table.filters = [{'field': 'C', 'type': 'keywords', 'value': 'foo3 foo5'}] @@ -1816,11 +1888,18 @@ def test_tabulator_keywords_filter_client_side(document, comm): '2009-01-07T00:00:00.000000000'], dtype='datetime64[ns]') }, index=[2, 4]) - pd.testing.assert_frame_equal(table._processed, expected) + pd.testing.assert_frame_equal( + table._processed, expected if pagination == 'remote' else df + ) -def test_tabulator_keywords_match_all_filter_client_side(document, comm): +@pytest.mark.parametrize('pagination', ['local', 'remote', None]) +def test_tabulator_keywords_match_all_filter_client_side(document, comm, pagination): df = makeMixedDataFrame() - table = Tabulator(df, header_filters={'C': {'type': 'input', 'func': 'keywords', 'matchAll': True}}) + table = Tabulator( + df, + header_filters={'C': {'type': 'input', 'func': 'keywords', 'matchAll': True}}, + pagination=pagination + ) table.filters = [{'field': 'C', 'type': 'keywords', 'value': 'f oo 3'}] @@ -1831,9 +1910,11 @@ def test_tabulator_keywords_match_all_filter_client_side(document, comm): 'D': np.array(['2009-01-05T00:00:00.000000000'], dtype='datetime64[ns]') }, index=[2]) - pd.testing.assert_frame_equal(table._processed, expected) + pd.testing.assert_frame_equal( + table._processed, expected if pagination == 'remote' else df + ) -def test_tabulator_constant_scalar_filter_with_pagination_client_side(document, comm): +def test_tabulator_constant_scalar_filter_client_side_with_pagination(document, comm): df = makeMixedDataFrame() table = Tabulator(df, pagination='remote') @@ -1852,7 +1933,7 @@ def test_tabulator_constant_scalar_filter_with_pagination_client_side(document, for col, values in model.source.data.items(): np.testing.assert_array_equal(values, expected[col]) -def test_tabulator_constant_scalar_filter_on_index_with_pagination_client_side(document, comm): +def test_tabulator_constant_scalar_filter_on_index_client_side_with_pagination(document, comm): df = makeMixedDataFrame() table = Tabulator(df, pagination='remote') @@ -1871,7 +1952,7 @@ def test_tabulator_constant_scalar_filter_on_index_with_pagination_client_side(d for col, values in model.source.data.items(): np.testing.assert_array_equal(values, expected[col]) -def test_tabulator_constant_scalar_filter_on_multi_index_with_pagination_client_side(document, comm): +def test_tabulator_constant_scalar_filter_on_multi_index_client_side_with_pagination(document, comm): df = makeMixedDataFrame() table = Tabulator(df.set_index(['A', 'C']), pagination='remote') @@ -1893,7 +1974,7 @@ def test_tabulator_constant_scalar_filter_on_multi_index_with_pagination_client_ for col, values in model.source.data.items(): np.testing.assert_array_equal(values, expected[col]) -def test_tabulator_constant_list_filter_with_pagination_client_side(document, comm): +def test_tabulator_constant_list_filter_client_side_with_pagination(document, comm): df = makeMixedDataFrame() table = Tabulator(df, pagination='remote') @@ -1913,8 +1994,7 @@ def test_tabulator_constant_list_filter_with_pagination_client_side(document, co for col, values in model.source.data.items(): np.testing.assert_array_equal(values, expected[col]) - -def test_tabulator_keywords_filter_with_pagination_client_side(document, comm): +def test_tabulator_keywords_filter_client_side_with_pagination(document, comm): df = makeMixedDataFrame() table = Tabulator(df, pagination='remote') @@ -1934,8 +2014,7 @@ def test_tabulator_keywords_filter_with_pagination_client_side(document, comm): for col, values in model.source.data.items(): np.testing.assert_array_equal(values, expected[col]) - -def test_tabulator_keywords_match_all_filter_with_pagination_client_side(document, comm): +def test_tabulator_keywords_match_all_filter_client_side_with_pagination(document, comm): df = makeMixedDataFrame() table = Tabulator( df, header_filters={'C': {'type': 'input', 'func': 'keywords', 'matchAll': True}}, diff --git a/panel/widgets/tables.py b/panel/widgets/tables.py index ea8b7b9af3..dad733ae6c 100644 --- a/panel/widgets/tables.py +++ b/panel/widgets/tables.py @@ -423,7 +423,7 @@ def tabulator_sorter(col): df_sorted.drop(columns=['_index_'], inplace=True) return df_sorted - def _filter_dataframe(self, df: pd.DataFrame) -> pd.DataFrame: + def _filter_dataframe(self, df: pd.DataFrame, header_filters: bool = True, internal_filters: bool = True) -> pd.DataFrame: """ Filter the DataFrame. @@ -438,7 +438,7 @@ def _filter_dataframe(self, df: pd.DataFrame) -> pd.DataFrame: The filtered DataFrame """ filters = [] - for col_name, filt in self._filters: + for col_name, filt in (self._filters if internal_filters else []): if col_name is not None and col_name not in df.columns: continue if isinstance(filt, (FunctionType, MethodType, partial)): @@ -477,7 +477,8 @@ def _filter_dataframe(self, df: pd.DataFrame) -> pd.DataFrame: "tuple or list.") filters.append(mask) - filters.extend(self._get_header_filters(df)) + if header_filters: + filters.extend(self._get_header_filters(df)) if filters: mask = filters[0] @@ -624,8 +625,13 @@ def _get_data(self) -> tuple[pd.DataFrame, DataDict]: return self._process_df_and_convert_to_cds(self.value) def _process_df_and_convert_to_cds(self, df: pd.DataFrame) -> tuple[pd.DataFrame, DataDict]: + # By default we potentially have two distinct views of the data + # locally we hold the fully filtered data, i.e. with header filters + # applied but since header filters are applied on the frontend + # we send the unfiltered data + import pandas as pd - df = self._filter_dataframe(df) + df = self._filter_dataframe(df, header_filters=False) if df is None: return [], {} if isinstance(self.value.index, pd.MultiIndex): @@ -1337,10 +1343,10 @@ def _process_event(self, event) -> None: self._validate_iloc(idx, iloc) event.row = iloc if event_col not in self.buttons: - if event_col not in self.value.columns: - event.value = self.value.index[event.row] - else: + if event_col in self.value.columns: event.value = self.value[event_col].iloc[event.row] + else: + event.value = self.value.index[event.row] # Set the old attribute on a table edit event if event.event_name == 'table-edit': @@ -1397,7 +1403,7 @@ def _process_data(self, data): import pandas as pd df = pd.DataFrame(data) - filters = self._get_header_filters(df) + filters = self._get_header_filters(df) if self.pagination == 'remote' else [] if filters: mask = filters[0] for f in filters: @@ -1414,6 +1420,9 @@ def _process_data(self, data): def _get_data(self): if self.pagination != 'remote' or self.value is None: return super()._get_data() + + # If data is paginated the current view on the frontend + # and locally are identical and both paginated import pandas as pd df = self._filter_dataframe(self.value) df = self._sort_df(df) @@ -1431,6 +1440,7 @@ def _get_data(self): indexes = [df.index.name or default_index] if len(indexes) > 1: page_df = page_df.reset_index() + df = df.reset_index() data = ColumnDataSource.from_df(page_df).items() return df, {k if isinstance(k, str) else str(k): v for k, v in data} @@ -2139,4 +2149,5 @@ def current_view(self) -> pd.DataFrame: df = self._processed if self.pagination == 'remote': return df + df = self._filter_dataframe(df, header_filters=True, internal_filters=False) return self._sort_df(df)