Tabulator: Fix editing a table that contains a timezone-aware datetim…

…e column (#6879)

panel/reactive.py

@@ -1221,9 +1221,27 @@ def _convert_column(
  dtype = old_values.dtype
  converted: list | np.ndarray | None = None
  if dtype.kind == 'M':
- if values.dtype.kind in 'if':# TODO: need to doublecheck if this is still needed
- NATs = np.isnan(values)
- converted = np.where(NATs, np.nan, values * 10e5).astype(dtype)
+ if values.dtype.kind in 'if':
+ if getattr(dtype, 'tz', None):
+ # dtype has a timezone
+ if dtype.tz == dt.timezone.utc:
+ # Milliseconds to nanoseconds, to datetime64.
+ converted = (values * 1e6).astype('datetime64[ns]')
+ else:
+ import pandas as pd
+
+ # Using pandas to convert from milliseconds
+ # timezone-aware, to UTC nanoseconds, to datetime64.
+ converted = (
+ pd.Series(pd.to_datetime(values, unit="ms"))
+ .dt.tz_localize(dtype.tz)
+ .dt.tz_convert('utc')
+ .dt.tz_localize(None)
+ )
+ else:
+ # Timestamps converted from milliseconds to nanoseconds,
+ # to datetime.
+ converted = (values * 1e6).astype(dtype)
  elif dtype.kind == 'O':
  if (all(isinstance(ov, dt.date) for ov in old_values) and
  not all(isinstance(iv, dt.date) for iv in values)):

panel/tests/widgets/test_tables.py

@@ -1,6 +1,8 @@
 import asyncio
 import datetime as dt
 
+from zoneinfo import ZoneInfo
+
 import numpy as np
 import pandas as pd
 import pytest
@@ -17,7 +19,6 @@
 from panel.io.state import set_curdoc
 from panel.models.tabulator import CellClickEvent, TableEditEvent
 from panel.tests.util import mpl_available, serve_and_request, wait_until
-from panel.util import BOKEH_JS_NAT
 from panel.widgets import Button, TextInput
 from panel.widgets.tables import DataFrame, Tabulator
 
@@ -1355,16 +1356,16 @@ def test_tabulator_patch_with_NaT(document, comm):
  table.patch({'A': [(0, pd.NaT)]})
 
  # We're also checking that the NaT value that was in the original table
- # at .loc[1, 'A'] is converted in the model as BOKEH_JS_NAT.
+ # at .loc[1, 'A'] is converted in the model as np.nan.
  expected = {
  'index': np.array([0, 1]),
- 'A': np.array([BOKEH_JS_NAT, BOKEH_JS_NAT])
+ 'A': np.array([np.nan, np.nan])
  }
  for col, values in model.source.data.items():
  expected_array = expected[col]
  np.testing.assert_array_equal(values, expected_array)
  # Not checking that the data in table.value is the same as expected
- # In table.value we have NaT values, in expected the BOKEH_JS_NAT constant.
+ # In table.value we have NaT values, in expected np.nan.
 
 
 def test_tabulator_stream_series_paginated_not_follow(document, comm):
@@ -2001,6 +2002,40 @@ def test_server_edit_event():
  assert events[0].value == 3.14
  assert events[0].old == 1
 
+
+def test_edit_with_datetime_aware_column():
+ # https://github.com/holoviz/panel/issues/6673
+
+ # The order of these columns matter, 'B' and 'C' should be first as it's in fact
+ # processed first when 'A' is edited.
+ data = {
+ "B": pd.date_range(start='2024-01-01', end='2024-01-03', freq='D', tz='utc'),
+ "C": pd.date_range(start='2024-01-01', end='2024-01-03', freq='D', tz=ZoneInfo('US/Eastern')),
+ "A": ['a', 'b', 'c'],
+ }
+ df = pd.DataFrame(data)
+
+ table = Tabulator(df)
+
+ serve_and_request(table)
+
+ wait_until(lambda: bool(table._models))
+ ref, (model, _) = list(table._models.items())[0]
+ doc = list(table._documents.keys())[0]
+
+ events = []
+ table.on_edit(lambda e: events.append(e))
+
+ new_data = dict(model.source.data)
+ new_data['A'][1] = 'new'
+
+ table._server_change(doc, ref, None, 'data', model.source.data, new_data)
+ table._server_event(doc, TableEditEvent(model, 'A', 1))
+
+ wait_until(lambda: len(events) == 1)
+ assert events[0].value == 'new'
+ assert events[0].old == 'b'
+
 def test_tabulator_cell_click_event():
  df = makeMixedDataFrame()
  table = Tabulator(df)

panel/util/__init__.py

@@ -44,7 +44,6 @@
 
 bokeh_version = Version(Version(bokeh.__version__).base_version)
 
-BOKEH_JS_NAT = np.nan
 PARAM_NAME_PATTERN = re.compile(r'^.*\d{5}$')
 
 class LazyHTMLSanitizer:

panel/widgets/tables.py

@@ -28,7 +28,7 @@
 from ..io.state import state
 from ..reactive import Reactive, ReactiveData
 from ..util import (
- BOKEH_JS_NAT, clone_model, datetime_as_utctimestamp, isdatetime, lazy_load,
+ clone_model, datetime_as_utctimestamp, isdatetime, lazy_load,
  styler_update, updating,
 )
 from ..util.warnings import warn
@@ -847,7 +847,7 @@ def patch(self, patch_value, as_index=True):
  if isinstance(value, pd.Timestamp):
  value = datetime_as_utctimestamp(value)
  elif value is pd.NaT:
- value = BOKEH_JS_NAT
+ value = np.nan
  values.append((patch_ind, value))
  patches[k] = values
  self._patch(patches)