API: Timestamp(pydatetime) microsecond reso (#49034)

* API: Timedelta(td64_obj) retain resolution

* BUG: preserve DTA/TDA+timedeltalike scalar with mismatched resos

* BUG: DatetimeArray-datetimelike mixed resos

* API: Timestamp(pydatetime) microsecond reso

* use willayd suggestion

* ci fixup

* mypy fixup

* ignore pyright

* fix doctest

* un-xfail

* Merge main follow-up

* s reso for pydate

* typo fixup

* post-merge fixups

* suggestion json validation

* extra Py_DECREF

* requested refactor

* fix doctest

* unit keyword

* Update pandas/_libs/tslibs/conversion.pyx

Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com>

* dedicate pydate reso test

* fix failing resample test

Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com>

Author: jbrockmendel

pandas/_libs/lib.pyx

@@ -109,12 +109,16 @@ from pandas._libs.missing cimport (
     is_null_datetime64,
     is_null_timedelta64,
 )
-from pandas._libs.tslibs.conversion cimport convert_to_tsobject
+from pandas._libs.tslibs.conversion cimport (
+    _TSObject,
+    convert_to_tsobject,
+)
 from pandas._libs.tslibs.nattype cimport (
     NPY_NAT,
     c_NaT as NaT,
     checknull_with_nat,
 )
+from pandas._libs.tslibs.np_datetime cimport NPY_FR_ns
 from pandas._libs.tslibs.offsets cimport is_offset_object
 from pandas._libs.tslibs.period cimport is_period_object
 from pandas._libs.tslibs.timedeltas cimport convert_to_timedelta64
@@ -2378,6 +2382,7 @@ def maybe_convert_objects(ndarray[object] objects,
         ndarray[uint8_t] bools
         Seen seen = Seen()
         object val
+        _TSObject tsobj
         float64_t fnan = np.nan
 
     if dtype_if_all_nat is not None:
@@ -2470,7 +2475,8 @@ def maybe_convert_objects(ndarray[object] objects,
                 else:
                     seen.datetime_ = True
                     try:
-                        convert_to_tsobject(val, None, None, 0, 0)
+                        tsobj = convert_to_tsobject(val, None, None, 0, 0)
+                        tsobj.ensure_reso(NPY_FR_ns)
                     except OutOfBoundsDatetime:
                         seen.object_ = True
                         break

pandas/_libs/src/ujson/python/objToJSON.c

@@ -278,11 +278,42 @@ static int is_simple_frame(PyObject *obj) {
 }
 
 static npy_int64 get_long_attr(PyObject *o, const char *attr) {
+    // NB we are implicitly assuming that o is a Timedelta or Timestamp, or NaT
+
     npy_int64 long_val;
     PyObject *value = PyObject_GetAttrString(o, attr);
     long_val =
         (PyLong_Check(value) ? PyLong_AsLongLong(value) : PyLong_AsLong(value));
+
     Py_DECREF(value);
+
+    if (object_is_nat_type(o)) {
+        // i.e. o is NaT, long_val will be NPY_MIN_INT64
+        return long_val;
+    }
+
+    // ensure we are in nanoseconds, similar to Timestamp._as_creso or _as_unit
+    PyObject* reso = PyObject_GetAttrString(o, "_creso");
+    if (!PyLong_Check(reso)) {
+        // https://github.com/pandas-dev/pandas/pull/49034#discussion_r1023165139
+        Py_DECREF(reso);
+        return -1;
+    }
+
+    long cReso = PyLong_AsLong(reso);
+    Py_DECREF(reso);
+    if (cReso == -1 && PyErr_Occurred()) {
+        return -1;
+    }
+
+    if (cReso == NPY_FR_us) {
+        long_val = long_val * 1000L;
+    } else if (cReso == NPY_FR_ms) {
+        long_val = long_val * 1000000L;
+    } else if (cReso == NPY_FR_s) {
+        long_val = long_val * 1000000000L;
+    }
+
     return long_val;
 }
 
@@ -1265,6 +1296,7 @@ char **NpyArr_encodeLabels(PyArrayObject *labels, PyObjectEncoder *enc,
         } else if (PyDate_Check(item) || PyDelta_Check(item)) {
             is_datetimelike = 1;
             if (PyObject_HasAttrString(item, "value")) {
+                // see test_date_index_and_values for case with non-nano
                 nanosecVal = get_long_attr(item, "value");
             } else {
                 if (PyDelta_Check(item)) {

pandas/_libs/tslib.pyx

@@ -841,16 +841,19 @@ cdef _array_to_datetime_object(
 cdef inline bint _parse_today_now(str val, int64_t* iresult, bint utc):
     # We delay this check for as long as possible
     # because it catches relatively rare cases
+
+    # Multiply by 1000 to convert to nanos, since these methods naturally have
+    #  microsecond resolution
     if val == "now":
         if utc:
-            iresult[0] = Timestamp.utcnow().value
+            iresult[0] = Timestamp.utcnow().value * 1000
         else:
             # GH#18705 make sure to_datetime("now") matches Timestamp("now")
             # Note using Timestamp.now() is faster than Timestamp("now")
-            iresult[0] = Timestamp.now().value
+            iresult[0] = Timestamp.now().value * 1000
         return True
     elif val == "today":
-        iresult[0] = Timestamp.today().value
+        iresult[0] = Timestamp.today().value * 1000
         return True
     return False
 

pandas/_libs/tslibs/conversion.pyx

@@ -32,6 +32,7 @@ from pandas._libs.tslibs.dtypes cimport (
 from pandas._libs.tslibs.np_datetime cimport (
     NPY_DATETIMEUNIT,
     NPY_FR_ns,
+    NPY_FR_us,
     check_dts_bounds,
     convert_reso,
     get_datetime64_unit,
@@ -212,7 +213,12 @@ cdef class _TSObject:
 
     cdef int64_t ensure_reso(self, NPY_DATETIMEUNIT creso) except? -1:
         if self.creso != creso:
-            self.value = convert_reso(self.value, self.creso, creso, False)
+            try:
+                self.value = convert_reso(self.value, self.creso, creso, False)
+            except OverflowError as err:
+                raise OutOfBoundsDatetime from err
+
+            self.creso = creso
         return self.value
 
 
@@ -288,11 +294,22 @@ cdef _TSObject convert_to_tsobject(object ts, tzinfo tz, str unit,
             obj.value = ts
             pandas_datetime_to_datetimestruct(ts, NPY_FR_ns, &obj.dts)
     elif PyDateTime_Check(ts):
-        return convert_datetime_to_tsobject(ts, tz, nanos)
+        if nanos == 0:
+            if isinstance(ts, ABCTimestamp):
+                reso = abbrev_to_npy_unit(ts.unit)  # TODO: faster way to do this?
+            else:
+                # TODO: what if user explicitly passes nanos=0?
+                reso = NPY_FR_us
+        else:
+            reso = NPY_FR_ns
+        return convert_datetime_to_tsobject(ts, tz, nanos, reso=reso)
     elif PyDate_Check(ts):
         # Keep the converter same as PyDateTime's
+        # For date object we give the lowest supported resolution, i.e. "s"
         ts = datetime.combine(ts, time())
-        return convert_datetime_to_tsobject(ts, tz)
+        return convert_datetime_to_tsobject(
+            ts, tz, nanos=0, reso=NPY_DATETIMEUNIT.NPY_FR_s
+        )
     else:
         from .period import Period
         if isinstance(ts, Period):
@@ -346,6 +363,7 @@ cdef _TSObject convert_datetime_to_tsobject(
         _TSObject obj = _TSObject()
         int64_t pps
 
+    obj.creso = reso
     obj.fold = ts.fold
     if tz is not None:
         tz = maybe_get_tz(tz)

pandas/_libs/tslibs/offsets.pyx

@@ -162,7 +162,8 @@ def apply_wraps(func):
 
         result = func(self, other)
 
-        result = Timestamp(result)
+        result = (<_Timestamp>Timestamp(result))._as_creso(other._creso)
+
         if self._adjust_dst:
             result = result.tz_localize(tz)
 
@@ -175,9 +176,10 @@ def apply_wraps(func):
             if result.nanosecond != nano:
                 if result.tz is not None:
                     # convert to UTC
-                    value = result.tz_localize(None).value
+                    res = result.tz_localize(None)
                 else:
-                    value = result.value
+                    res = result
+                value = res.as_unit("ns").value
                 result = Timestamp(value + nano)
 
         if tz is not None and result.tzinfo is None:

pandas/_libs/tslibs/timestamps.pxd

@@ -33,4 +33,4 @@ cdef class _Timestamp(ABCTimestamp):
     cdef bint _compare_outside_nanorange(_Timestamp self, datetime other,
                                          int op) except -1
     cdef bint _compare_mismatched_resos(_Timestamp self, _Timestamp other, int op)
-    cdef _Timestamp _as_creso(_Timestamp self, NPY_DATETIMEUNIT reso, bint round_ok=*)
+    cdef _Timestamp _as_creso(_Timestamp self, NPY_DATETIMEUNIT creso, bint round_ok=*)

pandas/_libs/tslibs/timestamps.pyx

@@ -497,9 +497,9 @@ cdef class _Timestamp(ABCTimestamp):
             # Matching numpy, we cast to the higher resolution. Unlike numpy,
             #  we raise instead of silently overflowing during this casting.
             if self._creso < other._creso:
-                self = (<_Timestamp>self)._as_creso(other._creso, round_ok=False)
+                self = (<_Timestamp>self)._as_creso(other._creso, round_ok=True)
             elif self._creso > other._creso:
-                other = (<_Timestamp>other)._as_creso(self._creso, round_ok=False)
+                other = (<_Timestamp>other)._as_creso(self._creso, round_ok=True)
 
             # scalar Timestamp/datetime - Timestamp/datetime -> yields a
             # Timedelta
@@ -983,15 +983,22 @@ cdef class _Timestamp(ABCTimestamp):
     # Conversion Methods
 
     @cython.cdivision(False)
-    cdef _Timestamp _as_creso(self, NPY_DATETIMEUNIT reso, bint round_ok=True):
+    cdef _Timestamp _as_creso(self, NPY_DATETIMEUNIT creso, bint round_ok=True):
         cdef:
             int64_t value
 
-        if reso == self._creso:
+        if creso == self._creso:
             return self
 
-        value = convert_reso(self.value, self._creso, reso, round_ok=round_ok)
-        return type(self)._from_value_and_reso(value, reso=reso, tz=self.tzinfo)
+        try:
+            value = convert_reso(self.value, self._creso, creso, round_ok=round_ok)
+        except OverflowError as err:
+            unit = npy_unit_to_abbrev(creso)
+            raise OutOfBoundsDatetime(
+                f"Cannot cast {self} to unit='{unit}' without overflow."
+            ) from err
+
+        return type(self)._from_value_and_reso(value, reso=creso, tz=self.tzinfo)
 
     def as_unit(self, str unit, bint round_ok=True):
         """
@@ -1025,7 +1032,7 @@ cdef class _Timestamp(ABCTimestamp):
         --------
         >>> ts = pd.Timestamp(2020, 3, 14, 15)
         >>> ts.asm8
-        numpy.datetime64('2020-03-14T15:00:00.000000000')
+        numpy.datetime64('2020-03-14T15:00:00.000000')
         """
         return self.to_datetime64()
 

pandas/core/array_algos/take.py

@@ -360,7 +360,14 @@ def wrapper(
         if out_dtype is not None:
             out = out.view(out_dtype)
         if fill_wrap is not None:
+            # FIXME: if we get here with dt64/td64 we need to be sure we have
+            #  matching resos
+            if fill_value.dtype.kind == "m":
+                fill_value = fill_value.astype("m8[ns]")
+            else:
+                fill_value = fill_value.astype("M8[ns]")
             fill_value = fill_wrap(fill_value)
+
         f(arr, indexer, out, fill_value=fill_value)
 
     return wrapper

pandas/core/arrays/datetimes.py

@@ -445,7 +445,7 @@ def _generate_range(  # type: ignore[override]
                 i8values = generate_regular_range(start, end, periods, freq, unit=unit)
             else:
                 xdr = _generate_range(
-                    start=start, end=end, periods=periods, offset=freq
+                    start=start, end=end, periods=periods, offset=freq, unit=unit
                 )
                 i8values = np.array([x.value for x in xdr], dtype=np.int64)
 
@@ -508,7 +508,10 @@ def _unbox_scalar(self, value) -> np.datetime64:
         if not isinstance(value, self._scalar_type) and value is not NaT:
             raise ValueError("'value' should be a Timestamp.")
         self._check_compatible_with(value)
-        return value.asm8
+        if value is NaT:
+            return np.datetime64(value.value, self.unit)
+        else:
+            return value.as_unit(self.unit).asm8
 
     def _scalar_from_string(self, value) -> Timestamp | NaTType:
         return Timestamp(value, tz=self.tz)
@@ -2475,6 +2478,8 @@ def _generate_range(
     end: Timestamp | None,
     periods: int | None,
     offset: BaseOffset,
+    *,
+    unit: str,
 ):
     """
     Generates a sequence of dates corresponding to the specified time
@@ -2486,7 +2491,8 @@ def _generate_range(
     start : Timestamp or None
     end : Timestamp or None
     periods : int or None
-    offset : DateOffset,
+    offset : DateOffset
+    unit : str
 
     Notes
     -----
@@ -2506,13 +2512,20 @@ def _generate_range(
     start = Timestamp(start)  # type: ignore[arg-type]
     # Non-overlapping identity check (left operand type: "Timestamp", right
     # operand type: "NaTType")
-    start = start if start is not NaT else None  # type: ignore[comparison-overlap]
+    if start is not NaT:  # type: ignore[comparison-overlap]
+        start = start.as_unit(unit)
+    else:
+        start = None
+
     # Argument 1 to "Timestamp" has incompatible type "Optional[Timestamp]";
     # expected "Union[integer[Any], float, str, date, datetime64]"
     end = Timestamp(end)  # type: ignore[arg-type]
     # Non-overlapping identity check (left operand type: "Timestamp", right
     # operand type: "NaTType")
-    end = end if end is not NaT else None  # type: ignore[comparison-overlap]
+    if end is not NaT:  # type: ignore[comparison-overlap]
+        end = end.as_unit(unit)
+    else:
+        end = None
 
     if start and not offset.is_on_offset(start):
         # Incompatible types in assignment (expression has type "datetime",
@@ -2553,7 +2566,7 @@ def _generate_range(
                 break
 
             # faster than cur + offset
-            next_date = offset._apply(cur)
+            next_date = offset._apply(cur).as_unit(unit)
             if next_date <= cur:
                 raise ValueError(f"Offset {offset} did not increment date")
             cur = next_date
@@ -2567,7 +2580,7 @@ def _generate_range(
                 break
 
             # faster than cur + offset
-            next_date = offset._apply(cur)
+            next_date = offset._apply(cur).as_unit(unit)
             if next_date >= cur:
                 raise ValueError(f"Offset {offset} did not decrement date")
             cur = next_date

pandas/core/computation/pytables.py

@@ -11,6 +11,7 @@
 import numpy as np
 
 from pandas._libs.tslibs import (
+    NaT,
     Timedelta,
     Timestamp,
 )
@@ -216,6 +217,8 @@ def stringify(value):
                 v = stringify(v)
             v = ensure_decoded(v)
             v = Timestamp(v)
+            if v is not NaT:
+                v = v.as_unit("ns")  # pyright: ignore[reportGeneralTypeIssues]
             if v.tz is not None:
                 v = v.tz_convert("UTC")
             return TermValue(v, v.value, kind)

pandas/core/dtypes/cast.py

@@ -754,16 +754,21 @@ def infer_dtype_from_scalar(val, pandas_dtype: bool = False) -> tuple[DtypeObj,
     elif isinstance(val, (np.datetime64, dt.datetime)):
         try:
             val = Timestamp(val)
+            # error: Non-overlapping identity check (left operand type:
+            # "Timestamp", right operand type: "NaTType")
+            if val is not NaT:  # type: ignore[comparison-overlap]
+                val = val.as_unit("ns")
         except OutOfBoundsDatetime:
             return _dtype_obj, val
 
         # error: Non-overlapping identity check (left operand type: "Timestamp",
         # right operand type: "NaTType")
         if val is NaT or val.tz is None:  # type: ignore[comparison-overlap]
-            dtype = np.dtype("M8[ns]")
             val = val.to_datetime64()
+            dtype = val.dtype
             # TODO(2.0): this should be dtype = val.dtype
             #  to get the correct M8 resolution
+            # TODO: test with datetime(2920, 10, 1) based on test_replace_dtypes
         else:
             if pandas_dtype:
                 dtype = DatetimeTZDtype(unit="ns", tz=val.tz)

pandas/core/indexes/datetimes.py

@@ -9,6 +9,7 @@
 import warnings
 
 import numpy as np
+import pytz
 
 from pandas._libs import (
     NaT,
@@ -578,7 +579,7 @@ def get_loc(self, key, method=None, tolerance=None):
 
             try:
                 parsed, reso = self._parse_with_reso(key)
-            except ValueError as err:
+            except (ValueError, pytz.NonExistentTimeError) as err:
                 raise KeyError(key) from err
             self._disallow_mismatched_indexing(parsed)