Skip to content

Add automatic fallback to coarser datetime units during inference mode #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 5 commits into
base: api-to_datetime-units
Choose a base branch
from
Draft

Add automatic fallback to coarser datetime units during inference mode #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a9380f3
Initial plan
Copilot Oct 17, 2025
7094cf2
Add automatic fallback to coarser units for OutOfBoundsDatetime
Copilot Oct 17, 2025
3c2d9ab
Fix fallback logic and add comprehensive tests
Copilot Oct 17, 2025
f091f9c
Fix inaccurate comment about unit_for_numerics
Copilot Oct 17, 2025
84d5d92
Refactor fallback logic to only apply during inference mode
Copilot Oct 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 54 additions & 2 deletions pandas/_libs/tslib.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,22 @@ from pandas._libs.missing cimport checknull_with_nat_and_na
from pandas._libs.tslibs.tzconversion cimport tz_localize_to_utc_single


cdef NPY_DATETIMEUNIT get_next_coarser_unit(NPY_DATETIMEUNIT creso):
"""
Get the next coarser unit in the sequence: ns -> us -> ms -> s
Returns NPY_FR_GENERIC if there is no coarser unit available.
"""
if creso == NPY_DATETIMEUNIT.NPY_FR_ns:
return NPY_DATETIMEUNIT.NPY_FR_us
elif creso == NPY_DATETIMEUNIT.NPY_FR_us:
return NPY_DATETIMEUNIT.NPY_FR_ms
elif creso == NPY_DATETIMEUNIT.NPY_FR_ms:
return NPY_DATETIMEUNIT.NPY_FR_s
else:
# No coarser unit available
return NPY_DATETIMEUNIT.NPY_FR_GENERIC


def _test_parse_iso8601(ts: str):
"""
TESTING ONLY: Parse string into Timestamp using iso8601 parser. Used
Expand Down Expand Up @@ -352,14 +368,50 @@ cpdef array_to_datetime(
if infer_reso:
creso = state.creso
tz_out = state.process_datetime(val, tz_out, utc_convert)
iresult[i] = parse_pydatetime(val, &dts, creso=creso)
try:
iresult[i] = parse_pydatetime(val, &dts, creso=creso)
except OverflowError:
if infer_reso:
# During inference, try falling back to coarser unit
next_creso = get_next_coarser_unit(creso)
if next_creso != NPY_DATETIMEUNIT.NPY_FR_GENERIC:
# Retry with coarser unit
return array_to_datetime(
values,
errors=errors,
dayfirst=dayfirst,
yearfirst=yearfirst,
utc=utc,
creso=next_creso,
unit_for_numerics=unit_for_numerics,
)
# Either not in inference mode or no coarser unit available
raise

elif PyDate_Check(val):
item_reso = NPY_DATETIMEUNIT.NPY_FR_s
state.update_creso(item_reso)
if infer_reso:
creso = state.creso
iresult[i] = pydate_to_dt64(val, &dts, reso=creso)
try:
iresult[i] = pydate_to_dt64(val, &dts, reso=creso)
except OverflowError:
if infer_reso:
# During inference, try falling back to coarser unit
next_creso = get_next_coarser_unit(creso)
if next_creso != NPY_DATETIMEUNIT.NPY_FR_GENERIC:
# Retry with coarser unit
return array_to_datetime(
values,
errors=errors,
dayfirst=dayfirst,
yearfirst=yearfirst,
utc=utc,
creso=next_creso,
unit_for_numerics=unit_for_numerics,
)
# Either not in inference mode or no coarser unit available
raise
state.found_other = True

elif cnp.is_datetime64_object(val):
Expand Down
44 changes: 44 additions & 0 deletions pandas/_libs/tslibs/strptime.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,22 @@ from pandas._libs.tslibs.tzconversion cimport tz_localize_to_utc_single
cnp.import_array()


cdef NPY_DATETIMEUNIT get_next_coarser_unit(NPY_DATETIMEUNIT creso):
"""
Get the next coarser unit in the sequence: ns -> us -> ms -> s
Returns NPY_FR_GENERIC if there is no coarser unit available.
"""
if creso == NPY_DATETIMEUNIT.NPY_FR_ns:
return NPY_DATETIMEUNIT.NPY_FR_us
elif creso == NPY_DATETIMEUNIT.NPY_FR_us:
return NPY_DATETIMEUNIT.NPY_FR_ms
elif creso == NPY_DATETIMEUNIT.NPY_FR_ms:
return NPY_DATETIMEUNIT.NPY_FR_s
else:
# No coarser unit available
return NPY_DATETIMEUNIT.NPY_FR_GENERIC


cdef bint format_is_iso(f: str):
"""
Does format match the iso8601 set that can be handled by the C parser?
Expand Down Expand Up @@ -472,6 +488,20 @@ def array_strptime(
try:
value = npy_datetimestruct_to_datetime(creso, &dts)
except OverflowError as err:
if infer_reso:
# During inference, try falling back to coarser unit
next_creso = get_next_coarser_unit(creso)
if next_creso != NPY_DATETIMEUNIT.NPY_FR_GENERIC:
# Retry with coarser unit
return array_strptime(
values,
fmt=fmt,
exact=exact,
errors=errors,
utc=utc,
creso=next_creso,
)
# Either not in inference mode or no coarser unit available
attrname = npy_unit_to_attrname[creso]
raise OutOfBoundsDatetime(
f"Out of bounds {attrname} timestamp: {val}"
Expand Down Expand Up @@ -517,6 +547,20 @@ def array_strptime(
try:
iresult[i] = npy_datetimestruct_to_datetime(creso, &dts)
except OverflowError as err:
if infer_reso:
# During inference, try falling back to coarser unit
next_creso = get_next_coarser_unit(creso)
if next_creso != NPY_DATETIMEUNIT.NPY_FR_GENERIC:
# Retry with coarser unit
return array_strptime(
values,
fmt=fmt,
exact=exact,
errors=errors,
utc=utc,
creso=next_creso,
)
# Either not in inference mode or no coarser unit available
attrname = npy_unit_to_attrname[creso]
raise OutOfBoundsDatetime(
f"Out of bounds {attrname} timestamp: {val}"
Expand Down