Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enh: Adds support for Polars Time datatype #2113

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
1 change: 1 addition & 0 deletions docs/api-reference/dtypes.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
- Object
- Unknown
- UnsignedIntegerType
- Time
show_root_heading: false
show_source: false
show_bases: false
2 changes: 2 additions & 0 deletions narwhals/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
from narwhals.dtypes import Object
from narwhals.dtypes import String
from narwhals.dtypes import Struct
from narwhals.dtypes import Time
from narwhals.dtypes import UInt8
from narwhals.dtypes import UInt16
from narwhals.dtypes import UInt32
Expand Down Expand Up @@ -109,6 +110,7 @@
"Series",
"String",
"Struct",
"Time",
"UInt8",
"UInt16",
"UInt32",
Expand Down
6 changes: 6 additions & 0 deletions narwhals/_arrow/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,10 @@ def native_to_narwhals_dtype(dtype: pa.DataType, version: Version) -> DType:
)
if pa.types.is_decimal(dtype):
return dtypes.Decimal()
if pa.types.is_time32(dtype):
return dtypes.Time()
if pa.types.is_time64(dtype):
return dtypes.Time()
return dtypes.Unknown() # pragma: no cover


Expand Down Expand Up @@ -203,6 +207,8 @@ def narwhals_to_native_dtype(dtype: DType | type[DType], version: Version) -> pa
inner = narwhals_to_native_dtype(dtype.inner, version=version)
list_size = dtype.size
return pa.list_(inner, list_size=list_size)
if isinstance_or_issubclass(dtype, dtypes.Time):
return pa.time64("ns")

msg = f"Unknown dtype: {dtype}" # pragma: no cover
raise AssertionError(msg)
Expand Down
3 changes: 3 additions & 0 deletions narwhals/_dask/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,9 @@ def narwhals_to_native_dtype(dtype: DType | type[DType], version: Version) -> An
if isinstance_or_issubclass(dtype, dtypes.Array): # pragma: no cover
msg = "Converting to Array dtype is not supported yet"
return NotImplementedError(msg)
if isinstance_or_issubclass(dtype, dtypes.Time): # pragma: no cover
msg = "Converting to Time dtype is not supported yet"
return NotImplementedError(msg)

msg = f"Unknown dtype: {dtype}" # pragma: no cover
raise AssertionError(msg)
Expand Down
4 changes: 4 additions & 0 deletions narwhals/_duckdb/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ def native_to_narwhals_dtype(duckdb_dtype: str, version: Version) -> DType:
)
if duckdb_dtype.startswith("DECIMAL("):
return dtypes.Decimal()
if duckdb_dtype == "TIME":
return dtypes.Time()
return dtypes.Unknown() # pragma: no cover


Expand Down Expand Up @@ -146,6 +148,8 @@ def narwhals_to_native_dtype(dtype: DType | type[DType], version: Version) -> st
return "VARCHAR"
if isinstance_or_issubclass(dtype, dtypes.Boolean): # pragma: no cover
return "BOOLEAN"
if isinstance_or_issubclass(dtype, dtypes.Time):
return "TIME"
if isinstance_or_issubclass(dtype, dtypes.Categorical):
msg = "Categorical not supported by DuckDB"
raise NotImplementedError(msg)
Expand Down
2 changes: 2 additions & 0 deletions narwhals/_ibis/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ def native_to_narwhals_dtype(ibis_dtype: Any, version: Version) -> DType:
if ibis_dtype.is_decimal(): # pragma: no cover
# TODO(unassigned): cover this
return dtypes.Decimal()
if ibis_dtype.is_time():
return dtypes.Time()
return dtypes.Unknown() # pragma: no cover


Expand Down
6 changes: 5 additions & 1 deletion narwhals/_pandas_like/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,8 @@ def non_object_native_to_narwhals_dtype(dtype: str, version: Version) -> DType:
return dtypes.Date()
if dtype.startswith("decimal") and dtype.endswith("[pyarrow]"):
return dtypes.Decimal()
if dtype.startswith("time") and dtype.endswith("[pyarrow]"):
return dtypes.Time()
Comment on lines +415 to +416
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pattern is time<32|64>[<unit>][pyarrow]

Example:

from datetime import time
import pandas as pd
import pyarrow as pa

data = {"a": [time(12, 0, 0), time(12, 0, 5)]}

pd.DataFrame(data).convert_dtypes(dtype_backend="pyarrow").astype(pd.ArrowDtype(pa.time64("ns"))).dtypes
a    time64[ns][pyarrow]
dtype: object

return dtypes.Unknown() # pragma: no cover


Expand Down Expand Up @@ -610,7 +612,9 @@ def narwhals_to_native_dtype( # noqa: PLR0915
if isinstance_or_issubclass(dtype, dtypes.Enum):
msg = "Converting to Enum is not (yet) supported"
raise NotImplementedError(msg)
if isinstance_or_issubclass(dtype, (dtypes.Struct, dtypes.Array, dtypes.List)):
if isinstance_or_issubclass(
dtype, (dtypes.Struct, dtypes.Array, dtypes.List, dtypes.Time)
):
if implementation is Implementation.PANDAS and backend_version >= (2, 2):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unrelated, but... should we check modin backed by pyarrow?

try:
import pandas as pd
Expand Down
4 changes: 4 additions & 0 deletions narwhals/_polars/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ def native_to_narwhals_dtype(
)
if dtype == pl.Decimal:
return dtypes.Decimal()
if dtype == pl.Time:
return dtypes.Time()
return dtypes.Unknown()


Expand Down Expand Up @@ -188,6 +190,8 @@ def narwhals_to_native_dtype(
raise NotImplementedError(msg)
if dtype == dtypes.Date:
return pl.Date()
if dtype == dtypes.Time:
return pl.Time()
if dtype == dtypes.Decimal:
msg = "Casting to Decimal is not supported yet."
raise NotImplementedError(msg)
Expand Down
3 changes: 2 additions & 1 deletion narwhals/_spark_like/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,9 +142,10 @@ def narwhals_to_native_dtype(
dtypes.UInt8,
dtypes.Enum,
dtypes.Categorical,
dtypes.Time,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't find a datatype in pyspark for this, so it will just end up in the unsupported list

),
): # pragma: no cover
msg = "Unsigned integer, Enum and Categorical types are not supported by spark-like backend"
msg = "Unsigned integer, Enum, Categorical and Time types are not supported by spark-like backend"
raise UnsupportedDTypeError(msg)

msg = f"Unknown dtype: {dtype}" # pragma: no cover
Expand Down
25 changes: 25 additions & 0 deletions narwhals/dtypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -671,3 +671,28 @@ class Date(TemporalType):
>>> nw.from_native(s_native, series_only=True).dtype
Date
"""


class Time(TemporalType):
"""Data type representing the time of day.

Examples:
>>> import polars as pl
>>> import pyarrow as pa
>>> import narwhals as nw
>>> import duckdb
>>> from datetime import time
>>> data = [time(9, 0), time(9, 1, 10), time(9, 2)]
>>> ser_pl = pl.Series(data)
>>> ser_pa = pa.chunked_array([pa.array(data, type=pa.time64("ns"))])
>>> rel = duckdb.sql(
... " SELECT * FROM (VALUES (TIME '12:00:00'), (TIME '14:30:15')) df(t)"
... )

>>> nw.from_native(ser_pl, series_only=True).dtype
Time
>>> nw.from_native(ser_pa, series_only=True).dtype
Time
>>> nw.from_native(rel).schema["t"]
Time
"""
2 changes: 2 additions & 0 deletions narwhals/stable/v1/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
from narwhals.stable.v1.dtypes import Object
from narwhals.stable.v1.dtypes import String
from narwhals.stable.v1.dtypes import Struct
from narwhals.stable.v1.dtypes import Time
from narwhals.stable.v1.dtypes import UInt8
from narwhals.stable.v1.dtypes import UInt16
from narwhals.stable.v1.dtypes import UInt32
Expand Down Expand Up @@ -2415,6 +2416,7 @@ def scan_parquet(
"Series",
"String",
"Struct",
"Time",
"UInt8",
"UInt16",
"UInt32",
Expand Down
2 changes: 2 additions & 0 deletions narwhals/stable/v1/_dtypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
from narwhals.dtypes import SignedIntegerType
from narwhals.dtypes import String
from narwhals.dtypes import Struct
from narwhals.dtypes import Time
from narwhals.dtypes import UInt8
from narwhals.dtypes import UInt16
from narwhals.dtypes import UInt32
Expand Down Expand Up @@ -97,6 +98,7 @@ def __hash__(self: Self) -> int:
"SignedIntegerType",
"String",
"Struct",
"Time",
"UInt8",
"UInt16",
"UInt32",
Expand Down
2 changes: 2 additions & 0 deletions narwhals/stable/v1/dtypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from narwhals.stable.v1._dtypes import SignedIntegerType
from narwhals.stable.v1._dtypes import String
from narwhals.stable.v1._dtypes import Struct
from narwhals.stable.v1._dtypes import Time
from narwhals.stable.v1._dtypes import UInt8
from narwhals.stable.v1._dtypes import UInt16
from narwhals.stable.v1._dtypes import UInt32
Expand Down Expand Up @@ -61,6 +62,7 @@
"SignedIntegerType",
"String",
"Struct",
"Time",
"UInt8",
"UInt16",
"UInt32",
Expand Down
1 change: 1 addition & 0 deletions narwhals/typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,7 @@ class DTypes:
List: type[dtypes.List]
Array: type[dtypes.Array]
Unknown: type[dtypes.Unknown]
Time: type[dtypes.Time]


__all__ = [
Expand Down
3 changes: 2 additions & 1 deletion tests/dtypes_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ def test_dtype_is_x() -> None:
nw.Object,
nw.String,
nw.Struct,
nw.Time,
nw.UInt8,
nw.UInt16,
nw.UInt32,
Expand All @@ -283,7 +284,7 @@ def test_dtype_is_x() -> None:
is_unsigned_integer = {nw.UInt8, nw.UInt16, nw.UInt32, nw.UInt64, nw.UInt128}
is_float = {nw.Float32, nw.Float64}
is_decimal = {nw.Decimal}
is_temporal = {nw.Datetime, nw.Date, nw.Duration}
is_temporal = {nw.Datetime, nw.Date, nw.Duration, nw.Time}
is_nested = {nw.Array, nw.List, nw.Struct}

for dtype in dtypes:
Expand Down
16 changes: 16 additions & 0 deletions tests/expr_and_series/cast_test.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

from datetime import datetime
from datetime import time
from datetime import timedelta
from datetime import timezone
from typing import Any
Expand Down Expand Up @@ -285,3 +286,18 @@ def test_raise_if_polars_dtype(constructor: Constructor, dtype: Any) -> None:
df = nw.from_native(constructor({"a": [1, 2, 3], "b": [4, 5, 6]}))
with pytest.raises(TypeError, match="Expected Narwhals dtype, got:"):
df.select(nw.col("a").cast(dtype))


def test_cast_time(request: pytest.FixtureRequest, constructor: Constructor) -> None:
if "pandas" in str(constructor) and PANDAS_VERSION < (2, 2):
request.applymarker(pytest.mark.xfail)

if any(backend in str(constructor) for backend in ("dask", "pyspark", "modin")):
request.applymarker(pytest.mark.xfail)

data = {"a": [time(12, 0, 0), time(12, 0, 5)]}
df_native = nw.from_native(constructor(data))
result = df_native.select(nw.col("a").cast(nw.Time))
assert result.collect_schema() == {"a": nw.Time}
result_native = nw.to_native(result)
assert isinstance(result_native, type(constructor(data)))
Loading