Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add _format_str mapping to interchange protocol #62

Closed
wants to merge 5 commits into from
Closed
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion protocol/pandas_implementation.py
Original file line number Diff line number Diff line change
Expand Up @@ -435,10 +435,32 @@ def _dtype_from_pandasdtype(self, dtype) -> Tuple[enum.IntEnum, int, str, str]:
raise NotImplementedError(f"Data type {dtype} not handled yet")

bitwidth = dtype.itemsize * 8
format_str = dtype.str
format_str = self._format_str(dtype.str)
endianness = dtype.byteorder if not kind == _k.CATEGORICAL else '='
return (kind, bitwidth, format_str, endianness)

def _format_str(self, format_str) -> str:
"""
Mapping of NumPy formt strings to
kgryte marked this conversation as resolved.
Show resolved Hide resolved
Apache Arrow C Data Interface format strings.
'O' categorical mapped as 'U': large utf-8 string for now
"""
_ints = {8: 'c', 16: 's', 32: 'i', 64: 'l'}
_uints = {8: 'C', 16: 'S', 32: 'I', 64: 'L'}
_floats = {16: 'e', 32: 'f', 64: 'g'}
_np_dtypes = {'i': _ints, 'u': _uints, 'f': _floats, 'b': {8: 'b'}, 'O': {64: 'U'}}

dt = np.dtype(format_str)
if dt.byteorder == '>':
raise ValueError(f"Big-endian not supported by exchange"
"protocol")

arrow_format_str = _np_dtypes.get(dt.kind, {}).get(dt.itemsize*8)

if arrow_format_str is None:
raise NotImplementedError(f"Format string {format_str} not handled yet")

return arrow_format_str

@property
def describe_categorical(self) -> Dict[str, Any]:
Expand Down Expand Up @@ -880,6 +902,12 @@ def test_metadata():
assert_dataframe_equal(df.__dataframe__(), df)
tm.assert_frame_equal(df, df2)

def test_fromat_str():
df = pd.DataFrame(data=dict(a=[1, 2, 3], B=[3, 4, 5],
c=[1.5, 2.5, 3.5], D=["a", "b", "cdef"]))
df["B"] = df["B"].astype("category")
df["D"] = df["D"].astype("object")
df2 = from_dataframe(df)

if __name__ == '__main__':
test_categorical_dtype()
Expand Down