Skip to content

Commit

Permalink
Change to lazy loading of io in pandas serializer
Browse files Browse the repository at this point in the history
  • Loading branch information
bolkedebruin committed Sep 29, 2023
1 parent 55fbdfe commit b774d31
Showing 1 changed file with 2 additions and 1 deletion.
3 changes: 2 additions & 1 deletion airflow/serialization/serializers/pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
# under the License.
from __future__ import annotations

from io import BytesIO
from typing import TYPE_CHECKING

from airflow.utils.module_loading import qualname
Expand Down Expand Up @@ -63,6 +62,8 @@ def deserialize(classname: str, version: int, data: object) -> pd.DataFrame:
if not isinstance(data, str):
raise TypeError(f"serialized {classname} has wrong data type {type(data)}")

from io import BytesIO

with BytesIO(bytes.fromhex(data)) as buf:
df = pq.read_table(buf).to_pandas()

Expand Down

0 comments on commit b774d31

Please sign in to comment.