Python write_deltalake fails if pyarrow table contains binary columns (…

…#1167)

# Description
Python write_deltalake fails if pyarrow table contains binary columns
ending with 0x5c

# Related Issue(s)
Python write_deltalake fails if pyarrow table contains binary columns
ending with 0x5c #1146

---------

Co-authored-by: rbushrian <rbushrian@akamai.com>
Co-authored-by: Will Jones <willjones127@gmail.com>

python/deltalake/writer.py

@@ -333,7 +333,7 @@ def __enforce_append_only(
 class DeltaJSONEncoder(json.JSONEncoder):
     def default(self, obj: Any) -> Any:
         if isinstance(obj, bytes):
-            return obj.decode("unicode_escape")
+            return obj.decode("unicode_escape", "backslashreplace")
         elif isinstance(obj, date):
             return obj.isoformat()
         elif isinstance(obj, datetime):

python/tests/test_writer.py

@@ -725,3 +725,13 @@ def test_partition_overwrite_with_wrong_partition(
             mode="overwrite",
             partition_filters=[("p999", "=", "1")],
         )
+
+
+def test_handles_binary_data(tmp_path: pathlib.Path):
+    value = b"\x00\\"
+    table = pa.Table.from_pydict({"field_one": [value]})
+    write_deltalake(tmp_path, table)
+
+    dt = DeltaTable(tmp_path)
+    out = dt.to_pyarrow_table()
+    assert table == out