Skip to content

Commit

Permalink
check_can_write_timestamp_ntz in python
Browse files Browse the repository at this point in the history
  • Loading branch information
ion-elgreco committed Apr 23, 2024
1 parent efe7ec9 commit a53464d
Show file tree
Hide file tree
Showing 4 changed files with 34 additions and 1 deletion.
1 change: 1 addition & 0 deletions python/deltalake/_internal.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ class RawDeltaTable:
custom_metadata: Optional[Dict[str, str]],
) -> None: ...
def cleanup_metadata(self) -> None: ...
def check_can_write_timestamp_ntz(self, schema: pyarrow.Schema) -> None: ...

def rust_core_version() -> str: ...
def write_new_deltalake(
Expand Down
1 change: 1 addition & 0 deletions python/deltalake/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -429,6 +429,7 @@ def visitor(written_file: Any) -> None:
# We don't currently provide a way to set invariants
# (and maybe never will), so only enforce if already exist.
table_protocol = table.protocol()
table._table.check_can_write_timestamp_ntz(schema)
if (
table_protocol.min_writer_version > MAX_SUPPORTED_PYARROW_WRITER_VERSION
or table_protocol.min_writer_version
Expand Down
13 changes: 12 additions & 1 deletion python/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ use deltalake::operations::filesystem_check::FileSystemCheckBuilder;
use deltalake::operations::merge::MergeBuilder;
use deltalake::operations::optimize::{OptimizeBuilder, OptimizeType};
use deltalake::operations::restore::RestoreBuilder;
use deltalake::operations::transaction::{CommitBuilder, CommitProperties};
use deltalake::operations::transaction::{CommitBuilder, CommitProperties, PROTOCOL};
use deltalake::operations::update::UpdateBuilder;
use deltalake::operations::vacuum::VacuumBuilder;
use deltalake::parquet::basic::Compression;
Expand Down Expand Up @@ -175,6 +175,17 @@ impl RawDeltaTable {
))
}

pub fn check_can_write_timestamp_ntz(&self, schema: PyArrowType<ArrowSchema>) -> PyResult<()> {
let schema: StructType = (&schema.0).try_into().map_err(PythonError::from)?;
Ok(PROTOCOL
.check_can_write_timestamp_ntz(
self._table.snapshot().map_err(PythonError::from)?,
&schema,
)
.map_err(|e| DeltaTableError::Generic(e.to_string()))
.map_err(PythonError::from)?)
}

pub fn load_version(&mut self, version: i64) -> PyResult<()> {
Ok(rt()
.block_on(self._table.load_version(version))
Expand Down
20 changes: 20 additions & 0 deletions python/tests/test_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1550,3 +1550,23 @@ def test_write_timestamp_ntz_nested(tmp_path: pathlib.Path, array: pa.array):
assert protocol.min_writer_version == 7
assert protocol.reader_features == ["timestampNtz"]
assert protocol.writer_features == ["timestampNtz"]


def test_write_timestamp_ntz_on_table_with_features_not_enabled(tmp_path: pathlib.Path):
data = pa.table({"x": pa.array(["foo"])})
write_deltalake(tmp_path, data, mode="append", engine="pyarrow")

dt = DeltaTable(tmp_path)

protocol = dt.protocol()
assert protocol.min_reader_version == 1
assert protocol.min_writer_version == 2

data = pa.table({"x": pa.array([datetime(2010, 1, 1)])})
with pytest.raises(
DeltaError,
match="Generic DeltaTable error: Writer features must be specified for writerversion >= 7, please specify: TimestampWithoutTimezone",
):
write_deltalake(
tmp_path, data, mode="overwrite", engine="pyarrow", schema_mode="overwrite"
)

0 comments on commit a53464d

Please sign in to comment.