Merge branch 'main' into fix/use_require_files

crates/core/Cargo.toml

@@ -97,7 +97,7 @@ tracing = { workspace = true }
 rand = "0.8"
 z85 = "3.0.5"
 maplit = "1"
-sqlparser = { version = "0.50" }
+sqlparser = { version = "0.51" }
 
 # Unity
 reqwest = { version = "0.11.18", default-features = false, features = [

crates/core/src/operations/transaction/mod.rs

@@ -346,6 +346,12 @@ impl CommitProperties {
         self
     }
 
+    /// Specify maximum number of times to retry the transaction before failing to commit
+    pub fn with_max_retries(mut self, max_retries: usize) -> Self {
+        self.max_retries = max_retries;
+        self
+    }
+
     /// Specify if it should create a checkpoint when the commit interval condition is met
     pub fn with_create_checkpoint(mut self, create_checkpoint: bool) -> Self {
         self.create_checkpoint = create_checkpoint;

python/deltalake/__init__.py

@@ -10,6 +10,7 @@
 from .table import (
     ColumnProperties as ColumnProperties,
 )
+from .table import CommitProperties as CommitProperties
 from .table import DeltaTable as DeltaTable
 from .table import Metadata as Metadata
 from .table import PostCommitHookProperties as PostCommitHookProperties

python/deltalake/_internal.pyi

@@ -3,7 +3,12 @@ from typing import Any, Dict, List, Literal, Mapping, Optional, Tuple, Union
 import pyarrow
 import pyarrow.fs as fs
 
-from deltalake.writer import AddAction, PostCommitHookProperties, WriterProperties
+from deltalake.writer import (
+    AddAction,
+    CommitProperties,
+    PostCommitHookProperties,
+    WriterProperties,
+)
 
 __version__: str
 
@@ -58,7 +63,7 @@ class RawDeltaTable:
         dry_run: bool,
         retention_hours: Optional[int],
         enforce_retention_duration: bool,
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
         post_commithook_properties: Optional[PostCommitHookProperties],
     ) -> List[str]: ...
     def compact_optimize(
@@ -68,7 +73,7 @@ class RawDeltaTable:
         max_concurrent_tasks: Optional[int],
         min_commit_interval: Optional[int],
         writer_properties: Optional[WriterProperties],
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
         post_commithook_properties: Optional[PostCommitHookProperties],
     ) -> str: ...
     def z_order_optimize(
@@ -80,40 +85,40 @@ class RawDeltaTable:
         max_spill_size: Optional[int],
         min_commit_interval: Optional[int],
         writer_properties: Optional[WriterProperties],
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
         post_commithook_properties: Optional[PostCommitHookProperties],
     ) -> str: ...
     def add_columns(
         self,
         fields: List[Field],
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
         post_commithook_properties: Optional[PostCommitHookProperties],
     ) -> None: ...
     def add_constraints(
         self,
         constraints: Dict[str, str],
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
         post_commithook_properties: Optional[PostCommitHookProperties],
     ) -> None: ...
     def drop_constraints(
         self,
         name: str,
         raise_if_not_exists: bool,
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
         post_commithook_properties: Optional[PostCommitHookProperties],
     ) -> None: ...
     def set_table_properties(
         self,
         properties: Dict[str, str],
         raise_if_not_exists: bool,
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
     ) -> None: ...
     def restore(
         self,
         target: Optional[Any],
         ignore_missing_files: bool,
         protocol_downgrade_allowed: bool,
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
     ) -> str: ...
     def history(self, limit: Optional[int]) -> List[str]: ...
     def update_incremental(self) -> None: ...
@@ -126,13 +131,13 @@ class RawDeltaTable:
         self,
         predicate: Optional[str],
         writer_properties: Optional[WriterProperties],
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
         post_commithook_properties: Optional[PostCommitHookProperties],
     ) -> str: ...
     def repair(
         self,
         dry_run: bool,
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
         post_commithook_properties: Optional[PostCommitHookProperties],
     ) -> str: ...
     def update(
@@ -141,7 +146,7 @@ class RawDeltaTable:
         predicate: Optional[str],
         writer_properties: Optional[WriterProperties],
         safe_cast: bool,
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
         post_commithook_properties: Optional[PostCommitHookProperties],
     ) -> str: ...
     def create_merge_builder(
@@ -151,7 +156,7 @@ class RawDeltaTable:
         source_alias: Optional[str],
         target_alias: Optional[str],
         writer_properties: Optional[WriterProperties],
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
         post_commithook_properties: Optional[PostCommitHookProperties],
         safe_cast: bool,
     ) -> PyMergeBuilder: ...
@@ -166,7 +171,7 @@ class RawDeltaTable:
         partition_by: List[str],
         schema: pyarrow.Schema,
         partitions_filters: Optional[FilterType],
-        custom_metadata: Optional[Dict[str, str]],
+        commit_properties: Optional[CommitProperties],
         post_commithook_properties: Optional[PostCommitHookProperties],
     ) -> None: ...
     def cleanup_metadata(self) -> None: ...
@@ -207,7 +212,7 @@ def write_to_deltalake(
     configuration: Optional[Mapping[str, Optional[str]]],
     storage_options: Optional[Dict[str, str]],
     writer_properties: Optional[WriterProperties],
-    custom_metadata: Optional[Dict[str, str]],
+    commit_properties: Optional[CommitProperties],
     post_commithook_properties: Optional[PostCommitHookProperties],
 ) -> None: ...
 def convert_to_deltalake(