diff --git a/Cargo.lock b/Cargo.lock index 6098aa8b52..5e6d4e4a56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -607,7 +607,7 @@ dependencies = [ [[package]] name = "deltalake" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "arrow 6.0.0-SNAPSHOT", @@ -620,7 +620,7 @@ dependencies = [ "clap", "crossbeam", "datafusion", - "env_logger 0.9.0", + "env_logger", "errno", "futures", "glibc_version 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -651,11 +651,11 @@ dependencies = [ [[package]] name = "deltalake-python" -version = "0.5.0" +version = "0.5.2" dependencies = [ "arrow 6.0.0-SNAPSHOT", "deltalake", - "env_logger 0.9.0", + "env_logger", "pyo3", "reqwest", "serde_json", @@ -753,20 +753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", - "humantime 1.3.0", - "log", - "regex", - "termcolor", -] - -[[package]] -name = "env_logger" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" -dependencies = [ - "atty", - "humantime 2.1.0", + "humantime", "log", "regex", "termcolor", @@ -1007,17 +994,6 @@ dependencies = [ "wasi 0.10.2+wasi-snapshot-preview1", ] -[[package]] -name = "ghost" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5bcf1bbeab73aa4cf2fde60a846858dc036163c7c33bec309f8d17de785479" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "gimli" version = "0.25.0" @@ -1145,12 +1121,6 @@ dependencies = [ "quick-error", ] -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - [[package]] name = "hyper" version = "0.14.11" @@ -1264,28 +1234,6 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f" -[[package]] -name = "inventory" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0f7efb804ec95e33db9ad49e4252f049e37e8b0a4652e3cd61f7999f2eff7f" -dependencies = [ - "ctor", - "ghost", - "inventory-impl", -] - -[[package]] -name = "inventory-impl" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75c094e94816723ab936484666968f5b58060492e880f3c8d00489a1e244fa51" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "ipnet" version = "2.3.1" @@ -1899,7 +1847,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" dependencies = [ - "env_logger 0.7.1", + "env_logger", "log", ] @@ -1964,26 +1912,34 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.13.2" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4837b8e8e18a102c23f79d1e9a110b597ea3b684c95e874eb1ad88f8683109c3" +checksum = "af205762ba65eec9f27a2fa1a57a40644e8e3368784b8c8b2f2de48f6e8ddd96" dependencies = [ "cfg-if", - "ctor", "indoc", - "inventory", "libc", "parking_lot", "paste 0.1.18", + "pyo3-build-config", "pyo3-macros", "unindent", ] +[[package]] +name = "pyo3-build-config" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "755944027ce803c7238e59c5a18e59c1d0a4553db50b23e9ba209a568353028d" +dependencies = [ + "once_cell", +] + [[package]] name = "pyo3-macros" -version = "0.13.2" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47f2c300ceec3e58064fd5f8f5b61230f2ffd64bde4970c81fdd0563a2db1bb" +checksum = "cd31b36bccfd902c78804bd96c28ea93eac6fa0ca311f9d21ef2230b6665b29a" dependencies = [ "pyo3-macros-backend", "quote", @@ -1992,11 +1948,12 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.13.2" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87b097e5d84fcbe3e167f400fbedd657820a375b034c78bd852050749a575d66" +checksum = "c21c59ba36db9c823e931c662766b0dd01a030b1d96585b67d8857a96a56b972" dependencies = [ "proc-macro2", + "pyo3-build-config", "quote", "syn", ] diff --git a/python/Cargo.toml b/python/Cargo.toml index 296de33431..dba1819294 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,12 +1,13 @@ [package] name = "deltalake-python" -version = "0.5.0" +version = "0.5.2" authors = ["Qingping Hou "] homepage = "https://github.com/delta-io/delta-rs" license = "Apache-2.0" -description = "Python binding for delta-rs" +description = "Native Delta Lake Python binding based on delta-rs with Pandas integration" readme = "README.md" edition = "2018" +keywords = ["deltalake", "delta", "datalake", "pandas", "arrow"] [lib] name = "deltalake" @@ -22,7 +23,7 @@ serde_json = "1" arrow = { git = "https://github.com/apache/arrow-rs", rev = "fa5acd971c973161f17e69d5c6b50d6e77c7da03" } [dependencies.pyo3] -version = "0.13" +version = "0.14" features = ["extension-module", "abi3", "abi3-py36"] [dependencies.deltalake] @@ -38,13 +39,13 @@ classifier = [ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only" ] -project-url = { Repo = "https://github.com/delta-io/delta-rs" } +project-url = { Repo = "https://github.com/delta-io/delta-rs", Documentation = "https://delta-io.github.io/delta-rs/python/", "Bug Tracker" = "https://github.com/delta-io/delta-rs/issues" } requires-dist = [ "pyarrow>=4", 'numpy<1.20.0;python_version<="3.6"', 'dataclasses;python_version<="3.6"', 'types-dataclasses;python_version<="3.6" and extra == "pandas"', - "pandas; extra == 'pandas'", + "pandas; extra == 'pandas' or extra == 'devel'", "mypy; extra == 'devel'", "isort; extra == 'devel'", "pytest; extra == 'devel'", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 65a140d282..20a718cb04 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltalake" -version = "0.4.0" +version = "0.4.1" authors = ["Qingping Hou "] homepage = "https://github.com/delta-io/delta.rs" license = "Apache-2.0" diff --git a/rust/src/delta_arrow.rs b/rust/src/delta_arrow.rs index 8533cc6f65..644745dafc 100644 --- a/rust/src/delta_arrow.rs +++ b/rust/src/delta_arrow.rs @@ -109,7 +109,7 @@ impl TryFrom<&schema::SchemaDataType> for ArrowDataType { } "date" => { // A calendar date, represented as a year-month-day triple without a - // timezone. + // timezone. Stored as 4 bytes integer representing days sinece 1970-01-01 Ok(ArrowDataType::Date32) } "timestamp" => { @@ -326,7 +326,7 @@ pub(crate) fn delta_log_schema_for_table( .fields() .iter() .map(|f| f.to_owned()) - .partition(|field| partition_columns.contains(&field.name())); + .partition(|field| partition_columns.contains(field.name())); let mut stats_parsed_fields: Vec = vec![ArrowField::new("numRecords", ArrowDataType::Int64, true)]; diff --git a/rust/tests/checkpoint_writer_test.rs b/rust/tests/checkpoint_writer_test.rs index a38e3f29d7..653d47072d 100644 --- a/rust/tests/checkpoint_writer_test.rs +++ b/rust/tests/checkpoint_writer_test.rs @@ -34,11 +34,6 @@ async fn write_simple_checkpoint() { let checkpoint_path = log_path.join("00000000000000000005.checkpoint.parquet"); assert!(checkpoint_path.as_path().exists()); - // HACK: seems like a race condition exists reading the file back in. - // Without the sleep, frequently fails with: - // Error("EOF while parsing a value", line: 1, column: 0)' - std::thread::sleep(std::time::Duration::from_secs(1)); - // _last_checkpoint should exist and point to the correct version let version = get_last_checkpoint_version(&log_path); assert_eq!(5, version); @@ -54,9 +49,6 @@ async fn write_simple_checkpoint() { let checkpoint_path = log_path.join("00000000000000000010.checkpoint.parquet"); assert!(checkpoint_path.as_path().exists()); - // see above - std::thread::sleep(std::time::Duration::from_secs(1)); - // _last_checkpoint should exist and point to the correct version let version = get_last_checkpoint_version(&log_path); assert_eq!(10, version);