diff --git a/Cargo.lock b/Cargo.lock index 1038d850f6e03..f2c5bf6fa2836 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -284,9 +284,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" +checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38" dependencies = [ "arrow-arith", "arrow-array", @@ -305,9 +305,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" +checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043" dependencies = [ "arrow-array", "arrow-buffer", @@ -320,16 +320,16 @@ dependencies = [ [[package]] name = "arrow-array" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" +checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97" dependencies = [ "ahash 0.8.10", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "chrono-tz", + "chrono-tz 0.9.0", "half", "hashbrown 0.14.3", "num", @@ -337,9 +337,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" +checksum = "fe9b95e825ae838efaf77e366c00d3fc8cca78134c9db497d6bda425f2e7b7c1" dependencies = [ "bytes", "half", @@ -348,9 +348,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" +checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab" dependencies = [ "arrow-array", "arrow-buffer", @@ -369,9 +369,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" +checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc" dependencies = [ "arrow-array", "arrow-buffer", @@ -388,9 +388,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" +checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10" dependencies = [ "arrow-buffer", "arrow-schema", @@ -400,9 +400,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3241ce691192d789b7b94f56a10e166ee608bdc3932c759eb0b85f09235352bb" +checksum = "cdd624aafd1f34710a1d6ed44ea0e9b06f7b75adc4277c53bac4a2d23229030b" dependencies = [ "arrow-arith", "arrow-array", @@ -440,9 +440,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" +checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -455,9 +455,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" +checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -475,9 +475,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" +checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413" dependencies = [ "arrow-array", "arrow-buffer", @@ -490,9 +490,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" +checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e" dependencies = [ "ahash 0.8.10", "arrow-array", @@ -505,18 +505,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" +checksum = "32aae6a60458a2389c0da89c9de0b7932427776127da1a738e2efc21d32f3393" dependencies = [ "serde", ] [[package]] name = "arrow-select" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" +checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102" dependencies = [ "ahash 0.8.10", "arrow-array", @@ -528,9 +528,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" +checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597" dependencies = [ "arrow-array", "arrow-buffer", @@ -664,7 +664,7 @@ name = "async-compression" version = "0.4.3" source = "git+https://github.com/everpcpc/async-compression?rev=dc81082#dc8108229e3a0288ee9097c986b89a90788fe9f6" dependencies = [ - "brotli", + "brotli 3.4.0", "bzip2", "deflate64", "flate2", @@ -683,7 +683,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ - "brotli", + "brotli 3.4.0", "bzip2", "flate2", "futures-core", @@ -1703,7 +1703,18 @@ checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", - "brotli-decompressor", + "brotli-decompressor 2.5.1", +] + +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 4.0.1", ] [[package]] @@ -1716,6 +1727,16 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bstr" version = "0.2.17" @@ -2113,11 +2134,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", - "chrono-tz-build", + "chrono-tz-build 0.2.1", "phf", "serde", ] +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build 0.3.0", + "phf", +] + [[package]] name = "chrono-tz-build" version = "0.2.1" @@ -2129,6 +2161,17 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "chumsky" version = "0.9.3" @@ -3006,9 +3049,8 @@ dependencies = [ [[package]] name = "databend-client" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5517338bec352f24119594d32c23e71553e005fc0fb1e9df994b85c4f4238645" +version = "0.18.3" +source = "git+https://github.com/datafuselabs/bendsql/?rev=7658a5#7658a593b557d90516e44398bf4ec477d3fb500e" dependencies = [ "async-trait", "log", @@ -3051,7 +3093,7 @@ dependencies = [ "byteorder", "bytes", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "databend-common-parquet2", "dyn-clone", "either", @@ -3070,7 +3112,7 @@ dependencies = [ "num", "num-traits", "opendal", - "ordered-float 3.9.2", + "ordered-float 4.2.0", "proptest", "quanta 0.11.1", "rand 0.8.5", @@ -3264,7 +3306,7 @@ name = "databend-common-compress" version = "0.1.0" dependencies = [ "async-compression 0.4.3", - "brotli", + "brotli 3.4.0", "bytes", "databend-common-exception", "env_logger", @@ -3280,7 +3322,7 @@ dependencies = [ name = "databend-common-config" version = "0.1.0" dependencies = [ - "chrono-tz", + "chrono-tz 0.8.6", "clap", "databend-common-base", "databend-common-building", @@ -3348,7 +3390,7 @@ dependencies = [ "borsh", "bumpalo", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "comfy-table 6.2.0", "dashmap", "databend-common-arrow", @@ -3400,7 +3442,7 @@ dependencies = [ "async-trait-fn", "base64 0.21.7", "bstr 1.9.1", - "chrono-tz", + "chrono-tz 0.8.6", "databend-common-arrow", "databend-common-exception", "databend-common-expression", @@ -3435,7 +3477,7 @@ dependencies = [ "bstr 1.9.1", "bumpalo", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "comfy-table 6.2.0", "crc32fast", "criterion", @@ -3540,7 +3582,7 @@ dependencies = [ "borsh", "bytes", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "databend-common-exception", "ethnum", "geo", @@ -3634,7 +3676,7 @@ dependencies = [ "anyerror", "anyhow", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "cron", "databend-common-ast", "databend-common-building", @@ -3893,7 +3935,7 @@ name = "databend-common-parquet2" version = "0.1.0" dependencies = [ "async-stream", - "brotli", + "brotli 3.4.0", "bytes", "criterion", "flate2", @@ -4004,7 +4046,7 @@ version = "0.1.0" dependencies = [ "anyhow", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "convert_case 0.6.0", "databend-common-expression", "databend-common-io", @@ -4052,7 +4094,7 @@ name = "databend-common-settings" version = "0.1.0" dependencies = [ "async-backtrace", - "chrono-tz", + "chrono-tz 0.8.6", "dashmap", "databend-common-ast", "databend-common-base", @@ -4105,7 +4147,7 @@ dependencies = [ "async-recursion", "async-trait-fn", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "cidr", "cron", "ctor", @@ -4195,7 +4237,6 @@ dependencies = [ "arrow-schema", "async-backtrace", "async-trait-fn", - "bytes", "databend-common-base", "databend-common-catalog", "databend-common-exception", @@ -4206,13 +4247,10 @@ dependencies = [ "databend-common-storages-parquet", "databend-storages-common-table-meta", "deltalake", - "flagset", - "futures", "maplit", "match-template", "minitrace", - "object_store", - "opendal", + "object_store_opendal", "ordered-float 4.2.0", "parquet", "serde", @@ -4619,7 +4657,7 @@ dependencies = [ "async-backtrace", "async-trait-fn", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "databend-common-ast", "databend-common-base", "databend-common-building", @@ -4734,9 +4772,8 @@ dependencies = [ [[package]] name = "databend-driver" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc41daaaff7b8235046c0b7ae1e29c0e5136cd3fbae57cb432f36fb0255671a" +version = "0.18.3" +source = "git+https://github.com/datafuselabs/bendsql/?rev=7658a5#7658a593b557d90516e44398bf4ec477d3fb500e" dependencies = [ "arrow", "async-compression 0.4.6", @@ -4759,9 +4796,8 @@ dependencies = [ [[package]] name = "databend-driver-macros" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad3df58cb5e1d7ffc9c2add629b8a0bc2aff6b872c92309dbf7c2355d58acada" +version = "0.18.3" +source = "git+https://github.com/datafuselabs/bendsql/?rev=7658a5#7658a593b557d90516e44398bf4ec477d3fb500e" dependencies = [ "quote", "syn 2.0.52", @@ -4855,7 +4891,7 @@ dependencies = [ "aws-config", "aws-sdk-s3", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "dashmap", "databend-common-ast", "databend-common-base", @@ -5071,7 +5107,7 @@ dependencies = [ "byte-unit", "byteorder", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "config", "criterion", "ctor", @@ -5220,9 +5256,8 @@ dependencies = [ [[package]] name = "databend-sql" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a09f0357ee782df7d50ef39d4e0a2f044fcc474b517717ca87e55fd3aca14d" +version = "0.18.3" +source = "git+https://github.com/datafuselabs/bendsql/?rev=7658a5#7658a593b557d90516e44398bf4ec477d3fb500e" dependencies = [ "arrow", "chrono", @@ -5266,7 +5301,7 @@ dependencies = [ name = "databend-sqlsmith" version = "0.1.0" dependencies = [ - "chrono-tz", + "chrono-tz 0.8.6", "clap", "databend-client", "databend-common-ast", @@ -5429,9 +5464,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "37.1.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85069782056753459dc47e386219aa1fdac5b731f26c28abb8c0ffd4b7c5ab11" +checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068" dependencies = [ "ahash 0.8.10", "arrow", @@ -5449,9 +5484,11 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-functions", + "datafusion-functions-aggregate", "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-sql", "flate2", @@ -5466,9 +5503,10 @@ dependencies = [ "object_store", "parking_lot 0.12.1", "parquet", + "paste", "pin-project-lite", "rand 0.8.5", - "sqlparser 0.44.0", + "sqlparser 0.47.0", "tempfile", "tokio", "tokio-util", @@ -5480,9 +5518,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "37.1.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "309d9040751f6dc9e33c85dce6abb55a46ef7ea3644577dd014611c379447ef3" +checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971" dependencies = [ "ahash 0.8.10", "arrow", @@ -5491,28 +5529,29 @@ dependencies = [ "arrow-schema", "chrono", "half", + "hashbrown 0.14.3", "instant", "libc", "num_cpus", "object_store", "parquet", - "sqlparser 0.44.0", + "sqlparser 0.47.0", ] [[package]] name = "datafusion-common-runtime" -version = "37.1.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e4a44d8ef1b1e85d32234e6012364c411c3787859bb3bba893b0332cb03dfd" +checksum = "d0091318129dad1359f08e4c6c71f855163c35bba05d1dbf983196f727857894" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "37.1.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06a3a29ae36bcde07d179cc33b45656a8e7e4d023623e320e48dcf1200eeee95" +checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1" dependencies = [ "arrow", "chrono", @@ -5531,26 +5570,28 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "37.1.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3542aa322029c2121a671ce08000d4b274171070df13f697b14169ccf4f628" +checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3" dependencies = [ "ahash 0.8.10", "arrow", "arrow-array", + "arrow-buffer", "chrono", "datafusion-common", "paste", - "sqlparser 0.44.0", + "serde_json", + "sqlparser 0.47.0", "strum 0.26.2", "strum_macros 0.26.2", ] [[package]] name = "datafusion-functions" -version = "37.1.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd221792c666eac174ecc09e606312844772acc12cbec61a420c2fca1ee70959" +checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b" dependencies = [ "arrow", "base64 0.22.0", @@ -5561,21 +5602,41 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", + "hashbrown 0.14.3", "hex", "itertools 0.12.1", "log", "md-5", + "rand 0.8.5", "regex", "sha2", "unicode-segmentation", "uuid", ] +[[package]] +name = "datafusion-functions-aggregate" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78" +dependencies = [ + "ahash 0.8.10", + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "log", + "paste", + "sqlparser 0.47.0", +] + [[package]] name = "datafusion-functions-array" -version = "37.1.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e501801e84d9c6ef54caaebcda1b18a6196a24176c12fb70e969bc0572e03c55" +checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a" dependencies = [ "arrow", "arrow-array", @@ -5593,9 +5654,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "37.1.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bd7f5087817deb961764e8c973d243b54f8572db414a8f0a8f33a48f991e0a" +checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76" dependencies = [ "arrow", "async-trait", @@ -5604,6 +5665,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.3", + "indexmap 2.2.5", "itertools 0.12.1", "log", "regex-syntax 0.8.2", @@ -5611,9 +5673,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "37.1.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cabc0d9aaa0f5eb1b472112f16223c9ffd2fb04e58cbf65c0a331ee6e993f96" +checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e" dependencies = [ "ahash 0.8.10", "arrow", @@ -5623,37 +5685,46 @@ dependencies = [ "arrow-schema", "arrow-string", "base64 0.22.0", - "blake2", - "blake3", "chrono", "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr-common", "half", "hashbrown 0.14.3", "hex", "indexmap 2.2.5", "itertools 0.12.1", "log", - "md-5", "paste", "petgraph", - "rand 0.8.5", "regex", - "sha2", - "unicode-segmentation", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "rand 0.8.5", ] [[package]] name = "datafusion-physical-plan" -version = "37.1.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17c0523e9c8880f2492a88bbd857dde02bed1ed23f3e9211a89d3d7ec3b44af9" +checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff" dependencies = [ "ahash 0.8.10", "arrow", "arrow-array", "arrow-buffer", + "arrow-ord", "arrow-schema", "async-trait", "chrono", @@ -5661,7 +5732,9 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate", "datafusion-physical-expr", + "datafusion-physical-expr-common", "futures", "half", "hashbrown 0.14.3", @@ -5677,9 +5750,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "37.1.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb54b42227136f6287573f2434b1de249fe1b8e6cd6cc73a634e4a3ec29356" +checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e" dependencies = [ "arrow", "arrow-array", @@ -5687,7 +5760,8 @@ dependencies = [ "datafusion-common", "datafusion-expr", "log", - "sqlparser 0.44.0", + "regex", + "sqlparser 0.47.0", "strum 0.26.2", ] @@ -5763,18 +5837,62 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9576c1de19747eb6f5efb6a806c3e836512bbdb17bfedc984ccb0bcc953c8390" +[[package]] +name = "delta_kernel" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1ddfe35af3696786ab5f23cd995df33a66f6cff272ac1f85e09c1a6316acd4c" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-json", + "arrow-ord", + "arrow-schema", + "arrow-select", + "bytes", + "chrono", + "delta_kernel_derive", + "either", + "fix-hidden-lifetime-bug", + "indexmap 2.2.5", + "itertools 0.13.0", + "lazy_static", + "parquet", + "roaring", + "rustc_version", + "serde", + "serde_json", + "thiserror", + "tracing", + "url", + "uuid", + "visibility", + "z85", +] + +[[package]] +name = "delta_kernel_derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4d2127a34b12919a6bce08225f0ca6fde8a19342a32675370edfc8795e7c38a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "deltalake" -version = "0.17.3" -source = "git+https://github.com/delta-io/delta-rs?rev=81593e9#81593e919497221a1a08bf8db9d20e8e4a39a8a6" +version = "0.18.0" +source = "git+https://github.com/delta-io/delta-rs?rev=57795da#57795da9d9cc86a460a5888713adfb3d0584b4cc" dependencies = [ "deltalake-core", ] [[package]] name = "deltalake-core" -version = "0.17.3" -source = "git+https://github.com/delta-io/delta-rs?rev=81593e9#81593e919497221a1a08bf8db9d20e8e4a39a8a6" +version = "0.18.0" +source = "git+https://github.com/delta-io/delta-rs?rev=57795da#57795da9d9cc86a460a5888713adfb3d0584b4cc" dependencies = [ "arrow", "arrow-arith", @@ -5792,13 +5910,14 @@ dependencies = [ "cfg-if", "chrono", "dashmap", + "delta_kernel", "either", "errno", "fix-hidden-lifetime-bug", "futures", "hashbrown 0.14.3", "indexmap 2.2.5", - "itertools 0.12.1", + "itertools 0.13.0", "lazy_static", "libc", "maplit", @@ -6630,9 +6749,9 @@ checksum = "d52a7e408202050813e6f1d9addadcaafef3dca7530c7ddfb005d4081cce6779" [[package]] name = "flatbuffers" -version = "23.5.26" +version = "24.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -9407,6 +9526,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.10" @@ -10855,9 +10983,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" +checksum = "fbebfd32c213ba1907fa7a9c9138015a8de2b43e30c5aa45b18f7deb46786ad6" dependencies = [ "async-trait", "bytes", @@ -10874,6 +11002,22 @@ dependencies = [ "walkdir", ] +[[package]] +name = "object_store_opendal" +version = "0.44.1" +source = "git+https://github.com/Xuanwo/opendal?rev=d6baf68#d6baf6851f6e4ec340e79613db5e83f77092ce11" +dependencies = [ + "async-trait", + "bytes", + "flagset", + "futures", + "futures-util", + "object_store", + "opendal", + "pin-project", + "tokio", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -10908,9 +11052,9 @@ dependencies = [ [[package]] name = "opendal" -version = "0.47.0" +version = "0.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3ba698f2258bebdf7a3a38862bb6ef1f96d351627002686dacc228f805bdd6" +checksum = "876c6655dd5b410c83e0c9edf38be60fed540a1cc1c2f3a2ac31830eb8a8ff45" dependencies = [ "anyhow", "async-backtrace", @@ -11086,13 +11230,13 @@ dependencies = [ [[package]] name = "orc-rust" version = "0.3.0" -source = "git+https://github.com/youngsofun/datafusion-orc?branch=pub#4d69fe9eefb32ab31ccf7e9dcb647fdfc7250caa" +source = "git+https://github.com/youngsofun/datafusion-orc?rev=51733de#51733de24819245eb195e5d01310c99ede815371" dependencies = [ "arrow", "async-trait", "bytes", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "datafusion", "datafusion-expr", "datafusion-physical-expr", @@ -11289,9 +11433,9 @@ dependencies = [ [[package]] name = "parquet" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32" +checksum = "29c3b5322cc1bbf67f11c079c42be41a55949099b78732f7dba9e15edde40eab" dependencies = [ "ahash 0.8.10", "arrow-array", @@ -11302,7 +11446,7 @@ dependencies = [ "arrow-schema", "arrow-select", "base64 0.22.0", - "brotli", + "brotli 6.0.0", "bytes", "chrono", "flate2", @@ -11320,6 +11464,7 @@ dependencies = [ "tokio", "twox-hash", "zstd 0.13.0", + "zstd-sys", ] [[package]] @@ -11352,9 +11497,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pathdiff" @@ -14146,9 +14291,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.44.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf9c7ff146298ffda83a200f8d5084f08dcee1edfc135fcc1d646a45d50ffd6" +checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" dependencies = [ "log", "sqlparser_derive", @@ -15680,6 +15825,17 @@ version = "0.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dcc60c0624df774c82a0ef104151231d37da4962957d691c011c852b2473314" +[[package]] +name = "visibility" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3fd98999db9227cf28e59d83e1f120f42bc233d4b152e8fab9bc87d5bb1e0f8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "volo" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 3dbb8688869f2..a159579973234 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -204,17 +204,17 @@ databend-storages-common-txn = { path = "src/query/storages/common/txn" } # Crates.io dependencies anyerror = { version = "=0.1.10" } anyhow = { version = "1.0.65" } -arrow = { version = "51" } -arrow-array = { version = "51" } -arrow-buffer = { version = "51" } -arrow-cast = { version = "51", features = ["prettyprint"] } -arrow-data = { version = "51" } -arrow-flight = { version = "51", features = ["flight-sql-experimental", "tls"] } +arrow = { version = "52" } +arrow-array = { version = "52" } +arrow-buffer = { version = "52" } +arrow-cast = { version = "52", features = ["prettyprint"] } +arrow-data = { version = "52" } +arrow-flight = { version = "52", features = ["flight-sql-experimental", "tls"] } arrow-format = { version = "0.8.1", features = ["flight-data", "flight-service", "ipc"] } -arrow-ipc = { version = "51" } -arrow-ord = { version = "51" } -arrow-schema = { version = "51", features = ["serde"] } -arrow-select = { version = "51" } +arrow-ipc = { version = "52" } +arrow-ord = { version = "52" } +arrow-schema = { version = "52", features = ["serde"] } +arrow-select = { version = "52" } arrow-udf-js = "0.3.1" arrow-udf-python = "0.2.1" arrow-udf-wasm = "0.2.2" @@ -233,13 +233,13 @@ clap = { version = "4.4.2", features = ["derive"] } criterion = "0.5" dashmap = "5.4.0" deepsize = { version = "0.2.0" } -deltalake = "0.17" +deltalake = "0.18" derive-visitor = { version = "0.4.0", features = ["std-types-drive"] } derive_more = "0.99.17" enumflags2 = { version = "0.7.7", features = ["serde"] } ethnum = { version = "1.5.0" } feature-set = { version = "0.1.1" } -flatbuffers = "23" # Must use the same version with arrow-ipc +flatbuffers = "24" # Must use the same version with arrow-ipc futures = "0.3.24" futures-async-stream = { version = "0.2.7" } futures-util = "0.3.24" @@ -254,7 +254,7 @@ match-template = "0.0.1" mysql_async = { version = "0.34", default-features = false, features = ["rustls-tls"] } once_cell = "1.15.0" openai_api_rust = "0.1" -opendal = { version = "0.47.0", features = [ +opendal = { version = "0.47.1", features = [ "layers-minitrace", "layers-prometheus-client", "layers-async-backtrace", @@ -283,8 +283,8 @@ orc-rust = "0.3" ordered-float = { version = "4.1.0", default-features = false } ordq = "0.2.0" parking_lot = "0.12.1" -parquet = { version = "51", features = ["async"] } -parquet_rs = { package = "parquet", version = "51" } +parquet = { version = "52", features = ["async"] } +paste = "1.0.15" poem = { version = "3.0", features = ["rustls", "multipart", "compression"] } prometheus-client = "0.22" prost = { version = "0.12.1" } @@ -393,11 +393,12 @@ arrow-udf-wasm = { git = "https://github.com/datafuse-extras/arrow-udf", rev = " async-backtrace = { git = "https://github.com/zhang2014/async-backtrace.git", rev = "dea4553" } async-recursion = { git = "https://github.com/zhang2014/async-recursion.git", rev = "a353334" } backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "6145fe6b" } -deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "81593e9" } +deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "57795da" } ethnum = { git = "https://github.com/ariesdevil/ethnum-rs", rev = "4cb05f1" } icelake = { git = "https://github.com/icelake-io/icelake", rev = "be8b2c2" } +object_store_opendal = { package = "object_store_opendal", git = "https://github.com/Xuanwo/opendal", rev = "d6baf68" } openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" } -orc-rust = { git = "https://github.com/youngsofun/datafusion-orc", branch = "pub" } +orc-rust = { git = "https://github.com/youngsofun/datafusion-orc", rev = "51733de" } recursive = { git = "https://github.com/zhang2014/recursive.git", rev = "6af35a1" } sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" } xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" } diff --git a/src/common/arrow/Cargo.toml b/src/common/arrow/Cargo.toml index ce18617fe3382..432999728ca10 100644 --- a/src/common/arrow/Cargo.toml +++ b/src/common/arrow/Cargo.toml @@ -113,7 +113,7 @@ lz4 = { version = "1.24" } num = { version = "0.4", default-features = false, features = ["std"] } num-traits = "0.2" opendal = { workspace = true } -ordered-float = "3.7.0" +ordered-float = { workspace = true } parquet2 = { package = "databend-common-parquet2", path = "../parquet2", default_features = false, features = [ "serde_types", "async", diff --git a/src/common/exception/Cargo.toml b/src/common/exception/Cargo.toml index a1b5e3409fa68..a939ae1a6743a 100644 --- a/src/common/exception/Cargo.toml +++ b/src/common/exception/Cargo.toml @@ -23,7 +23,7 @@ geozero = { workspace = true } http = { workspace = true } opendal = { workspace = true } parquet = { workspace = true } -paste = "1.0.9" +paste = { workspace = true } prost = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/src/meta/api/src/txn_backoff.rs b/src/meta/api/src/txn_backoff.rs index b049d38440681..95e33284a50da 100644 --- a/src/meta/api/src/txn_backoff.rs +++ b/src/meta/api/src/txn_backoff.rs @@ -153,8 +153,9 @@ mod tests { } let elapsed = now.elapsed().as_secs_f64(); + println!("elapsed: {elapsed}"); assert!( - (0.041..0.070).contains(&elapsed), + (0.041..0.080).contains(&elapsed), "{} is expected to be 2 + 5 + 10 + 14 + 20 milliseconds", elapsed ); diff --git a/src/meta/app/Cargo.toml b/src/meta/app/Cargo.toml index 1be881e723688..12bcc64ee16f4 100644 --- a/src/meta/app/Cargo.toml +++ b/src/meta/app/Cargo.toml @@ -31,7 +31,7 @@ maplit = "1.0.2" num-derive = "0.3.3" num-traits = "0.2.15" opendal = { workspace = true } -paste = "1.0.9" +paste = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha1 = "0.10.5" diff --git a/src/query/catalog/Cargo.toml b/src/query/catalog/Cargo.toml index 397fe9b0cee63..8450fbc41ee26 100644 --- a/src/query/catalog/Cargo.toml +++ b/src/query/catalog/Cargo.toml @@ -35,7 +35,7 @@ databend-storages-common-txn = { workspace = true } dyn-clone = "1.0.9" log = { workspace = true } parking_lot = { workspace = true } -parquet_rs = { workspace = true } +parquet = { workspace = true } rand = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/src/query/catalog/src/plan/datasource/datasource_info/parquet.rs b/src/query/catalog/src/plan/datasource/datasource_info/parquet.rs index 07ef121dd106f..fa41bb025eaba 100644 --- a/src/query/catalog/src/plan/datasource/datasource_info/parquet.rs +++ b/src/query/catalog/src/plan/datasource/datasource_info/parquet.rs @@ -26,12 +26,12 @@ use databend_common_meta_app::schema::TableInfo; use databend_common_storage::StageFileInfo; use databend_common_storage::StageFilesInfo; use databend_storages_common_table_meta::meta::ColumnStatistics; -use parquet_rs::file::metadata::ParquetMetaData; -use parquet_rs::format::SchemaElement; -use parquet_rs::schema::types; -use parquet_rs::schema::types::SchemaDescPtr; -use parquet_rs::schema::types::SchemaDescriptor; -use parquet_rs::thrift::TSerializable; +use parquet::file::metadata::ParquetMetaData; +use parquet::format::SchemaElement; +use parquet::schema::types; +use parquet::schema::types::SchemaDescPtr; +use parquet::schema::types::SchemaDescriptor; +use parquet::thrift::TSerializable; use serde::Deserialize; use thrift::protocol::TCompactInputProtocol; use thrift::protocol::TCompactOutputProtocol; @@ -139,13 +139,13 @@ mod tests { use arrow_schema::Schema as ArrowSchema; use databend_common_base::base::tokio::sync::Mutex; use databend_common_storage::StageFilesInfo; - use parquet_rs::basic::ConvertedType; - use parquet_rs::basic::Repetition; - use parquet_rs::basic::Type as PhysicalType; - use parquet_rs::errors::ParquetError; - use parquet_rs::schema::types::SchemaDescPtr; - use parquet_rs::schema::types::SchemaDescriptor; - use parquet_rs::schema::types::Type; + use parquet::basic::ConvertedType; + use parquet::basic::Repetition; + use parquet::basic::Type as PhysicalType; + use parquet::errors::ParquetError; + use parquet::schema::types::SchemaDescPtr; + use parquet::schema::types::SchemaDescriptor; + use parquet::schema::types::Type; use super::ParquetTableInfo; diff --git a/src/query/catalog/src/plan/projection.rs b/src/query/catalog/src/plan/projection.rs index 280b599bcb790..453833753d13e 100644 --- a/src/query/catalog/src/plan/projection.rs +++ b/src/query/catalog/src/plan/projection.rs @@ -22,8 +22,8 @@ use databend_common_storage::parquet_rs::build_parquet_schema_tree; use databend_common_storage::parquet_rs::traverse_parquet_schema_tree; use databend_common_storage::ColumnNode; use databend_common_storage::ColumnNodes; -use parquet_rs::arrow::ProjectionMask; -use parquet_rs::schema::types::SchemaDescriptor; +use parquet::arrow::ProjectionMask; +use parquet::schema::types::SchemaDescriptor; #[derive(serde::Serialize, serde::Deserialize, Clone, PartialEq, Eq)] pub enum Projection { diff --git a/src/query/catalog/tests/it/projection.rs b/src/query/catalog/tests/it/projection.rs index 41c747278849a..9bb405a85d161 100644 --- a/src/query/catalog/tests/it/projection.rs +++ b/src/query/catalog/tests/it/projection.rs @@ -20,7 +20,7 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::TableDataType; use databend_common_expression::TableField; use databend_common_expression::TableSchema; -use parquet_rs::arrow::arrow_to_parquet_schema; +use parquet::arrow::arrow_to_parquet_schema; #[test] fn test_to_projection_mask() -> Result<()> { diff --git a/src/query/service/Cargo.toml b/src/query/service/Cargo.toml index cef8d90d4e806..d958af0c419a9 100644 --- a/src/query/service/Cargo.toml +++ b/src/query/service/Cargo.toml @@ -146,7 +146,7 @@ opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true } parking_lot = { workspace = true } parquet = { workspace = true } -paste = "1.0.9" +paste = { workspace = true } petgraph = { version = "0.6.2", features = ["serde-1"] } pin-project-lite = "0.2.9" poem = { workspace = true } diff --git a/src/query/service/src/servers/flight_sql/flight_sql_service/service.rs b/src/query/service/src/servers/flight_sql/flight_sql_service/service.rs index 4fdf4012ae383..249c1c2e8e171 100644 --- a/src/query/service/src/servers/flight_sql/flight_sql_service/service.rs +++ b/src/query/service/src/servers/flight_sql/flight_sql_service/service.rs @@ -48,6 +48,7 @@ use arrow_flight::sql::CommandPreparedStatementUpdate; use arrow_flight::sql::CommandStatementQuery; use arrow_flight::sql::CommandStatementSubstraitPlan; use arrow_flight::sql::CommandStatementUpdate; +use arrow_flight::sql::DoPutPreparedStatementResult; use arrow_flight::sql::DoPutUpdateResult; use arrow_flight::sql::ProstMessageExt; use arrow_flight::sql::SqlInfo; @@ -525,7 +526,7 @@ impl FlightSqlService for FlightSqlServiceImpl { &self, query: CommandPreparedStatementQuery, request: Request, - ) -> Result::DoPutStream>, Status> { + ) -> Result { let session = self.get_session(&request)?; let handle = Uuid::from_slice(query.prepared_statement_handle.as_ref()) .map_err(|e| Status::internal(format!("Error decoding handle: {e}")))?; @@ -541,8 +542,10 @@ impl FlightSqlService for FlightSqlServiceImpl { let result = PutResult { app_metadata: result.as_any().encode_to_vec().into(), }; - let result = futures::stream::iter(vec![Ok(result)]); - return Ok(Response::new(Box::pin(result))); + + Ok(DoPutPreparedStatementResult { + prepared_statement_handle: Some(result.encode_to_vec().into()), + }) } // called by JDBC diff --git a/src/query/sharing/src/layer.rs b/src/query/sharing/src/layer.rs index 8a6de02cf59e5..c8d0addf763b1 100644 --- a/src/query/sharing/src/layer.rs +++ b/src/query/sharing/src/layer.rs @@ -239,7 +239,7 @@ pub async fn parse_error(er: Response) -> Error { _ => (ErrorKind::Unexpected, false), }; - let mut err = Error::new(kind, &String::from_utf8_lossy(&message)); + let mut err = Error::new(kind, String::from_utf8_lossy(&message)); if retryable { err = err.set_temporary(); diff --git a/src/query/storages/common/blocks/Cargo.toml b/src/query/storages/common/blocks/Cargo.toml index dea091ea91e4e..5c1d425eefc9d 100644 --- a/src/query/storages/common/blocks/Cargo.toml +++ b/src/query/storages/common/blocks/Cargo.toml @@ -14,7 +14,7 @@ test = true databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } databend-storages-common-table-meta = { workspace = true } -parquet_rs = { workspace = true } +parquet = { workspace = true } [build-dependencies] diff --git a/src/query/storages/common/blocks/src/parquet_rs.rs b/src/query/storages/common/blocks/src/parquet_rs.rs index 62891bc1f67b6..c002896a74dfd 100644 --- a/src/query/storages/common/blocks/src/parquet_rs.rs +++ b/src/query/storages/common/blocks/src/parquet_rs.rs @@ -19,11 +19,11 @@ use databend_common_expression::converts::arrow::table_schema_to_arrow_schema; use databend_common_expression::DataBlock; use databend_common_expression::TableSchema; use databend_storages_common_table_meta::table::TableCompression; -use parquet_rs::arrow::ArrowWriter; -use parquet_rs::basic::Encoding; -use parquet_rs::file::properties::EnabledStatistics; -use parquet_rs::file::properties::WriterProperties; -use parquet_rs::format::FileMetaData; +use parquet::arrow::ArrowWriter; +use parquet::basic::Encoding; +use parquet::file::properties::EnabledStatistics; +use parquet::file::properties::WriterProperties; +use parquet::format::FileMetaData; /// Serialize data blocks to parquet format. pub fn blocks_to_parquet( diff --git a/src/query/storages/common/table_meta/Cargo.toml b/src/query/storages/common/table_meta/Cargo.toml index d57795903f192..9a7c09967d7d6 100644 --- a/src/query/storages/common/table_meta/Cargo.toml +++ b/src/query/storages/common/table_meta/Cargo.toml @@ -19,7 +19,7 @@ databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } databend-common-io = { workspace = true } enum-as-inner = "0.5" -parquet_rs = { workspace = true } +parquet = { workspace = true } rmp-serde = "1.1.1" serde = { workspace = true } serde_json = { workspace = true } diff --git a/src/query/storages/common/table_meta/src/table/table_compression.rs b/src/query/storages/common/table_meta/src/table/table_compression.rs index 18f9c72547b4b..71cab1e435169 100644 --- a/src/query/storages/common/table_meta/src/table/table_compression.rs +++ b/src/query/storages/common/table_meta/src/table/table_compression.rs @@ -13,11 +13,11 @@ // limitations under the License. use databend_common_arrow::native; -use databend_common_arrow::parquet; +use databend_common_arrow::parquet as databend_parquet; use databend_common_exception::ErrorCode; -use parquet_rs::basic::Compression as ParquetCompression; -use parquet_rs::basic::GzipLevel; -use parquet_rs::basic::ZstdLevel; +use parquet::basic::Compression as ParquetCompression; +use parquet::basic::GzipLevel; +use parquet::basic::ZstdLevel; use crate::meta; @@ -50,13 +50,15 @@ impl TryFrom<&str> for TableCompression { } /// Convert to parquet CompressionOptions. -impl From for parquet::compression::CompressionOptions { +impl From for databend_parquet::compression::CompressionOptions { fn from(value: TableCompression) -> Self { match value { - TableCompression::None => parquet::compression::CompressionOptions::Uncompressed, - TableCompression::LZ4 => parquet::compression::CompressionOptions::Lz4Raw, - TableCompression::Snappy => parquet::compression::CompressionOptions::Snappy, - TableCompression::Zstd => parquet::compression::CompressionOptions::Zstd(None), + TableCompression::None => { + databend_parquet::compression::CompressionOptions::Uncompressed + } + TableCompression::LZ4 => databend_parquet::compression::CompressionOptions::Lz4Raw, + TableCompression::Snappy => databend_parquet::compression::CompressionOptions::Snappy, + TableCompression::Zstd => databend_parquet::compression::CompressionOptions::Zstd(None), } } } diff --git a/src/query/storages/delta/Cargo.toml b/src/query/storages/delta/Cargo.toml index 820afc7f43077..68e966f8dbde2 100644 --- a/src/query/storages/delta/Cargo.toml +++ b/src/query/storages/delta/Cargo.toml @@ -10,7 +10,6 @@ publish = false arrow-schema = { workspace = true } async-backtrace = { workspace = true } async-trait = { workspace = true } -bytes = { workspace = true } databend-common-base = { workspace = true } databend-common-catalog = { workspace = true } databend-common-exception = { workspace = true } @@ -21,12 +20,9 @@ databend-common-storage = { workspace = true } databend-common-storages-parquet = { workspace = true } databend-storages-common-table-meta = { workspace = true } deltalake = { workspace = true } -flagset = "0.4" -futures = "0.3" match-template = "0.0.1" minitrace = { workspace = true } -object_store = "0.9" -opendal = { workspace = true } +object_store_opendal = "0.44" ordered-float = { workspace = true } parquet = { workspace = true } serde = { workspace = true } diff --git a/src/query/storages/delta/src/dal.rs b/src/query/storages/delta/src/dal.rs deleted file mode 100644 index 15c73ea4cc1ba..0000000000000 --- a/src/query/storages/delta/src/dal.rs +++ /dev/null @@ -1,481 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -/// will be replaced by crate object_store_opendal later! -use std::ops::Range; - -use async_trait::async_trait; -use bytes::Bytes; -use futures::stream::BoxStream; -use futures::FutureExt; -use futures::StreamExt; -use futures::TryStreamExt; -use object_store::path::Path; -use object_store::GetOptions; -use object_store::GetResult; -use object_store::GetResultPayload; -use object_store::ListResult; -use object_store::MultipartId; -use object_store::ObjectMeta; -use object_store::ObjectStore; -use object_store::PutOptions; -use object_store::PutResult; -use object_store::Result; -use opendal::Entry; -use opendal::Metadata; -use opendal::Metakey; -use opendal::Operator; -use tokio::io::AsyncWrite; - -#[derive(Debug)] -pub struct OpendalStore { - inner: Operator, - meta_keys: Vec, -} - -impl OpendalStore { - /// Create OpendalStore by given Operator. - pub fn new(op: Operator) -> Self { - Self { - inner: op, - meta_keys: vec![Metakey::Mode, Metakey::ContentLength, Metakey::LastModified], - } - } - - pub fn with_metakey(mut self, metakey: Metakey) -> Self { - self.meta_keys.push(metakey); - self - } - - pub fn meta_key_flag(&self) -> flagset::FlagSet { - let mut res = Metakey::ContentLength.into(); - for key in self.meta_keys.iter() { - res |= *key; - } - res - } -} - -impl std::fmt::Display for OpendalStore { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "OpenDAL({:?})", self.inner) - } -} - -#[async_trait] -impl ObjectStore for OpendalStore { - async fn put(&self, location: &Path, bytes: Bytes) -> Result { - self.inner - .write(location.as_ref(), bytes) - .await - .map_err(|err| format_object_store_error(err, location.as_ref()))?; - Ok(PutResult { - e_tag: None, - version: None, - }) - } - - async fn put_opts( - &self, - _location: &Path, - _bytes: Bytes, - _opts: PutOptions, - ) -> Result { - Err(object_store::Error::NotSupported { - source: Box::new(opendal::Error::new( - opendal::ErrorKind::Unsupported, - "put_opts is not implemented so far", - )), - }) - } - - async fn put_multipart( - &self, - _location: &Path, - ) -> Result<(MultipartId, Box)> { - Err(object_store::Error::NotSupported { - source: Box::new(opendal::Error::new( - opendal::ErrorKind::Unsupported, - "put_multipart is not implemented so far", - )), - }) - } - - async fn abort_multipart(&self, _location: &Path, _multipart_id: &MultipartId) -> Result<()> { - Err(object_store::Error::NotSupported { - source: Box::new(opendal::Error::new( - opendal::ErrorKind::Unsupported, - "abort_multipart is not implemented so far", - )), - }) - } - - async fn get_opts(&self, _location: &Path, _options: GetOptions) -> Result { - Err(object_store::Error::NotSupported { - source: Box::new(opendal::Error::new( - opendal::ErrorKind::Unsupported, - "get_opts is not implemented so far", - )), - }) - } - - async fn get(&self, location: &Path) -> Result { - let meta = self - .inner - .stat(location.as_ref()) - .await - .map_err(|err| format_object_store_error(err, location.as_ref()))?; - - let meta = ObjectMeta { - location: location.clone(), - last_modified: meta.last_modified().unwrap_or_default(), - size: meta.content_length() as usize, - e_tag: meta.etag().map(|x| x.to_string()), - version: meta.version().map(|x| x.to_string()), - }; - let r = self - .inner - .reader(location.as_ref()) - .await - .map_err(|err| format_object_store_error(err, location.as_ref()))?; - let stream = r - .into_bytes_stream(0..meta.size as u64) - .await - .map_err(|err| object_store::Error::Generic { - store: "IoError", - source: Box::new(err), - })? - .map_err(|err| object_store::Error::Generic { - store: "IoError", - source: Box::new(err), - }); - - Ok(GetResult { - payload: GetResultPayload::Stream(Box::pin(stream)), - range: (0..meta.size), - meta, - }) - } - - async fn get_range(&self, location: &Path, range: Range) -> Result { - let bs = self - .inner - .read_with(location.as_ref()) - .range(range.start as u64..range.end as u64) - .await - .map_err(|err| format_object_store_error(err, location.as_ref()))?; - - Ok(bs.to_bytes()) - } - - async fn head(&self, location: &Path) -> Result { - let meta = self - .inner - .stat(location.as_ref()) - .await - .map_err(|err| format_object_store_error(err, location.as_ref()))?; - - Ok(ObjectMeta { - location: location.clone(), - last_modified: meta.last_modified().unwrap_or_default(), - size: meta.content_length() as usize, - e_tag: meta.etag().map(|x| x.to_string()), - version: meta.version().map(|x| x.to_string()), - }) - } - - async fn delete(&self, location: &Path) -> Result<()> { - self.inner - .delete(location.as_ref()) - .await - .map_err(|err| format_object_store_error(err, location.as_ref()))?; - - Ok(()) - } - - fn list(&self, prefix: Option<&Path>) -> BoxStream<'_, Result> { - // object_store `Path` always removes trailing slash - // need to add it back - let path = prefix.map_or("".into(), |x| format!("{}/", x)); - - let fut = async move { - let stream = self - .inner - .lister_with(&path) - .metakey(self.meta_key_flag()) - .recursive(true) - .await - .map_err(|err| format_object_store_error(err, &path))?; - - let stream = stream.then(|res| async { - let entry = res.map_err(|err| format_object_store_error(err, ""))?; - let meta = entry.metadata(); - - Ok(format_object_meta(entry.path(), meta)) - }); - Ok::<_, object_store::Error>(stream) - }; - - fut.into_stream().try_flatten().boxed() - } - - fn list_with_offset( - &self, - prefix: Option<&Path>, - offset: &Path, - ) -> BoxStream<'_, Result> { - let path = prefix.map_or("".into(), |x| format!("{}/", x)); - let offset = offset.clone(); - - let fut = async move { - let fut = if self.inner.info().full_capability().list_with_start_after { - self.inner - .lister_with(&path) - .start_after(offset.as_ref()) - .metakey(self.meta_key_flag()) - .recursive(true) - .await - .map_err(|err| format_object_store_error(err, &path))? - .then(try_format_object_meta) - .boxed() - } else { - self.inner - .lister_with(&path) - .metakey(self.meta_key_flag()) - .recursive(true) - .await - .map_err(|err| format_object_store_error(err, &path))? - .try_filter(move |entry| futures::future::ready(entry.path() > offset.as_ref())) - .then(try_format_object_meta) - .boxed() - }; - Ok::<_, object_store::Error>(fut) - }; - - fut.into_stream().try_flatten().boxed() - } - - async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { - let path = prefix.map_or("".into(), |x| format!("{}/", x)); - let mut stream = self - .inner - .lister_with(&path) - .metakey(self.meta_key_flag()) - .await - .map_err(|err| format_object_store_error(err, &path))?; - - let mut common_prefixes = Vec::new(); - let mut objects = Vec::new(); - - while let Some(res) = stream.next().await { - let entry = res.map_err(|err| format_object_store_error(err, ""))?; - let meta = entry.metadata(); - - if meta.is_dir() { - common_prefixes.push(entry.path().into()); - } else { - objects.push(format_object_meta(entry.path(), meta)); - } - } - - Ok(ListResult { - common_prefixes, - objects, - }) - } - - async fn copy(&self, _from: &Path, _to: &Path) -> Result<()> { - Err(object_store::Error::NotSupported { - source: Box::new(opendal::Error::new( - opendal::ErrorKind::Unsupported, - "copy is not implemented so far", - )), - }) - } - - async fn rename(&self, _from: &Path, _to: &Path) -> Result<()> { - Err(object_store::Error::NotSupported { - source: Box::new(opendal::Error::new( - opendal::ErrorKind::Unsupported, - "rename is not implemented so far", - )), - }) - } - - async fn copy_if_not_exists(&self, _from: &Path, _to: &Path) -> Result<()> { - Err(object_store::Error::NotSupported { - source: Box::new(opendal::Error::new( - opendal::ErrorKind::Unsupported, - "copy_if_not_exists is not implemented so far", - )), - }) - } -} - -fn format_object_store_error(err: opendal::Error, path: &str) -> object_store::Error { - use opendal::ErrorKind; - match err.kind() { - ErrorKind::NotFound => object_store::Error::NotFound { - path: path.to_string(), - source: Box::new(err), - }, - ErrorKind::Unsupported => object_store::Error::NotSupported { - source: Box::new(err), - }, - ErrorKind::AlreadyExists => object_store::Error::AlreadyExists { - path: path.to_string(), - source: Box::new(err), - }, - kind => object_store::Error::Generic { - store: kind.into_static(), - source: Box::new(err), - }, - } -} - -fn format_object_meta(path: &str, meta: &Metadata) -> ObjectMeta { - ObjectMeta { - location: path.into(), - last_modified: meta.last_modified().unwrap_or_default(), - size: meta.content_length() as usize, - e_tag: meta.etag().map(|x| x.to_string()), - version: meta.version().map(|x| x.to_string()), - } -} - -async fn try_format_object_meta(res: Result) -> Result { - let entry = res.map_err(|err| format_object_store_error(err, ""))?; - let meta = entry.metadata(); - - Ok(format_object_meta(entry.path(), meta)) -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use object_store::path::Path; - use object_store::ObjectStore; - use opendal::services; - - use super::*; - - async fn create_test_object_store() -> Arc { - let op = Operator::new(services::Memory::default()).unwrap().finish(); - let object_store = Arc::new(OpendalStore::new(op)); - - let path: Path = "data/test.txt".into(); - let bytes = Bytes::from_static(b"hello, world!"); - object_store.put(&path, bytes).await.unwrap(); - - let path: Path = "data/nested/test.txt".into(); - let bytes = Bytes::from_static(b"hello, world! I am nested."); - object_store.put(&path, bytes).await.unwrap(); - - object_store - } - - #[tokio::test] - async fn test_basic() { - let op = Operator::new(services::Memory::default()).unwrap().finish(); - let object_store: Arc = Arc::new(OpendalStore::new(op)); - - // Retrieve a specific file - let path: Path = "data/test.txt".into(); - - let bytes = Bytes::from_static(b"hello, world!"); - object_store.put(&path, bytes.clone()).await.unwrap(); - - let meta = object_store.head(&path).await.unwrap(); - - assert_eq!(meta.size, 13); - - assert_eq!( - object_store - .get(&path) - .await - .unwrap() - .bytes() - .await - .unwrap(), - bytes - ); - } - - #[tokio::test] - async fn test_list() { - let object_store = create_test_object_store().await; - let path: Path = "data/".into(); - let results = object_store.list(Some(&path)).collect::>().await; - assert_eq!(results.len(), 2); - let mut locations = results - .iter() - .map(|x| x.as_ref().unwrap().location.as_ref()) - .collect::>(); - - let expected_files = vec![ - ( - "data/nested/test.txt", - Bytes::from_static(b"hello, world! I am nested."), - ), - ("data/test.txt", Bytes::from_static(b"hello, world!")), - ]; - - let expected_locations = expected_files.iter().map(|x| x.0).collect::>(); - - locations.sort(); - assert_eq!(locations, expected_locations); - - for (location, bytes) in expected_files { - let path: Path = location.into(); - assert_eq!( - object_store - .get(&path) - .await - .unwrap() - .bytes() - .await - .unwrap(), - bytes - ); - } - } - - #[tokio::test] - async fn test_list_with_delimiter() { - let object_store = create_test_object_store().await; - let path: Path = "data/".into(); - let result = object_store.list_with_delimiter(Some(&path)).await.unwrap(); - assert_eq!(result.objects.len(), 1); - assert_eq!(result.common_prefixes.len(), 1); - assert_eq!(result.objects[0].location.as_ref(), "data/test.txt"); - assert_eq!(result.common_prefixes[0].as_ref(), "data/nested"); - } - - #[tokio::test] - async fn test_list_with_offset() { - let object_store = create_test_object_store().await; - let path: Path = "data/".into(); - let offset: Path = "data/nested/test.txt".into(); - let result = object_store - .list_with_offset(Some(&path), &offset) - .collect::>() - .await; - assert_eq!(result.len(), 1); - assert_eq!( - result[0].as_ref().unwrap().location.as_ref(), - "data/test.txt" - ); - } -} diff --git a/src/query/storages/delta/src/lib.rs b/src/query/storages/delta/src/lib.rs index ad3cbf34623a5..7532257a68458 100644 --- a/src/query/storages/delta/src/lib.rs +++ b/src/query/storages/delta/src/lib.rs @@ -16,7 +16,6 @@ #![feature(impl_trait_in_assoc_type)] #![allow(clippy::diverging_sub_expression)] -mod dal; mod partition; mod partition_columns; mod table; diff --git a/src/query/storages/delta/src/table.rs b/src/query/storages/delta/src/table.rs index 1ef460a58a0eb..eb1bd7b6b12f4 100644 --- a/src/query/storages/delta/src/table.rs +++ b/src/query/storages/delta/src/table.rs @@ -45,14 +45,12 @@ use databend_common_storages_parquet::ParquetRSReaderBuilder; use databend_storages_common_table_meta::table::OPT_KEY_ENGINE_META; use deltalake::kernel::Add; use deltalake::DeltaTableBuilder; -use opendal::Metakey; +use object_store_opendal::OpendalStore; use serde::Deserialize; use serde::Serialize; use tokio::sync::OnceCell; use url::Url; -// use object_store_opendal::OpendalStore; -use crate::dal::OpendalStore; use crate::partition::DeltaPartInfo; use crate::partition_columns::get_partition_values; use crate::partition_columns::get_pushdown_without_partition_columns; @@ -156,7 +154,7 @@ impl DeltaTable { #[async_backtrace::framed] pub async fn load(sp: &StorageParams) -> Result { let op = init_operator(sp)?; - let opendal_store = Arc::new(OpendalStore::new(op).with_metakey(Metakey::Version)); + let opendal_store = Arc::new(OpendalStore::new(op)); let mut table = DeltaTableBuilder::from_uri(Url::from_directory_path("/").unwrap()) .with_storage_backend(opendal_store, Url::from_directory_path("/").unwrap()) diff --git a/src/query/storages/fuse/Cargo.toml b/src/query/storages/fuse/Cargo.toml index 875d738a3b70e..b6d0dfd8d42db 100644 --- a/src/query/storages/fuse/Cargo.toml +++ b/src/query/storages/fuse/Cargo.toml @@ -56,7 +56,7 @@ jsonb = { workspace = true } log = { workspace = true } minitrace = { workspace = true } opendal = { workspace = true } -parquet_rs = { workspace = true } +parquet = { workspace = true } rand = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/src/query/storages/fuse/src/io/read/block/parquet/adapter.rs b/src/query/storages/fuse/src/io/read/block/parquet/adapter.rs index 63f2146843a3d..74029ed2b320b 100644 --- a/src/query/storages/fuse/src/io/read/block/parquet/adapter.rs +++ b/src/query/storages/fuse/src/io/read/block/parquet/adapter.rs @@ -18,14 +18,14 @@ use std::collections::HashMap; use std::sync::Arc; use bytes::Bytes; -use parquet_rs::arrow::arrow_reader::RowGroups; -use parquet_rs::basic::Compression; -use parquet_rs::column::page::PageIterator; -use parquet_rs::column::page::PageReader; -use parquet_rs::errors::Result as ParquetResult; -use parquet_rs::file::metadata::ColumnChunkMetaData; -use parquet_rs::file::serialized_reader::SerializedPageReader; -use parquet_rs::schema::types::SchemaDescriptor; +use parquet::arrow::arrow_reader::RowGroups; +use parquet::basic::Compression; +use parquet::column::page::PageIterator; +use parquet::column::page::PageReader; +use parquet::errors::Result as ParquetResult; +use parquet::file::metadata::ColumnChunkMetaData; +use parquet::file::serialized_reader::SerializedPageReader; +use parquet::schema::types::SchemaDescriptor; pub struct RowGroupImplBuilder<'a> { num_rows: usize, diff --git a/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs b/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs index bcc46a34206bb..83a4e1452ac99 100644 --- a/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs +++ b/src/query/storages/fuse/src/io/read/block/parquet/deserialize.rs @@ -19,11 +19,11 @@ use databend_common_expression::converts::arrow::table_schema_to_arrow_schema; use databend_common_expression::ColumnId; use databend_common_expression::TableSchema; use databend_storages_common_table_meta::meta::Compression; -use parquet_rs::arrow::arrow_reader::ParquetRecordBatchReader; -use parquet_rs::arrow::arrow_to_parquet_schema; -use parquet_rs::arrow::parquet_to_arrow_field_levels; -use parquet_rs::arrow::ProjectionMask; -use parquet_rs::basic::Compression as ParquetCompression; +use parquet::arrow::arrow_reader::ParquetRecordBatchReader; +use parquet::arrow::arrow_to_parquet_schema; +use parquet::arrow::parquet_to_arrow_field_levels; +use parquet::arrow::ProjectionMask; +use parquet::basic::Compression as ParquetCompression; use crate::io::read::block::block_reader_merge_io::DataItem; use crate::io::read::block::parquet::adapter::RowGroupImplBuilder; diff --git a/src/query/storages/fuse/src/io/read/bloom/block_filter_reader.rs b/src/query/storages/fuse/src/io/read/bloom/block_filter_reader.rs index 6a46de50928ad..0426710c0ee85 100644 --- a/src/query/storages/fuse/src/io/read/bloom/block_filter_reader.rs +++ b/src/query/storages/fuse/src/io/read/bloom/block_filter_reader.rs @@ -35,8 +35,8 @@ use databend_storages_common_table_meta::meta::Location; use databend_storages_common_table_meta::meta::SingleColumnMeta; use futures_util::future::try_join_all; use opendal::Operator; -use parquet_rs::arrow::arrow_to_parquet_schema; -use parquet_rs::schema::types::SchemaDescPtr; +use parquet::arrow::arrow_to_parquet_schema; +use parquet::schema::types::SchemaDescPtr; use crate::index::filters::BlockBloomFilterIndexVersion; use crate::index::filters::BlockFilter; diff --git a/src/query/storages/fuse/src/io/read/bloom/column_filter_reader.rs b/src/query/storages/fuse/src/io/read/bloom/column_filter_reader.rs index bb16a91b688ca..959f41642943a 100644 --- a/src/query/storages/fuse/src/io/read/bloom/column_filter_reader.rs +++ b/src/query/storages/fuse/src/io/read/bloom/column_filter_reader.rs @@ -28,11 +28,11 @@ use databend_storages_common_index::filters::Filter; use databend_storages_common_index::filters::Xor8Filter; use databend_storages_common_table_meta::meta::SingleColumnMeta; use opendal::Operator; -use parquet_rs::arrow::arrow_reader::ParquetRecordBatchReader; -use parquet_rs::arrow::parquet_to_arrow_field_levels; -use parquet_rs::arrow::ProjectionMask; -use parquet_rs::basic::Compression as ParquetCompression; -use parquet_rs::schema::types::SchemaDescPtr; +use parquet::arrow::arrow_reader::ParquetRecordBatchReader; +use parquet::arrow::parquet_to_arrow_field_levels; +use parquet::arrow::ProjectionMask; +use parquet::basic::Compression as ParquetCompression; +use parquet::schema::types::SchemaDescPtr; use crate::io::read::block::parquet::RowGroupImplBuilder; diff --git a/src/query/storages/fuse/src/io/read/meta/meta_readers.rs b/src/query/storages/fuse/src/io/read/meta/meta_readers.rs index d0b08f84fef18..2a5d30a780073 100644 --- a/src/query/storages/fuse/src/io/read/meta/meta_readers.rs +++ b/src/query/storages/fuse/src/io/read/meta/meta_readers.rs @@ -39,8 +39,8 @@ use futures::AsyncSeek; use futures_util::AsyncSeekExt; use opendal::Buffer; use opendal::Operator; -use parquet_rs::format::FileMetaData; -use parquet_rs::thrift::TSerializable; +use parquet::format::FileMetaData; +use parquet::thrift::TSerializable; use self::thrift_file_meta_read::read_thrift_file_metadata; diff --git a/src/query/storages/fuse/src/operations/util.rs b/src/query/storages/fuse/src/operations/util.rs index b4305ea08c94f..8142852a6142a 100644 --- a/src/query/storages/fuse/src/operations/util.rs +++ b/src/query/storages/fuse/src/operations/util.rs @@ -69,7 +69,7 @@ pub fn set_backoff( } pub fn column_parquet_metas( - file_meta: &parquet_rs::format::FileMetaData, + file_meta: &parquet::format::FileMetaData, schema: &TableSchemaRef, ) -> Result> { // currently we use one group only diff --git a/src/query/storages/result_cache/Cargo.toml b/src/query/storages/result_cache/Cargo.toml index 06e0e35d02cd9..f1cd8211a7dde 100644 --- a/src/query/storages/result_cache/Cargo.toml +++ b/src/query/storages/result_cache/Cargo.toml @@ -28,7 +28,7 @@ databend-common-storage = { workspace = true } databend-storages-common-blocks = { workspace = true } databend-storages-common-table-meta = { workspace = true } opendal = { workspace = true } -parquet_rs = { workspace = true } +parquet = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha2 = "0.10.6" diff --git a/src/query/storages/result_cache/src/read/reader.rs b/src/query/storages/result_cache/src/read/reader.rs index 36f9e74210476..60bf670113376 100644 --- a/src/query/storages/result_cache/src/read/reader.rs +++ b/src/query/storages/result_cache/src/read/reader.rs @@ -22,9 +22,9 @@ use databend_common_expression::TableSchema; use databend_common_meta_store::MetaStore; use databend_common_storage::DataOperator; use opendal::Operator; -use parquet_rs::arrow::arrow_reader::ParquetRecordBatchReader; -use parquet_rs::arrow::parquet_to_arrow_schema; -use parquet_rs::file::footer::parse_metadata; +use parquet::arrow::arrow_reader::ParquetRecordBatchReader; +use parquet::arrow::parquet_to_arrow_schema; +use parquet::file::footer::parse_metadata; use crate::common::gen_result_cache_meta_key; use crate::common::ResultCacheValue; diff --git a/src/query/storages/stage/Cargo.toml b/src/query/storages/stage/Cargo.toml index 3b2cc1854e778..3e8ea9814c5af 100644 --- a/src/query/storages/stage/Cargo.toml +++ b/src/query/storages/stage/Cargo.toml @@ -38,7 +38,7 @@ enum-as-inner = "0.6.0" futures = { workspace = true } log = { workspace = true } opendal = { workspace = true } -parquet_rs = { workspace = true } +parquet = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } typetag = { workspace = true } diff --git a/src/query/storages/stage/src/append/parquet_file/writer_processor.rs b/src/query/storages/stage/src/append/parquet_file/writer_processor.rs index 33f0bd9de0bd9..8e96840264aa7 100644 --- a/src/query/storages/stage/src/append/parquet_file/writer_processor.rs +++ b/src/query/storages/stage/src/append/parquet_file/writer_processor.rs @@ -32,10 +32,10 @@ use databend_common_pipeline_core::processors::Processor; use databend_common_pipeline_core::processors::ProcessorPtr; use databend_storages_common_table_meta::table::TableCompression; use opendal::Operator; -use parquet_rs::arrow::ArrowWriter; -use parquet_rs::basic::Encoding; -use parquet_rs::file::properties::EnabledStatistics; -use parquet_rs::file::properties::WriterProperties; +use parquet::arrow::ArrowWriter; +use parquet::basic::Encoding; +use parquet::file::properties::EnabledStatistics; +use parquet::file::properties::WriterProperties; use super::block_batch::BlockBatch; use crate::append::output::DataSummary; diff --git a/src/tests/sqlsmith/Cargo.toml b/src/tests/sqlsmith/Cargo.toml index fa69a0a4d1f42..c49ed9a025151 100644 --- a/src/tests/sqlsmith/Cargo.toml +++ b/src/tests/sqlsmith/Cargo.toml @@ -9,15 +9,15 @@ edition = { workspace = true } [dependencies] chrono-tz = { workspace = true } clap = { workspace = true } -databend-client = "0.18.1" +databend-client = { git = "https://github.com/datafuselabs/bendsql/", rev = "7658a5" } databend-common-ast = { workspace = true } databend-common-expression = { workspace = true } databend-common-formats = { workspace = true } databend-common-functions = { workspace = true } databend-common-io = { workspace = true } databend-common-sql = { workspace = true } -databend-driver = "0.18.1" -databend-sql = "0.18.1" +databend-driver = { git = "https://github.com/datafuselabs/bendsql/", rev = "7658a5" } +databend-sql = { git = "https://github.com/datafuselabs/bendsql/", rev = "7658a5" } ethnum = { workspace = true } itertools = { workspace = true } jsonb = { workspace = true }