Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Reduce dependencies of datafusion-sql crate #3566

Merged
merged 7 commits into from
Sep 21, 2022
Merged

Reduce dependencies of datafusion-sql crate #3566

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8d13cc7
Remove tokio dependency from datafusion-sql crate
mbrobbel Sep 20, 2022
c3f09ee
Remove hashbrown dependency from datafusion-sql crate
mbrobbel Sep 20, 2022
b74a066
Remove ahash dependency from datafusion-sql crate
mbrobbel Sep 20, 2022
d3eaa2b
Reduce features in common, expr and sql crates
mbrobbel Sep 20, 2022
06e626d
Disable default features of arrow dependency in jit crate
mbrobbel Sep 20, 2022
8cbb48e
Enable `snappy` feature of `apache-avro` dependency in `common` crate
mbrobbel Sep 21, 2022
f522f88
Fix Cargo.toml feature ordering
mbrobbel Sep 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions datafusion/common/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,16 @@ path = "src/lib.rs"

[features]
avro = ["apache-avro"]
default = []
jit = ["cranelift-module"]
pyarrow = ["pyo3"]
pyarrow = ["pyo3", "arrow/pyarrow"]

[dependencies]
apache-avro = { version = "0.14", features = ["snappy"], optional = true }
arrow = { version = "23.0.0", features = ["prettyprint"] }
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true }
arrow = { version = "23.0.0", default-features = false }
cranelift-module = { version = "0.87.0", optional = true }
object_store = { version = "0.5.0", optional = true }
object_store = { version = "0.5.0", default-features = false, optional = true }
ordered-float = "3.0"
parquet = { version = "23.0.0", features = ["arrow"], optional = true }
parquet = { version = "23.0.0", default-features = false, optional = true }
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given the only reason for these dependencies is deriving From conversions for ArrowError, perhaps we could/should just remove them. This is what I have been doing when splitting apart arrow-rs - see apache/arrow-rs#2711.

In general the error handling is a little bit unfortunate, so perhaps this will all get addressed eventually by apache/arrow-rs#2725

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes absolutely. I was planning on revisiting this after the arrow-rs split.

pyo3 = { version = "0.17.1", optional = true }
sqlparser = "0.23"
2 changes: 1 addition & 1 deletion datafusion/expr/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,6 @@ path = "src/lib.rs"

[dependencies]
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", default-features = false }
datafusion-common = { path = "../common", version = "12.0.0" }
sqlparser = "0.23"
2 changes: 1 addition & 1 deletion datafusion/jit/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ path = "src/lib.rs"
jit = []

[dependencies]
arrow = "23.0.0"
arrow = { version = "23.0.0", default-features = false }
cranelift = "0.87.0"
cranelift-jit = "0.87.0"
cranelift-module = "0.87.0"
Expand Down
5 changes: 1 addition & 4 deletions datafusion/sql/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,7 @@ default = ["unicode_expressions"]
unicode_expressions = []

[dependencies]
ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", default-features = false }
datafusion-common = { path = "../common", version = "12.0.0" }
datafusion-expr = { path = "../expr", version = "12.0.0" }
hashbrown = "0.12"
sqlparser = "0.23"
tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync", "fs", "parking_lot"] }
31 changes: 15 additions & 16 deletions datafusion/sql/src/planner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,7 @@ use datafusion_expr::{
use datafusion_expr::{
window_function::WindowFunction, BuiltinScalarFunction, TableSource,
};
use hashbrown::HashMap;
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::str::FromStr;
use std::sync::Arc;
use std::{convert::TryInto, vec};
Expand Down Expand Up @@ -4995,8 +4994,8 @@ mod tests {
quick_test(sql, expected);
}

#[tokio::test]
async fn subquery_references_cte() {
#[test]
fn subquery_references_cte() {
let sql = "WITH \
cte AS (SELECT * FROM person) \
SELECT * FROM person WHERE EXISTS (SELECT * FROM cte WHERE id = person.id)";
Expand All @@ -5013,17 +5012,17 @@ mod tests {
quick_test(sql, expected)
}

#[tokio::test]
async fn aggregate_with_rollup() {
#[test]
fn aggregate_with_rollup() {
let sql = "SELECT id, state, age, COUNT(*) FROM person GROUP BY id, ROLLUP (state, age)";
let expected = "Projection: #person.id, #person.state, #person.age, #COUNT(UInt8(1))\
\n Aggregate: groupBy=[[#person.id, ROLLUP (#person.state, #person.age)]], aggr=[[COUNT(UInt8(1))]]\
\n TableScan: person";
quick_test(sql, expected);
}

#[tokio::test]
async fn aggregate_with_rollup_with_grouping() {
#[test]
fn aggregate_with_rollup_with_grouping() {
let sql = "SELECT id, state, age, grouping(state), grouping(age), grouping(state) + grouping(age), COUNT(*) \
FROM person GROUP BY id, ROLLUP (state, age)";
let expected = "Projection: #person.id, #person.state, #person.age, #GROUPING(person.state), #GROUPING(person.age), #GROUPING(person.state) + #GROUPING(person.age), #COUNT(UInt8(1))\
Expand All @@ -5032,8 +5031,8 @@ mod tests {
quick_test(sql, expected);
}

#[tokio::test]
async fn rank_partition_grouping() {
#[test]
fn rank_partition_grouping() {
let sql = "select
sum(age) as total_sum,
state,
Expand All @@ -5054,8 +5053,8 @@ mod tests {
quick_test(sql, expected);
}

#[tokio::test]
async fn aggregate_with_cube() {
#[test]
fn aggregate_with_cube() {
let sql =
"SELECT id, state, age, COUNT(*) FROM person GROUP BY id, CUBE (state, age)";
let expected = "Projection: #person.id, #person.state, #person.age, #COUNT(UInt8(1))\
Expand All @@ -5064,17 +5063,17 @@ mod tests {
quick_test(sql, expected);
}

#[tokio::test]
async fn round_decimal() {
#[test]
fn round_decimal() {
let sql = "SELECT round(price/3, 2) FROM test_decimal";
let expected = "Projection: round(#test_decimal.price / Int64(3), Int64(2))\
\n TableScan: test_decimal";
quick_test(sql, expected);
}

#[ignore] // see https://github.com/apache/arrow-datafusion/issues/2469
#[tokio::test]
async fn aggregate_with_grouping_sets() {
#[test]
fn aggregate_with_grouping_sets() {
let sql = "SELECT id, state, age, COUNT(*) FROM person GROUP BY id, GROUPING SETS ((state), (state, age), (id, state))";
let expected = "TBD";
quick_test(sql, expected);
Expand Down