Skip to content

Commit

Permalink
chore: Update git reference; fix remaining types and tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@stuartcarnie
stuartcarnie committed Aug 2, 2022
1 parent b885f56 commit cc19881
Show file tree
Hide file tree
Showing 16 changed files with 132 additions and 134 deletions.
2 changes: 0 additions & 2 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,4 @@ lto = true

[patch.crates-io]
arrow = { git = "https://github.com/stuartcarnie/arrow-rs.git", rev = "7340b1935479c5338a75980e69080508d0a76bf7" }
arrow-flight = { git = "https://github.com/stuartcarnie/arrow-rs.git", rev = "7340b1935479c5338a75980e69080508d0a76bf7" }
object_store = { git = "https://github.com/stuartcarnie/arrow-rs.git", rev = "7340b1935479c5338a75980e69080508d0a76bf7" }
parquet = { git = "https://github.com/stuartcarnie/arrow-rs.git", rev = "7340b1935479c5338a75980e69080508d0a76bf7" }
10 changes: 5 additions & 5 deletions datafusion/common/src/scalar.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2176,7 +2176,7 @@ mod tests {
#[test]
fn scalar_decimal_test() {
let decimal_value = ScalarValue::Decimal128(Some(123), 10, 1);
assert_eq!(DataType::Decimal(10, 1), decimal_value.get_datatype());
assert_eq!(DataType::Decimal128(10, 1), decimal_value.get_datatype());
let try_into_value: i128 = decimal_value.clone().try_into().unwrap();
assert_eq!(123_i128, try_into_value);
assert!(!decimal_value.is_null());
Expand All @@ -2194,14 +2194,14 @@ mod tests {
let array = decimal_value.to_array();
let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
assert_eq!(1, array.len());
assert_eq!(DataType::Decimal(10, 1), array.data_type().clone());
assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone());
assert_eq!(123i128, array.value(0).as_i128());

// decimal scalar to array with size
let array = decimal_value.to_array_of_size(10);
let array_decimal = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
assert_eq!(10, array.len());
assert_eq!(DataType::Decimal(10, 1), array.data_type().clone());
assert_eq!(DataType::Decimal128(10, 1), array.data_type().clone());
assert_eq!(123i128, array_decimal.value(0).as_i128());
assert_eq!(123i128, array_decimal.value(9).as_i128());
// test eq array
Expand Down Expand Up @@ -2239,7 +2239,7 @@ mod tests {
// convert the vec to decimal array and check the result
let array = ScalarValue::iter_to_array(decimal_vec.into_iter()).unwrap();
assert_eq!(3, array.len());
assert_eq!(DataType::Decimal(10, 2), array.data_type().clone());
assert_eq!(DataType::Decimal128(10, 2), array.data_type().clone());

let decimal_vec = vec![
ScalarValue::Decimal128(Some(1), 10, 2),
Expand All @@ -2249,7 +2249,7 @@ mod tests {
];
let array = ScalarValue::iter_to_array(decimal_vec.into_iter()).unwrap();
assert_eq!(4, array.len());
assert_eq!(DataType::Decimal(10, 2), array.data_type().clone());
assert_eq!(DataType::Decimal128(10, 2), array.data_type().clone());

assert!(ScalarValue::try_new_decimal128(1, 10, 2)
.unwrap()
Expand Down
4 changes: 2 additions & 2 deletions datafusion/core/src/avro_to_arrow/schema.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ fn schema_to_field_with_props(
AvroSchema::Fixed { size, .. } => DataType::FixedSizeBinary(*size as i32),
AvroSchema::Decimal {
precision, scale, ..
} => DataType::Decimal(*precision, *scale),
} => DataType::Decimal128(*precision, *scale),
AvroSchema::Uuid => DataType::FixedSizeBinary(16),
AvroSchema::Date => DataType::Date32,
AvroSchema::TimeMillis => DataType::Time32(TimeUnit::Millisecond),
Expand Down Expand Up @@ -216,7 +216,7 @@ fn default_field_name(dt: &DataType) -> &str {
DataType::Union(_, _, _) => "union",
DataType::Dictionary(_, _) => "map",
DataType::Map(_, _) => unimplemented!("Map support not implemented"),
DataType::Decimal(_, _) => "decimal",
DataType::Decimal128(_, _) => "decimal",
DataType::Decimal256(_, _) => "decimal",
}
}
Expand Down
8 changes: 4 additions & 4 deletions datafusion/core/src/datasource/file_format/parquet.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1073,30 +1073,30 @@ mod tests {
assert_eq!(1, batches.len());
assert_eq!(1, batches[0].num_columns());
let column = batches[0].column(0);
assert_eq!(&DataType::Decimal(4, 2), column.data_type());
assert_eq!(&DataType::Decimal128(4, 2), column.data_type());

// parquet use the int64 as the physical type to store decimal
let exec = get_exec("int64_decimal.parquet", None, None).await?;
let batches = collect(exec, task_ctx.clone()).await?;
assert_eq!(1, batches.len());
assert_eq!(1, batches[0].num_columns());
let column = batches[0].column(0);
assert_eq!(&DataType::Decimal(10, 2), column.data_type());
assert_eq!(&DataType::Decimal128(10, 2), column.data_type());

// parquet use the fixed length binary as the physical type to store decimal
let exec = get_exec("fixed_length_decimal.parquet", None, None).await?;
let batches = collect(exec, task_ctx.clone()).await?;
assert_eq!(1, batches.len());
assert_eq!(1, batches[0].num_columns());
let column = batches[0].column(0);
assert_eq!(&DataType::Decimal(25, 2), column.data_type());
assert_eq!(&DataType::Decimal128(25, 2), column.data_type());

let exec = get_exec("fixed_length_decimal_legacy.parquet", None, None).await?;
let batches = collect(exec, task_ctx.clone()).await?;
assert_eq!(1, batches.len());
assert_eq!(1, batches[0].num_columns());
let column = batches[0].column(0);
assert_eq!(&DataType::Decimal(13, 2), column.data_type());
assert_eq!(&DataType::Decimal128(13, 2), column.data_type());

// parquet use the fixed length binary as the physical type to store decimal
// TODO: arrow-rs don't support convert the physical type of binary to decimal
Expand Down
6 changes: 3 additions & 3 deletions datafusion/core/src/physical_optimizer/pruning.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1457,7 +1457,7 @@ mod tests {
// decimal(9,2)
let schema = Arc::new(Schema::new(vec![Field::new(
"s1",
DataType::Decimal(9, 2),
DataType::Decimal128(9, 2),
true,
)]));
// s1 > 5
Expand All @@ -1479,7 +1479,7 @@ mod tests {
// decimal(18,2)
let schema = Arc::new(Schema::new(vec![Field::new(
"s1",
DataType::Decimal(18, 2),
DataType::Decimal128(18, 2),
true,
)]));
// s1 > 5
Expand All @@ -1501,7 +1501,7 @@ mod tests {
// decimal(23,2)
let schema = Arc::new(Schema::new(vec![Field::new(
"s1",
DataType::Decimal(23, 2),
DataType::Decimal128(23, 2),
true,
)]));
// s1 > 5
Expand Down
2 changes: 1 addition & 1 deletion datafusion/core/tests/parquet_pruning.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -880,7 +880,7 @@ fn make_f64_batch(v: Vec<f64>) -> RecordBatch {
fn make_decimal_batch(v: Vec<i128>, precision: usize, scale: usize) -> RecordBatch {
let schema = Arc::new(Schema::new(vec![Field::new(
"decimal_col",
DataType::Decimal(precision, scale),
DataType::Decimal128(precision, scale),
true,
)]));
let array = Arc::new(
Expand Down
98 changes: 49 additions & 49 deletions datafusion/core/tests/sql/decimal.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ async fn decimal_cast() -> Result<()> {
actual[0].schema().field(0).data_type()
);
let expected = vec![
"+---------------------------------------+",
"+------------------------------------------+",
"| CAST(Float64(1.23) AS Decimal128(10, 4)) |",
"+---------------------------------------+",
"| 1.2300 |",
"+---------------------------------------+",
"+------------------------------------------+",
"| 1.2300 |",
"+------------------------------------------+",
];
assert_batches_eq!(expected, &actual);

Expand All @@ -42,11 +42,11 @@ async fn decimal_cast() -> Result<()> {
actual[0].schema().field(0).data_type()
);
let expected = vec![
"+---------------------------------------------------------------+",
"+---------------------------------------------------------------------+",
"| CAST(CAST(Float64(1.23) AS Decimal128(10, 3)) AS Decimal128(10, 4)) |",
"+---------------------------------------------------------------+",
"| 1.2300 |",
"+---------------------------------------------------------------+",
"+---------------------------------------------------------------------+",
"| 1.2300 |",
"+---------------------------------------------------------------------+",
];
assert_batches_eq!(expected, &actual);

Expand All @@ -57,11 +57,11 @@ async fn decimal_cast() -> Result<()> {
actual[0].schema().field(0).data_type()
);
let expected = vec![
"+-----------------------------------------+",
"+--------------------------------------------+",
"| CAST(Float64(1.2345) AS Decimal128(24, 2)) |",
"+-----------------------------------------+",
"| 1.23 |",
"+-----------------------------------------+",
"+--------------------------------------------+",
"| 1.23 |",
"+--------------------------------------------+",
];
assert_batches_eq!(expected, &actual);

Expand Down Expand Up @@ -231,7 +231,7 @@ async fn decimal_logic_op() -> Result<()> {
let ctx = SessionContext::new();
register_decimal_csv_table_by_sql(&ctx).await;
// logic operation: eq
let sql = "select * from decimal_simple where c1=CAST(0.00002 as Decimal128(10,8))";
let sql = "select * from decimal_simple where c1=CAST(0.00002 as Decimal(10,8))";
let actual = execute_to_batches(&ctx, sql).await;
assert_eq!(
&DataType::Decimal128(10, 6),
Expand Down Expand Up @@ -550,25 +550,25 @@ async fn decimal_arithmetic_op() -> Result<()> {
actual[0].schema().field(0).data_type()
);
let expected = vec![
"+-------------------------------------------------------------+",
"+----------------------------------------------------------------+",
"| decimal_simple.c1 / CAST(Float64(0.00001) AS Decimal128(5, 5)) |",
"+-------------------------------------------------------------+",
"| 1.000000000000 |",
"| 2.000000000000 |",
"| 2.000000000000 |",
"| 3.000000000000 |",
"| 3.000000000000 |",
"| 3.000000000000 |",
"| 4.000000000000 |",
"| 4.000000000000 |",
"| 4.000000000000 |",
"| 4.000000000000 |",
"| 5.000000000000 |",
"| 5.000000000000 |",
"| 5.000000000000 |",
"| 5.000000000000 |",
"| 5.000000000000 |",
"+-------------------------------------------------------------+",
"+----------------------------------------------------------------+",
"| 1.000000000000 |",
"| 2.000000000000 |",
"| 2.000000000000 |",
"| 3.000000000000 |",
"| 3.000000000000 |",
"| 3.000000000000 |",
"| 4.000000000000 |",
"| 4.000000000000 |",
"| 4.000000000000 |",
"| 4.000000000000 |",
"| 5.000000000000 |",
"| 5.000000000000 |",
"| 5.000000000000 |",
"| 5.000000000000 |",
"| 5.000000000000 |",
"+----------------------------------------------------------------+",
];
assert_batches_eq!(expected, &actual);

Expand Down Expand Up @@ -609,25 +609,25 @@ async fn decimal_arithmetic_op() -> Result<()> {
actual[0].schema().field(0).data_type()
);
let expected = vec![
"+-------------------------------------------------------------+",
"+----------------------------------------------------------------+",
"| decimal_simple.c5 % CAST(Float64(0.00001) AS Decimal128(5, 5)) |",
"+-------------------------------------------------------------+",
"| 0.0000040 |",
"| 0.0000050 |",
"| 0.0000090 |",
"| 0.0000020 |",
"| 0.0000050 |",
"| 0.0000010 |",
"| 0.0000040 |",
"| 0.0000000 |",
"| 0.0000000 |",
"| 0.0000040 |",
"| 0.0000020 |",
"| 0.0000080 |",
"| 0.0000030 |",
"| 0.0000080 |",
"| 0.0000000 |",
"+-------------------------------------------------------------+",
"+----------------------------------------------------------------+",
"| 0.0000040 |",
"| 0.0000050 |",
"| 0.0000090 |",
"| 0.0000020 |",
"| 0.0000050 |",
"| 0.0000010 |",
"| 0.0000040 |",
"| 0.0000000 |",
"| 0.0000000 |",
"| 0.0000040 |",
"| 0.0000020 |",
"| 0.0000080 |",
"| 0.0000030 |",
"| 0.0000080 |",
"| 0.0000000 |",
"+----------------------------------------------------------------+",
];
assert_batches_eq!(expected, &actual);

Expand Down
Loading

0 comments on commit cc19881

Please sign in to comment.