Skip to content

Commit

Permalink
Add numeric limits tests for statistics reading
Browse files Browse the repository at this point in the history
  • Loading branch information
alamb committed May 23, 2024
1 parent 1d60257 commit de935ed
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 0 deletions.
20 changes: 20 additions & 0 deletions datafusion/core/tests/parquet/arrow_statistics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -802,6 +802,26 @@ async fn test_uint32_range() {
.run();
}

#[tokio::test]
async fn test_numeric_limits() {
// This creates a parquet file of 1 column "u"
// file has 7 rows, 2 row groups: one with 5 rows, one with 2 rows.
let reader = TestReader {
scenario: Scenario::NumericLimits,
row_per_group: 5,
};

Test {
reader: reader.build().await,
expected_min: Arc::new(Int8Array::from(vec![i8::MIN, -100])),
expected_max: Arc::new(Int8Array::from(vec![100, i8::MAX])),
expected_null_counts: UInt64Array::from(vec![0,0]),
expected_row_counts: UInt64Array::from(vec![5,2]),
column_name: "i8",
}
.run();
}

#[tokio::test]
async fn test_float64() {
// This creates a parquet file of 1 column "f"
Expand Down
11 changes: 11 additions & 0 deletions datafusion/core/tests/parquet/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ enum Scenario {
Int32Range,
UInt,
UInt32Range,
/// 7 Rows, for each i8, i16, i32, i64, u8, u16, u32, u64, f32, f64
/// -MIN, -100, -1, 0, 1, 100, MAX
NumericLimits,
Float64,
Decimal,
DecimalBloomFilterInt32,
Expand Down Expand Up @@ -768,6 +771,14 @@ fn create_data_batch(scenario: Scenario) -> Vec<RecordBatch> {
Scenario::UInt32Range => {
vec![make_uint32_range(0, 10), make_uint32_range(200000, 300000)]
}
Scenario::NumericLimits => {
let i8: ArrayRef = Arc::new(Int8Array::from(vec![i8::MIN, 100, -1, 0, 1, -100, i8::MAX]));
vec![
RecordBatch::try_from_iter(vec![
("i8", i8)
]).unwrap()
]
}
Scenario::Float64 => {
vec![
make_f64_batch(vec![-5.0, -4.0, -3.0, -2.0, -1.0]),
Expand Down

0 comments on commit de935ed

Please sign in to comment.