Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

parquet: Optimized ByteArrayReader, Add UTF-8 Validation (#1040) #1082

Merged
merged 18 commits into from
Jan 18, 2022
+1,299 −162
Merged

parquet: Optimized ByteArrayReader, Add UTF-8 Validation (#1040) #1082

Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
585d0f5
Optimized ByteArrayReader (#1040)
tustvold Dec 19, 2021
503f31f
Fix arrow_array_reader benchmark
tustvold Jan 16, 2022
38da212
Allow running subset of arrow_array_reader benchmarks
tustvold Jan 16, 2022
3d59a4c
Faster UTF-8 validation
tustvold Jan 16, 2022
f510f8e
Tweak null handling
tustvold Jan 16, 2022
9626728
Add license
tustvold Jan 16, 2022
014be9f
Merge remote-tracking branch 'upstream/master' into string-array-reader
tustvold Jan 16, 2022
0ba1ca7
Refine `ValuesBuffer::pad_nulls`
tustvold Jan 16, 2022
23887f5
Tweak error handling
tustvold Jan 16, 2022
7812799
Use page null count if available
tustvold Jan 16, 2022
2057048
Merge remote-tracking branch 'upstream/master' into string-array-reader
tustvold Jan 16, 2022
9f6bcac
Doc comments
tustvold Jan 16, 2022
8e27ce2
Test DELTA_BYTE_ARRAY encoding
tustvold Jan 16, 2022
a12babb
Support legacy Encoding::PLAIN_DICTIONARY
tustvold Jan 16, 2022
971622c
Add OffsetBuffer unit tests
tustvold Jan 16, 2022
dbf7b56
More tests
tustvold Jan 17, 2022
cf099c3
Fix lint
tustvold Jan 17, 2022
c941606
Review feedback
tustvold Jan 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 24 additions & 24 deletions parquet/benches/arrow_array_reader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -366,10 +366,10 @@ fn add_benches(c: &mut Criterion) {
mandatory_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
tustvold marked this conversation as resolved.
Show resolved Hide resolved
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read Int32Array, plain encoded, mandatory, no NULLs - new",
Expand All @@ -380,10 +380,10 @@ fn add_benches(c: &mut Criterion) {
mandatory_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

let plain_int32_no_null_data = build_plain_encoded_int32_page_iterator(
schema.clone(),
Expand All @@ -399,10 +399,10 @@ fn add_benches(c: &mut Criterion) {
optional_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read Int32Array, plain encoded, optional, no NULLs - new",
Expand All @@ -413,10 +413,10 @@ fn add_benches(c: &mut Criterion) {
optional_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

// int32, plain encoded, half NULLs
let plain_int32_half_null_data = build_plain_encoded_int32_page_iterator(
Expand All @@ -433,10 +433,10 @@ fn add_benches(c: &mut Criterion) {
optional_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read Int32Array, plain encoded, optional, half NULLs - new",
Expand All @@ -447,10 +447,10 @@ fn add_benches(c: &mut Criterion) {
optional_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

// int32, dictionary encoded, no NULLs
let dictionary_int32_no_null_data = build_dictionary_encoded_int32_page_iterator(
Expand All @@ -467,10 +467,10 @@ fn add_benches(c: &mut Criterion) {
mandatory_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read Int32Array, dictionary encoded, mandatory, no NULLs - new",
Expand All @@ -481,10 +481,10 @@ fn add_benches(c: &mut Criterion) {
mandatory_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

let dictionary_int32_no_null_data = build_dictionary_encoded_int32_page_iterator(
schema.clone(),
Expand All @@ -500,10 +500,10 @@ fn add_benches(c: &mut Criterion) {
optional_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read Int32Array, dictionary encoded, optional, no NULLs - new",
Expand All @@ -514,10 +514,10 @@ fn add_benches(c: &mut Criterion) {
optional_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

// int32, dictionary encoded, half NULLs
let dictionary_int32_half_null_data = build_dictionary_encoded_int32_page_iterator(
Expand All @@ -534,10 +534,10 @@ fn add_benches(c: &mut Criterion) {
optional_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read Int32Array, dictionary encoded, optional, half NULLs - new",
Expand All @@ -548,10 +548,10 @@ fn add_benches(c: &mut Criterion) {
optional_int32_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

// string benchmarks
//==============================
Expand All @@ -571,10 +571,10 @@ fn add_benches(c: &mut Criterion) {
mandatory_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read StringArray, plain encoded, mandatory, no NULLs - new",
Expand All @@ -585,10 +585,10 @@ fn add_benches(c: &mut Criterion) {
mandatory_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

let plain_string_no_null_data = build_plain_encoded_string_page_iterator(
schema.clone(),
Expand All @@ -604,10 +604,10 @@ fn add_benches(c: &mut Criterion) {
optional_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read StringArray, plain encoded, optional, no NULLs - new",
Expand All @@ -618,10 +618,10 @@ fn add_benches(c: &mut Criterion) {
optional_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

// string, plain encoded, half NULLs
let plain_string_half_null_data = build_plain_encoded_string_page_iterator(
Expand All @@ -638,10 +638,10 @@ fn add_benches(c: &mut Criterion) {
optional_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read StringArray, plain encoded, optional, half NULLs - new",
Expand All @@ -652,10 +652,10 @@ fn add_benches(c: &mut Criterion) {
optional_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

// string, dictionary encoded, no NULLs
let dictionary_string_no_null_data = build_dictionary_encoded_string_page_iterator(
Expand All @@ -672,10 +672,10 @@ fn add_benches(c: &mut Criterion) {
mandatory_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read StringArray, dictionary encoded, mandatory, no NULLs - new",
Expand All @@ -686,10 +686,10 @@ fn add_benches(c: &mut Criterion) {
mandatory_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

let dictionary_string_no_null_data = build_dictionary_encoded_string_page_iterator(
schema.clone(),
Expand All @@ -705,10 +705,10 @@ fn add_benches(c: &mut Criterion) {
optional_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read StringArray, dictionary encoded, optional, no NULLs - new",
Expand All @@ -719,10 +719,10 @@ fn add_benches(c: &mut Criterion) {
optional_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

// string, dictionary encoded, half NULLs
let dictionary_string_half_null_data = build_dictionary_encoded_string_page_iterator(
Expand All @@ -739,10 +739,10 @@ fn add_benches(c: &mut Criterion) {
optional_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.bench_function(
"read StringArray, dictionary encoded, optional, half NULLs - new",
Expand All @@ -753,10 +753,10 @@ fn add_benches(c: &mut Criterion) {
optional_string_column_desc.clone(),
);
count = bench_array_reader(array_reader);
assert_eq!(count, EXPECTED_VALUE_COUNT);
})
},
);
assert_eq!(count, EXPECTED_VALUE_COUNT);

group.finish();
}
Expand Down