Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions native/spark-expr/src/string_funcs/string_space.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,14 @@ fn generic_string_space<OffsetSize: OffsetSizeTrait>(length: &Int32Array) -> Arr
let null_bit_buffer = length.to_data().nulls().map(|b| b.buffer().clone());

// Gets slice of length array to access it directly for performance.
// Negative length values are set to zero to match Spark behavior
let length_data = length.to_data();
let lengths = length_data.buffers()[0].typed_data::<i32>();
let total = lengths.iter().map(|l| *l as usize).sum::<usize>();
let lengths: Vec<_> = length_data.buffers()[0]
.typed_data::<i32>()
.iter()
.map(|l| (*l).max(0) as usize)
.collect();
let total = lengths.iter().sum::<usize>();
let mut values = MutableBuffer::new(total);

offsets.push(length_so_far);
Expand All @@ -130,7 +135,7 @@ fn generic_string_space<OffsetSize: OffsetSizeTrait>(length: &Int32Array) -> Arr
values.resize(total, blank);

(0..array_len).for_each(|i| {
let current_len = lengths[i] as usize;
let current_len = lengths[i];

length_so_far += OffsetSize::from_usize(current_len).unwrap();
offsets.push(length_so_far);
Expand All @@ -149,3 +154,24 @@ fn generic_string_space<OffsetSize: OffsetSizeTrait>(length: &Int32Array) -> Arr
};
make_array(data)
}

#[cfg(test)]
mod tests {
use super::*;
use arrow::array::StringArray;
use datafusion::common::cast::as_string_array;

#[test]
fn test_negative_length() {
let input = Int32Array::from(vec![Some(-1), Some(-2), None]);
let args = ColumnarValue::Array(Arc::new(input));
match spark_string_space(&[args]) {
Ok(ColumnarValue::Array(result)) => {
let actual = as_string_array(&result).unwrap();
let expected = StringArray::from(vec![Some(""), Some(""), None]);
assert_eq!(actual, &expected)
}
_ => unreachable!(),
}
}
}
2 changes: 1 addition & 1 deletion native/spark-expr/src/string_funcs/substring.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ impl Display for SubstringExpr {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
f,
"StringSpace [start: {}, len: {}, child: {}]",
"Substring [start: {}, len: {}, child: {}]",
self.start, self.len, self.child
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,7 @@ class CometStringExpressionSuite extends CometTestBase {
}

test("string_space") {
withParquetTable((0 until 5).map(i => (i, i + 1)), "tbl") {
withParquetTable((0 until 5).map(i => (-i, i + 1)), "tbl") {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will change string_space.sql once #3328 is merged.

checkSparkAnswerAndOperator("SELECT space(_1), space(_2) FROM tbl")
}
}
Expand Down
Loading