Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

[NSE-400] Fix the bug for negative decimal data #686

Merged
merged 3 commits into from
Jan 10, 2022
Merged

[NSE-400] Fix the bug for negative decimal data #686

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a66e63d
Fix the bug for positive decimal data
haojinIntel Jan 10, 2022
02c1ba7
Fix format
haojinIntel Jan 10, 2022
0386e02
Fix the unit test for 'SPARK-22348: table cache should do partition b…
haojinIntel Jan 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ...e/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ package org.apache.spark.sql.execution.columnar
import java.nio.charset.StandardCharsets
import java.sql.{Date, Timestamp}

import com.intel.oap.execution.ColumnarConditionProjectExec
import com.intel.oap.sql.execution.RowToArrowColumnarExec
import com.intel.oap.execution.{ArrowRowToColumnarExec, ColumnarConditionProjectExec}

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, QueryTest, Row}
import org.apache.spark.sql.catalyst.InternalRow
Expand Down Expand Up @@ -506,7 +506,7 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession {
val df2 = df1.where("y = 3")

val planBeforeFilter = df2.queryExecution.executedPlan.collect {
case ColumnarConditionProjectExec(_, _, c: RowToArrowColumnarExec) => c.child
case ColumnarConditionProjectExec(_, _, c: ArrowRowToColumnarExec) => c.child
case FilterExec(_, c: ColumnarToRowExec) => c.child
case WholeStageCodegenExec(FilterExec(_, ColumnarToRowExec(i: InputAdapter))) => i.child
}
Expand Down
10 changes: 10 additions & 0 deletions native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,11 @@ arrow::Status CreateArrayData(std::shared_ptr<arrow::Schema> schema, int64_t num
for (int k = length - 1; k >= 0; k--) {
bytesValue2[length - 1 - k] = bytesValue[k];
}
if (int8_t(bytesValue[0]) < 0) {
for (int k = length; k < 16; k++) {
bytesValue2[k] = 255;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

note: for negative decimal, all unused bits are set to 1

}
}
arrow::Decimal128 value =
arrow::Decimal128(arrow::BasicDecimal128(bytesValue2));
array_data[position] = value;
Expand Down Expand Up @@ -950,6 +955,11 @@ arrow::Status CreateArrayData(std::shared_ptr<arrow::Schema> schema, int64_t num
for (int k = elementLength - 1; k >= 0; k--) {
bytesValue2[elementLength - 1 - k] = bytesValue[k];
}
if (int8_t(bytesValue[0]) < 0) {
for (int k = elementLength; k < 16; k++) {
bytesValue2[k] = 255;
}
}
arrow::Decimal128 value =
arrow::Decimal128(arrow::BasicDecimal128(bytesValue2));
RETURN_NOT_OK(child_builder.Append(value));
Expand Down