From a26ab5a55d281a4981cf08950930ee658946fb6f Mon Sep 17 00:00:00 2001 From: haojinIntel Date: Mon, 10 Jan 2022 16:14:45 +0800 Subject: [PATCH] [NSE-400] Fix the bug for negative decimal data (#686) * Fix the bug for positive decimal data * Fix format * Fix the unit test for 'SPARK-22348: table cache should do partition batch pruning (whole-stage-codegen on)' --- .../columnar/InMemoryColumnarQuerySuite.scala | 6 +++--- .../cpp/src/operators/row_to_columnar_converter.cc | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala index 4d753e94c..2ae01caf2 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/execution/columnar/InMemoryColumnarQuerySuite.scala @@ -20,8 +20,8 @@ package org.apache.spark.sql.execution.columnar import java.nio.charset.StandardCharsets import java.sql.{Date, Timestamp} -import com.intel.oap.execution.ColumnarConditionProjectExec -import com.intel.oap.sql.execution.RowToArrowColumnarExec +import com.intel.oap.execution.{ArrowRowToColumnarExec, ColumnarConditionProjectExec} + import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, QueryTest, Row} import org.apache.spark.sql.catalyst.InternalRow @@ -506,7 +506,7 @@ class InMemoryColumnarQuerySuite extends QueryTest with SharedSparkSession { val df2 = df1.where("y = 3") val planBeforeFilter = df2.queryExecution.executedPlan.collect { - case ColumnarConditionProjectExec(_, _, c: RowToArrowColumnarExec) => c.child + case ColumnarConditionProjectExec(_, _, c: ArrowRowToColumnarExec) => c.child case FilterExec(_, c: ColumnarToRowExec) => c.child case WholeStageCodegenExec(FilterExec(_, ColumnarToRowExec(i: InputAdapter))) => i.child } diff --git a/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc b/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc index 18928a481..2cbcd618a 100644 --- a/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc +++ b/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc @@ -393,6 +393,11 @@ arrow::Status CreateArrayData(std::shared_ptr schema, int64_t num for (int k = length - 1; k >= 0; k--) { bytesValue2[length - 1 - k] = bytesValue[k]; } + if (int8_t(bytesValue[0]) < 0) { + for (int k = length; k < 16; k++) { + bytesValue2[k] = 255; + } + } arrow::Decimal128 value = arrow::Decimal128(arrow::BasicDecimal128(bytesValue2)); array_data[position] = value; @@ -950,6 +955,11 @@ arrow::Status CreateArrayData(std::shared_ptr schema, int64_t num for (int k = elementLength - 1; k >= 0; k--) { bytesValue2[elementLength - 1 - k] = bytesValue[k]; } + if (int8_t(bytesValue[0]) < 0) { + for (int k = elementLength; k < 16; k++) { + bytesValue2[k] = 255; + } + } arrow::Decimal128 value = arrow::Decimal128(arrow::BasicDecimal128(bytesValue2)); RETURN_NOT_OK(child_builder.Append(value));