From 572d39d685062c464eed3f37416faeab60c00952 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Wed, 20 Jul 2022 20:27:59 +0800 Subject: [PATCH 1/9] fix codegen for unixtimestamp Signed-off-by: Yuan Zhou --- .../intel/oap/expression/ColumnarDateTimeExpressions.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala index a13650a1d..cec558315 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala @@ -485,6 +485,11 @@ object ColumnarDateTimeExpressions { } } + override def supportColumnarCodegen(args: Object): Boolean = { + false && left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = { val (leftNode, leftType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) val outType = CodeGeneration.getResultType(dataType) From 5de484fb7a3e3d150663f414cb8674b54352cbe7 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Fri, 5 Aug 2022 13:25:33 +0800 Subject: [PATCH 2/9] adding new supportcodegen check method Signed-off-by: Yuan Zhou --- .../ColumnarDateTimeExpressions.scala | 5 +- .../oap/expression/ColumnarExpression.scala | 62 ++++++++++++++++++- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala index cec558315..de338d322 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala @@ -453,6 +453,8 @@ object ColumnarDateTimeExpressions { extends UnixTimestamp(left, right, timeZoneId, failOnError) with ColumnarExpression { + val gName = "unix_seconds" + val yearMonthDayFormat = "yyyy-MM-dd" val yearMonthDayTimeFormat = "yyyy-MM-dd HH:mm:ss" val yearMonthDayTimeNoSepFormat = "yyyyMMddHHmmss" @@ -486,7 +488,8 @@ object ColumnarDateTimeExpressions { } override def supportColumnarCodegen(args: Object): Boolean = { - false && left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) } diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala index 967271768..4868bc2bd 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala @@ -27,9 +27,67 @@ import scala.collection.mutable.ListBuffer trait ColumnarExpression { + val codegenFuncList: List[String] = List( + "less_than", + "less_than_with_nan", + "greater_than", + "greater_than_with_nan", + "less_than_or_equal_to", + "less_than_or_equal_to_with_nan", + "greater_than_or_equal_to", + "greater_than_or_equal_to_with_nan", + "equal", + "equal_with_nan", + "not", + "isnotnull", + "isnull", + "starts_with", + "like", + "get_json_object", + "translate", + "substr", + "instr", + "btrim", + "ltrim", + "rtrim", + "upper", + "lower", + "castDATE", + "castDECIMAL", + "castDECIMALNullOnOverflow", + "castINTOrNull", + "castBIGINTOrNull", + "castFLOAT4OrNull", + "castFLOAT8OrNull", + "castDATE", + "castDECIMAL", + "castDECIMALNullOnOverflow", + "castINTOrNull", + "castBIGINTOrNull", + "castFLOAT4OrNull", + "castFLOAT8OrNull", + "castINTOrNull", + "castBIGINTOrNull", + "castFLOAT4OrNull", + "rescaleDECIMAL", + "extractYear", + "round", + "abs", + "add", + "subtract", + "multiply", + "divide", + "shift_left", + "shift_right", + "bitwise_and", + "bitwise_or", + "normalize", + "convertTimestampUnit", + "micros_to_timestamp" + ) + def supportColumnarCodegen(args: java.lang.Object): (Boolean) = { - // TODO: disable all codegen unless manuall enabled - true + false } def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = { From 41e530ddcc724358ed37d88d1ac12543409da1fe Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Fri, 5 Aug 2022 14:26:03 +0800 Subject: [PATCH 3/9] add missing codegen check Signed-off-by: Yuan Zhou --- .../oap/expression/ColumnarArithmetic.scala | 51 +++++++++++- .../expression/ColumnarBinaryExpression.scala | 8 ++ .../expression/ColumnarBinaryOperator.scala | 83 +++++++++++++++++-- .../ColumnarDateTimeExpressions.scala | 12 +++ .../oap/expression/ColumnarExpression.scala | 2 +- .../expression/ColumnarRoundOperator.scala | 7 ++ .../ColumnarString2TrimOperator.scala | 19 +++++ .../expression/ColumnarTernaryOperator.scala | 19 +++++ .../expression/ColumnarUnaryOperator.scala | 49 +++++++++-- 9 files changed, 236 insertions(+), 14 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarArithmetic.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarArithmetic.scala index 6331c2014..049641ab0 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarArithmetic.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarArithmetic.scala @@ -39,6 +39,13 @@ class ColumnarAdd(left: Expression, right: Expression, original: Expression) extends Add(left: Expression, right: Expression) with ColumnarExpression with Logging { + val gName = "add" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } // If casting between DecimalType, unnecessary cast is skipped to avoid data loss, // because actually res type of "cast" is the res type in "add/subtract", @@ -103,6 +110,14 @@ class ColumnarSubtract(left: Expression, right: Expression, original: Expression with ColumnarExpression with Logging { + val gName = "subtract" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + val left_val: Any = left match { case c: ColumnarCast => if (c.child.dataType.isInstanceOf[DecimalType] && @@ -163,6 +178,14 @@ class ColumnarMultiply(left: Expression, right: Expression, original: Expression with ColumnarExpression with Logging { + val gName = "multiply" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + val left_val: Any = left match { case c: ColumnarCast => if (c.child.dataType.isInstanceOf[DecimalType] && @@ -247,9 +270,6 @@ class ColumnarMultiply(left: Expression, right: Expression, original: Expression } } - override def supportColumnarCodegen(args: java.lang.Object): Boolean = { - return left_val.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && right_val.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) - } } class ColumnarDivide(left: Expression, right: Expression, @@ -257,6 +277,13 @@ class ColumnarDivide(left: Expression, right: Expression, extends Divide(left: Expression, right: Expression) with ColumnarExpression with Logging { + val gName = "divide" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } val left_val: Any = left match { case c: ColumnarCast => @@ -349,6 +376,15 @@ class ColumnarBitwiseAnd(left: Expression, right: Expression, original: Expressi extends BitwiseAnd(left: Expression, right: Expression) with ColumnarExpression with Logging { + + val gName = "bitwise_and" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = { val (left_node, left_type): (TreeNode, ArrowType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) @@ -370,6 +406,15 @@ class ColumnarBitwiseOr(left: Expression, right: Expression, original: Expressio extends BitwiseOr(left: Expression, right: Expression) with ColumnarExpression with Logging { + + val gName = "bitwise_or" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = { val (left_node, left_type): (TreeNode, ArrowType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryExpression.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryExpression.scala index 6fcd6a5a2..7402b37f8 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryExpression.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryExpression.scala @@ -103,6 +103,14 @@ class ColumnarGetJsonObject(left: Expression, right: Expression, original: GetJs class ColumnarStringInstr(left: Expression, right: Expression, original: StringInstr) extends StringInstr(original.str, original.substr) with ColumnarExpression with Logging { + val gName = "locate" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = { val (left_node, _): (TreeNode, ArrowType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryOperator.scala index 443c0b3e8..e64be10f3 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryOperator.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryOperator.scala @@ -103,6 +103,14 @@ class ColumnarStartsWith(left: Expression, right: Expression, original: Expressi extends StartsWith(left: Expression, right: Expression) with ColumnarExpression with Logging { + val gName = "starts_with" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = { val (left_node, left_type): (TreeNode, ArrowType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) @@ -121,6 +129,14 @@ class ColumnarLike(left: Expression, right: Expression, original: Expression) with ColumnarExpression with Logging { + val gName = "like" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + buildCheck() def buildCheck(): Unit = { @@ -154,7 +170,14 @@ class ColumnarRLike(left: Expression, right: Expression, original: Expression) extends RLike(left: Expression, right: Expression) with ColumnarExpression with Logging { + val gName = "equal" + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + buildCheck() def buildCheck(): Unit = { @@ -164,10 +187,6 @@ class ColumnarRLike(left: Expression, right: Expression, original: Expression) } } - override def supportColumnarCodegen(args: java.lang.Object): Boolean = { - false - } - override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = { val (left_node, left_type): (TreeNode, ArrowType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) @@ -206,6 +225,14 @@ class ColumnarEqualTo(left: Expression, right: Expression, original: Expression) extends EqualTo(left: Expression, right: Expression) with ColumnarExpression with Logging { + val gName = "equal" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = { var (left_node, left_type): (TreeNode, ArrowType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) @@ -305,8 +332,11 @@ class ColumnarLessThan(left: Expression, right: Expression, original: Expression with ColumnarExpression with Logging { + val gName = "less_than" + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { - true && left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) } @@ -352,6 +382,14 @@ class ColumnarLessThanOrEqual(left: Expression, right: Expression, original: Exp extends LessThanOrEqual(left: Expression, right: Expression) with ColumnarExpression with Logging { + val gName = "less_than_or_equal_to" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = { var (left_node, left_type): (TreeNode, ArrowType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) @@ -396,6 +434,15 @@ class ColumnarGreaterThan(left: Expression, right: Expression, original: Express extends GreaterThan(left: Expression, right: Expression) with ColumnarExpression with Logging { + + val gName = "greater_than" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = { var (left_node, left_type): (TreeNode, ArrowType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) @@ -440,6 +487,14 @@ class ColumnarGreaterThanOrEqual(left: Expression, right: Expression, original: extends GreaterThanOrEqual(left: Expression, right: Expression) with ColumnarExpression with Logging { + val gName = "greater_than_or_equal_to" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = { var (left_node, left_type): (TreeNode, ArrowType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) @@ -484,6 +539,15 @@ class ColumnarShiftLeft(left: Expression, right: Expression, original: Expressio extends ShiftLeft(left: Expression, right: Expression) with ColumnarExpression with Logging { + + val gName = "shift_left" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = { var (left_node, left_type): (TreeNode, ArrowType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) @@ -506,6 +570,15 @@ class ColumnarShiftRight(left: Expression, right: Expression, original: Expressi extends ShiftRight(left: Expression, right: Expression) with ColumnarExpression with Logging { + + val gName = "shift_right" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = { val (left_node, left_type): (TreeNode, ArrowType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala index de338d322..937193ebe 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala @@ -255,6 +255,12 @@ object ColumnarDateTimeExpressions { class ColumnarYear(child: Expression) extends Year(child) with ColumnarExpression { + val gName = "extractYear" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() @@ -420,6 +426,12 @@ object ColumnarDateTimeExpressions { class ColumnarMicrosToTimestamp(child: Expression) extends MicrosToTimestamp(child) with ColumnarExpression { + val gName = "micros_to_timestamp" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala index 4868bc2bd..371acd47a 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala @@ -28,7 +28,7 @@ import scala.collection.mutable.ListBuffer trait ColumnarExpression { val codegenFuncList: List[String] = List( - "less_than", + "less_than", "less_than_with_nan", "greater_than", "greater_than_with_nan", diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarRoundOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarRoundOperator.scala index ec7808a68..1cb733a6e 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarRoundOperator.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarRoundOperator.scala @@ -38,6 +38,13 @@ class ColumnarRound(child: Expression, scale: Expression, original: Expression) extends Round(child: Expression, scale: Expression) with ColumnarExpression with Logging { + val gName = "round" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + scale.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarString2TrimOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarString2TrimOperator.scala index 5e2c18c20..a47ae226b 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarString2TrimOperator.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarString2TrimOperator.scala @@ -32,6 +32,13 @@ class ColumnarStringTrim(srcStr: Expression, trimStr: Option[Expression], origin extends StringTrim(srcStr: Expression, trimStr: Option[Expression]) with ColumnarExpression with Logging { + val gName = "btrim" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + trimStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() @@ -63,7 +70,13 @@ class ColumnarStringTrimLeft(srcStr: Expression, trimStr: Option[Expression], or extends StringTrimLeft(srcStr: Expression, trimStr: Option[Expression]) with ColumnarExpression with Logging { + val gName = "ltrim" + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + trimStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() def buildCheck(): Unit = { @@ -93,7 +106,13 @@ class ColumnarStringTrimRight(child: Expression, trimStr: Option[Expression], or extends StringTrimRight(child: Expression, trimStr: Option[Expression]) with ColumnarExpression with Logging { + val gName = "rtrim" + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + trimStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() def buildCheck(): Unit = { diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarTernaryOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarTernaryOperator.scala index 2bb88571e..e16987be0 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarTernaryOperator.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarTernaryOperator.scala @@ -39,6 +39,16 @@ class ColumnarSubString(str: Expression, pos: Expression, len: Expression, origi with ColumnarExpression with Logging { + + val gName = "substr" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + str.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + pos.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + len.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + buildCheck() def buildCheck(): Unit = { @@ -117,6 +127,15 @@ class ColumnarStringTranslate(src: Expression, matchingExpr: Expression, replaceExpr: Expression, original: Expression) extends StringTranslate(src, matchingExpr, replaceExpr) with ColumnarExpression { + val gName = "translate" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + src.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + matchingExpr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + replaceExpr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + buildCheck def buildCheck: Unit = { diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala index bd847b969..db1c117fb 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala @@ -58,6 +58,12 @@ class ColumnarIsNotNull(child: Expression, original: Expression) extends IsNotNull(child: Expression) with ColumnarExpression with Logging { + val gName = "isnotnull" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() @@ -92,10 +98,17 @@ class ColumnarIsNotNull(child: Expression, original: Expression) } class ColumnarIsNull(child: Expression, original: Expression) - extends IsNotNull(child: Expression) + extends IsNull(child: Expression) with ColumnarExpression with Logging { + val gName = "isnull" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + buildCheck() def buildCheck(): Unit = { @@ -282,6 +295,12 @@ class ColumnarAbs(child: Expression, original: Expression) extends Abs(child: Expression) with ColumnarExpression with Logging { + val gName = "abs" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() @@ -369,6 +388,12 @@ class ColumnarUpper(child: Expression, original: Expression) extends Upper(child: Expression) with ColumnarExpression with Logging { + val gName = "upper" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() @@ -395,6 +420,12 @@ class ColumnarLower(child: Expression, original: Expression) extends Lower(child: Expression) with ColumnarExpression with Logging { + val gName = "lower" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() @@ -415,10 +446,6 @@ class ColumnarLower(child: Expression, original: Expression) TreeBuilder.makeFunction("lower", Lists.newArrayList(child_node), resultType) (funcNode, resultType) } - - override def supportColumnarCodegen(args: java.lang.Object): Boolean = { - false - } } class ColumnarBitwiseNot(child: Expression, original: Expression) @@ -492,6 +519,12 @@ class ColumnarCast( extends Cast(child: Expression, datatype: DataType, timeZoneId: Option[String]) with ColumnarExpression with Logging { + val gName = "Cast" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + true && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() @@ -874,6 +907,12 @@ class ColumnarNormalizeNaNAndZero(child: Expression, original: NormalizeNaNAndZe extends NormalizeNaNAndZero(child: Expression) with ColumnarExpression with Logging { + val gName = "normalize" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() From f4b1b8c6eb6f0da5228fbce70f12e13ba2cecee3 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Tue, 20 Sep 2022 13:40:07 +0800 Subject: [PATCH 4/9] enable more codegen Signed-off-by: Yuan Zhou --- .../com/intel/oap/expression/ColumnarExpression.scala | 10 ---------- .../com/intel/oap/expression/ColumnarLiterals.scala | 4 ++++ .../oap/expression/ColumnarNamedExpressions.scala | 4 ++++ .../intel/oap/expression/ColumnarUnaryOperator.scala | 7 +++++++ 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala index 371acd47a..07989caf9 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala @@ -59,16 +59,6 @@ trait ColumnarExpression { "castBIGINTOrNull", "castFLOAT4OrNull", "castFLOAT8OrNull", - "castDATE", - "castDECIMAL", - "castDECIMALNullOnOverflow", - "castINTOrNull", - "castBIGINTOrNull", - "castFLOAT4OrNull", - "castFLOAT8OrNull", - "castINTOrNull", - "castBIGINTOrNull", - "castFLOAT4OrNull", "rescaleDECIMAL", "extractYear", "round", diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarLiterals.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarLiterals.scala index 5e5070823..4c025a636 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarLiterals.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarLiterals.scala @@ -38,6 +38,10 @@ class ColumnarLiteral(lit: Literal) extends Literal(lit.value, lit.dataType) with ColumnarExpression { + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + true + } + val resultType: ArrowType = buildCheck() def buildCheck(): ArrowType = { diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarNamedExpressions.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarNamedExpressions.scala index 0032d59b2..3d50a011d 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarNamedExpressions.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarNamedExpressions.scala @@ -55,6 +55,10 @@ class ColumnarAttributeReference( extends AttributeReference(name, dataType, nullable, metadata)(exprId, qualifier) with ColumnarExpression { + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + true + } + override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = { val resultType = CodeGeneration.getResultType(dataType) val field = Field.nullable(s"${name}#${exprId.id}", resultType) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala index fa34e2572..6ea4a9bc0 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala @@ -848,6 +848,13 @@ class ColumnarUnscaledValue(child: Expression, original: Expression) with ColumnarExpression with Logging { + val gName = "multiply" + + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } + buildCheck() def buildCheck(): Unit = { From cd1d310ec72a872ce4e6df8753d639ed42a7aaee Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Tue, 20 Sep 2022 13:58:05 +0800 Subject: [PATCH 5/9] enable more codegen Signed-off-by: Yuan Zhou --- .../com/intel/oap/expression/ColumnarCaseWhenOperator.scala | 3 --- .../com/intel/oap/expression/ColumnarUnaryOperator.scala | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala index dcc097af4..55814db8b 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala @@ -64,9 +64,6 @@ class ColumnarCaseWhen( } override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = { - logInfo(s"children: ${branches.flatMap(b => b._1 :: b._2 :: Nil) ++ elseValue}") - logInfo(s"branches: $branches") - logInfo(s"else: $elseValue") val i = 0 val exprs = branches.flatMap(b => b._1 :: b._2 :: Nil) ++ elseValue val exprList = { exprs.filter(expr => !expr.isInstanceOf[Literal]) } diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala index 6ea4a9bc0..02f99365c 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUnaryOperator.scala @@ -480,7 +480,12 @@ class ColumnarCheckOverflow(child: Expression, original: CheckOverflow) original.nullOnOverflow: Boolean) with ColumnarExpression with Logging { + val gName = "castDECIMAL" + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + codegenFuncList.contains(gName) && + child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } buildCheck() def buildCheck(): Unit = { From 440245678b358099d010c3e2bc706b92a27c2b57 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Tue, 20 Sep 2022 14:56:10 +0800 Subject: [PATCH 6/9] enable more codegen Signed-off-by: Yuan Zhou --- .../oap/expression/ColumnarCoalesceOperator.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCoalesceOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCoalesceOperator.scala index 00b422364..43c2d934c 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCoalesceOperator.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCoalesceOperator.scala @@ -46,6 +46,16 @@ class ColumnarCoalesce(exps: Seq[Expression], original: Expression) with ColumnarExpression with Logging { + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + for (expr <- exps) { + val colExpr = ColumnarExpressionConverter.replaceWithColumnarExpression(expr) + if (!colExpr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(Lists.newArrayList())) { + return false + } + } + return true + } + buildCheck() def buildCheck(): Unit = { From 7f38a8e39f24cc93c479b9e7621476453d140ebc Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Tue, 20 Sep 2022 15:03:35 +0800 Subject: [PATCH 7/9] fix in string Signed-off-by: Yuan Zhou --- .../scala/com/intel/oap/expression/ColumnarExpression.scala | 2 +- .../src/codegen/arrow_compute/ext/expression_codegen_visitor.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala index 07989caf9..b7b911767 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala @@ -46,7 +46,7 @@ trait ColumnarExpression { "get_json_object", "translate", "substr", - "instr", + "locate", "btrim", "ltrim", "rtrim", diff --git a/native-sql-engine/cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.cc index b60aaa624..265dba5e1 100644 --- a/native-sql-engine/cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.cc +++ b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/expression_codegen_visitor.cc @@ -324,7 +324,7 @@ arrow::Status ExpressionCodegenVisitor::Visit(const gandiva::FunctionNode& node) prepare_ss << "if (" << check_str_ << ")" << std::endl; prepare_ss << codes_str_ << " = " << ss.str() << ";" << std::endl; prepare_str_ += prepare_ss.str(); - } else if (func_name.compare("instr") == 0) { + } else if (func_name.compare("locate") == 0) { codes_str_ = func_name + "_" + std::to_string(cur_func_id); auto validity = codes_str_ + "_validity"; real_codes_str_ = codes_str_; From cfc35ae5bbb2a435cea0c668352f6548f10a3ed0 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Tue, 20 Sep 2022 15:50:43 +0800 Subject: [PATCH 8/9] fix comments for case when Signed-off-by: Yuan Zhou --- .../com/intel/oap/expression/ColumnarCaseWhenOperator.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala index 55814db8b..5606c9544 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarCaseWhenOperator.scala @@ -33,7 +33,7 @@ import org.apache.spark.sql.types._ import scala.collection.mutable.ListBuffer /** - * A version of substring that supports columnar processing for utf8. + * Columnar impl for Case When base on Gandiva if/else */ class ColumnarCaseWhen( branches: Seq[(Expression, Expression)], From fc2c23e7402137f578b43cc2048801591ac9a853 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Tue, 20 Sep 2022 22:47:52 +0800 Subject: [PATCH 9/9] fix codegen check for trim func Signed-off-by: Yuan Zhou --- .../ColumnarString2TrimOperator.scala | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarString2TrimOperator.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarString2TrimOperator.scala index a47ae226b..55e244374 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarString2TrimOperator.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarString2TrimOperator.scala @@ -35,9 +35,14 @@ class ColumnarStringTrim(srcStr: Expression, trimStr: Option[Expression], origin val gName = "btrim" override def supportColumnarCodegen(args: java.lang.Object): Boolean = { - codegenFuncList.contains(gName) && - srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && - trimStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + if (trimStr.isEmpty) { + codegenFuncList.contains(gName) && + srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } else { + codegenFuncList.contains(gName) && + srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + trimStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } } buildCheck() @@ -73,9 +78,14 @@ class ColumnarStringTrimLeft(srcStr: Expression, trimStr: Option[Expression], or val gName = "ltrim" override def supportColumnarCodegen(args: java.lang.Object): Boolean = { - codegenFuncList.contains(gName) && - srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && - trimStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + if (trimStr.isEmpty) { + codegenFuncList.contains(gName) && + srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } else { + codegenFuncList.contains(gName) && + srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + trimStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } } buildCheck() @@ -109,9 +119,14 @@ class ColumnarStringTrimRight(child: Expression, trimStr: Option[Expression], or val gName = "rtrim" override def supportColumnarCodegen(args: java.lang.Object): Boolean = { - codegenFuncList.contains(gName) && - srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && - trimStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + if (trimStr.isEmpty) { + codegenFuncList.contains(gName) && + srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } else { + codegenFuncList.contains(gName) && + srcStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && + trimStr.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) + } } buildCheck()