Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

[NSE-1019] fix codegen for all expressions #1039

Merged
merged 10 commits into from
Sep 21, 2022
Merged

[NSE-1019] fix codegen for all expressions #1039

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
572d39d
fix codegen for unixtimestamp
zhouyuan Jul 20, 2022
5de484f
adding new supportcodegen check method
zhouyuan Aug 5, 2022
41e530d
add missing codegen check
zhouyuan Aug 5, 2022
7f80f01
Merge branch 'main' into wip_unixtimestamp
zhouyuan Sep 19, 2022
f4b1b8c
enable more codegen
zhouyuan Sep 20, 2022
cd1d310
enable more codegen
zhouyuan Sep 20, 2022
4402456
enable more codegen
zhouyuan Sep 20, 2022
7f38a8e
fix in string
zhouyuan Sep 20, 2022
cfc35ae
fix comments for case when
zhouyuan Sep 20, 2022
fc2c23e
fix codegen check for trim func
zhouyuan Sep 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 48 additions & 3 deletions native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarArithmetic.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,13 @@ class ColumnarAdd(left: Expression, right: Expression, original: Expression)
extends Add(left: Expression, right: Expression)
with ColumnarExpression
with Logging {
val gName = "add"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

// If casting between DecimalType, unnecessary cast is skipped to avoid data loss,
// because actually res type of "cast" is the res type in "add/subtract",
Expand Down Expand Up @@ -103,6 +110,14 @@ class ColumnarSubtract(left: Expression, right: Expression, original: Expression
with ColumnarExpression
with Logging {

val gName = "subtract"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

val left_val: Any = left match {
case c: ColumnarCast =>
if (c.child.dataType.isInstanceOf[DecimalType] &&
Expand Down Expand Up @@ -163,6 +178,14 @@ class ColumnarMultiply(left: Expression, right: Expression, original: Expression
with ColumnarExpression
with Logging {

val gName = "multiply"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

val left_val: Any = left match {
case c: ColumnarCast =>
if (c.child.dataType.isInstanceOf[DecimalType] &&
Expand Down Expand Up @@ -247,16 +270,20 @@ class ColumnarMultiply(left: Expression, right: Expression, original: Expression
}
}

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
return left_val.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) && right_val.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}
}

class ColumnarDivide(left: Expression, right: Expression,
original: Expression, resType: DecimalType = null)
extends Divide(left: Expression, right: Expression)
with ColumnarExpression
with Logging {
val gName = "divide"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

val left_val: Any = left match {
case c: ColumnarCast =>
Expand Down Expand Up @@ -349,6 +376,15 @@ class ColumnarBitwiseAnd(left: Expression, right: Expression, original: Expressi
extends BitwiseAnd(left: Expression, right: Expression)
with ColumnarExpression
with Logging {

val gName = "bitwise_and"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = {
val (left_node, left_type): (TreeNode, ArrowType) =
left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
Expand All @@ -370,6 +406,15 @@ class ColumnarBitwiseOr(left: Expression, right: Expression, original: Expressio
extends BitwiseOr(left: Expression, right: Expression)
with ColumnarExpression
with Logging {

val gName = "bitwise_or"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = {
val (left_node, left_type): (TreeNode, ArrowType) =
left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
Expand Down
8 changes: 8 additions & 0 deletions ...ve-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryExpression.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ class ColumnarGetJsonObject(left: Expression, right: Expression, original: GetJs
class ColumnarStringInstr(left: Expression, right: Expression, original: StringInstr)
extends StringInstr(original.str, original.substr) with ColumnarExpression with Logging {

val gName = "locate"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = {
val (left_node, _): (TreeNode, ArrowType) =
left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
Expand Down
83 changes: 78 additions & 5 deletions native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarBinaryOperator.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,14 @@ class ColumnarStartsWith(left: Expression, right: Expression, original: Expressi
extends StartsWith(left: Expression, right: Expression)
with ColumnarExpression
with Logging {
val gName = "starts_with"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = {
val (left_node, left_type): (TreeNode, ArrowType) =
left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
Expand All @@ -121,6 +129,14 @@ class ColumnarLike(left: Expression, right: Expression, original: Expression)
with ColumnarExpression
with Logging {

val gName = "like"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

buildCheck()

def buildCheck(): Unit = {
Expand Down Expand Up @@ -154,7 +170,14 @@ class ColumnarRLike(left: Expression, right: Expression, original: Expression)
extends RLike(left: Expression, right: Expression)
with ColumnarExpression
with Logging {
val gName = "equal"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

buildCheck()

def buildCheck(): Unit = {
Expand All @@ -164,10 +187,6 @@ class ColumnarRLike(left: Expression, right: Expression, original: Expression)
}
}

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
false
}

override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = {
val (left_node, left_type): (TreeNode, ArrowType) =
left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
Expand Down Expand Up @@ -206,6 +225,14 @@ class ColumnarEqualTo(left: Expression, right: Expression, original: Expression)
extends EqualTo(left: Expression, right: Expression)
with ColumnarExpression
with Logging {
val gName = "equal"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = {
var (left_node, left_type): (TreeNode, ArrowType) =
left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
Expand Down Expand Up @@ -305,8 +332,11 @@ class ColumnarLessThan(left: Expression, right: Expression, original: Expression
with ColumnarExpression
with Logging {

val gName = "less_than"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
true && left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

Expand Down Expand Up @@ -352,6 +382,14 @@ class ColumnarLessThanOrEqual(left: Expression, right: Expression, original: Exp
extends LessThanOrEqual(left: Expression, right: Expression)
with ColumnarExpression
with Logging {
val gName = "less_than_or_equal_to"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = {
var (left_node, left_type): (TreeNode, ArrowType) =
left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
Expand Down Expand Up @@ -396,6 +434,15 @@ class ColumnarGreaterThan(left: Expression, right: Expression, original: Express
extends GreaterThan(left: Expression, right: Expression)
with ColumnarExpression
with Logging {

val gName = "greater_than"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = {
var (left_node, left_type): (TreeNode, ArrowType) =
left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
Expand Down Expand Up @@ -440,6 +487,14 @@ class ColumnarGreaterThanOrEqual(left: Expression, right: Expression, original:
extends GreaterThanOrEqual(left: Expression, right: Expression)
with ColumnarExpression
with Logging {
val gName = "greater_than_or_equal_to"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

override def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = {
var (left_node, left_type): (TreeNode, ArrowType) =
left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
Expand Down Expand Up @@ -484,6 +539,15 @@ class ColumnarShiftLeft(left: Expression, right: Expression, original: Expressio
extends ShiftLeft(left: Expression, right: Expression)
with ColumnarExpression
with Logging {

val gName = "shift_left"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = {
var (left_node, left_type): (TreeNode, ArrowType) =
left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
Expand All @@ -506,6 +570,15 @@ class ColumnarShiftRight(left: Expression, right: Expression, original: Expressi
extends ShiftRight(left: Expression, right: Expression)
with ColumnarExpression
with Logging {

val gName = "shift_right"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = {
val (left_node, left_type): (TreeNode, ArrowType) =
left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
Expand Down
20 changes: 20 additions & 0 deletions ...sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarDateTimeExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,12 @@ object ColumnarDateTimeExpressions {

class ColumnarYear(child: Expression) extends Year(child) with
ColumnarExpression {
val gName = "extractYear"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

buildCheck()

Expand Down Expand Up @@ -420,6 +426,12 @@ object ColumnarDateTimeExpressions {

class ColumnarMicrosToTimestamp(child: Expression) extends MicrosToTimestamp(child) with
ColumnarExpression {
val gName = "micros_to_timestamp"

override def supportColumnarCodegen(args: java.lang.Object): Boolean = {
codegenFuncList.contains(gName) &&
child.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

buildCheck()

Expand Down Expand Up @@ -453,6 +465,8 @@ object ColumnarDateTimeExpressions {
extends UnixTimestamp(left, right, timeZoneId, failOnError) with
ColumnarExpression {

val gName = "unix_seconds"

val yearMonthDayFormat = "yyyy-MM-dd"
val yearMonthDayTimeFormat = "yyyy-MM-dd HH:mm:ss"
val yearMonthDayTimeNoSepFormat = "yyyyMMddHHmmss"
Expand Down Expand Up @@ -485,6 +499,12 @@ object ColumnarDateTimeExpressions {
}
}

override def supportColumnarCodegen(args: Object): Boolean = {
codegenFuncList.contains(gName) &&
left.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args) &&
right.asInstanceOf[ColumnarExpression].supportColumnarCodegen(args)
}

override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = {
val (leftNode, leftType) = left.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args)
val outType = CodeGeneration.getResultType(dataType)
Expand Down
62 changes: 60 additions & 2 deletions native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpression.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,67 @@ import scala.collection.mutable.ListBuffer

trait ColumnarExpression {

val codegenFuncList: List[String] = List(
"less_than",
"less_than_with_nan",
"greater_than",
"greater_than_with_nan",
"less_than_or_equal_to",
"less_than_or_equal_to_with_nan",
"greater_than_or_equal_to",
"greater_than_or_equal_to_with_nan",
"equal",
"equal_with_nan",
"not",
"isnotnull",
"isnull",
"starts_with",
"like",
"get_json_object",
"translate",
"substr",
"instr",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like instr should be replaced by locate here and in expression_codegen_visitor.cc. Please refer to the code in ColumnarStringInstr.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed in 7f38a8e

"btrim",
"ltrim",
"rtrim",
"upper",
"lower",
"castDATE",
zhouyuan marked this conversation as resolved.
Show resolved Hide resolved
"castDECIMAL",
"castDECIMALNullOnOverflow",
"castINTOrNull",
"castBIGINTOrNull",
"castFLOAT4OrNull",
"castFLOAT8OrNull",
"castDATE",
"castDECIMAL",
"castDECIMALNullOnOverflow",
"castINTOrNull",
"castBIGINTOrNull",
"castFLOAT4OrNull",
"castFLOAT8OrNull",
"castINTOrNull",
"castBIGINTOrNull",
"castFLOAT4OrNull",
"rescaleDECIMAL",
"extractYear",
"round",
"abs",
"add",
"subtract",
"multiply",
"divide",
"shift_left",
"shift_right",
"bitwise_and",
"bitwise_or",
"normalize",
"convertTimestampUnit",
"micros_to_timestamp"
)

def supportColumnarCodegen(args: java.lang.Object): (Boolean) = {
// TODO: disable all codegen unless manuall enabled
true
false
}

def doColumnarCodeGen(args: java.lang.Object): (TreeNode, ArrowType) = {
Expand Down
Loading