Skip to content
This repository has been archived by the owner on Sep 18, 2023. It is now read-only.

Commit

Permalink
limit the max column num in WSCG (#1165)
Browse files Browse the repository at this point in the history
  • Loading branch information
jackylee-ch authored Nov 9, 2022
1 parent b839c15 commit ff92e00
Show file tree
Hide file tree
Showing 8 changed files with 14 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,9 @@ class GazellePluginConfig(conf: SQLConf) extends Logging {
conf
.getConfString("spark.oap.sql.columnar.shuffleSplitDefaultSize", "8192").toInt

val maxWholeStageCodegenColumnNum: Int =
conf.getConfString("spark.oap.sql.columnar.codegen.maxColumnNum", "30").toInt

val numaBindingInfo: GazelleNumaBindingInfo = {
val enableNumaBinding: Boolean =
conf.getConfString("spark.oap.sql.columnar.numaBinding", "false").toBoolean
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ case class ColumnarConditionProjectExec(
}
}
}
true
super.supportColumnarCodegen
}

def getKernelFunction(childTreeNode: TreeNode): TreeNode = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,7 @@ case class ColumnarBroadcastHashJoinExec(
override def getChild: SparkPlan = streamedPlan

override def supportColumnarCodegen: Boolean = {
this.supportCodegen
this.supportCodegen && super.supportColumnarCodegen
}

val output_skip_alias =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -729,7 +729,7 @@ case class ColumnarHashAggregateExec(
}
}
}
return true
super.supportColumnarCodegen
}

def getKernelFunction: TreeNode = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ case class ColumnarShuffledHashJoinExec(
}

override def supportColumnarCodegen: Boolean = {
this.supportCodegen
this.supportCodegen && super.supportColumnarCodegen
}

val output_skip_alias =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ case class ColumnarSortExec(
Seq(child.executeColumnar())
}

override def supportColumnarCodegen: Boolean = true
override def supportColumnarCodegen: Boolean = super.supportColumnarCodegen

override def getBuildPlans: Seq[(SparkPlan, SparkPlan)] = child match {
case c: ColumnarCodegenSupport if c.supportColumnarCodegen == true =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ case class ColumnarSortMergeJoinExec(
return false
}
}
true
super.supportColumnarCodegen
}

val output_skip_alias =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,10 @@ trait ColumnarCodegenSupport extends SparkPlan {
/**
* Whether this SparkPlan supports whole stage codegen or not.
*/
def supportColumnarCodegen: Boolean = true
def supportColumnarCodegen: Boolean = {
// TODO: support large num columns in WSCG
output.size <= GazellePluginConfig.getSessionConf.maxWholeStageCodegenColumnNum
}

/**
* Returns all the RDDs of ColumnarBatch which generates the input rows.
Expand Down Expand Up @@ -91,7 +94,7 @@ case class ColumnarWholeStageCodegenExec(child: SparkPlan)(val codegenStageId: I

// This is not strictly needed because the codegen transformation happens after the columnar
// transformation but just for consistency
override def supportColumnarCodegen: Boolean = true
override def supportColumnarCodegen: Boolean = super.supportColumnarCodegen

override def supportsColumnar: Boolean = true

Expand Down

0 comments on commit ff92e00

Please sign in to comment.