Skip to content

Commit b5c5bd9

Browse files
committed
Disable generate codegen since it fails my workload.
1 parent 91fa80f commit b5c5bd9

File tree

2 files changed

+1
-29
lines changed

2 files changed

+1
-29
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/GenerateExec.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ case class GenerateExec(
119119
}
120120
}
121121

122-
override def supportCodegen: Boolean = generator.supportCodegen
122+
override def supportCodegen: Boolean = false
123123

124124
override def inputRDDs(): Seq[RDD[InternalRow]] = {
125125
child.asInstanceOf[CodegenSupport].inputRDDs()

sql/core/src/test/scala/org/apache/spark/sql/execution/WholeStageCodegenSuite.scala

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -116,34 +116,6 @@ class WholeStageCodegenSuite extends SparkPlanTest with SharedSQLContext {
116116
assert(ds.collect() === Array(("a", 10.0), ("b", 3.0), ("c", 1.0)))
117117
}
118118

119-
test("generate should be included in WholeStageCodegen") {
120-
import org.apache.spark.sql.functions._
121-
val ds = spark.range(2).select(
122-
col("id"),
123-
explode(array(col("id") + 1, col("id") + 2)).as("value"))
124-
val plan = ds.queryExecution.executedPlan
125-
assert(plan.find(p =>
126-
p.isInstanceOf[WholeStageCodegenExec] &&
127-
p.asInstanceOf[WholeStageCodegenExec].child.isInstanceOf[GenerateExec]).isDefined)
128-
assert(ds.collect() === Array(Row(0, 1), Row(0, 2), Row(1, 2), Row(1, 3)))
129-
}
130-
131-
test("large stack generator should not use WholeStageCodegen") {
132-
def createStackGenerator(rows: Int): SparkPlan = {
133-
val id = UnresolvedAttribute("id")
134-
val stack = Stack(Literal(rows) +: Seq.tabulate(rows)(i => Add(id, Literal(i))))
135-
spark.range(500).select(Column(stack)).queryExecution.executedPlan
136-
}
137-
val isCodeGenerated: SparkPlan => Boolean = {
138-
case WholeStageCodegenExec(_: GenerateExec) => true
139-
case _ => false
140-
}
141-
142-
// Only 'stack' generators that produce 50 rows or less are code generated.
143-
assert(createStackGenerator(50).find(isCodeGenerated).isDefined)
144-
assert(createStackGenerator(100).find(isCodeGenerated).isEmpty)
145-
}
146-
147119
test("SPARK-19512 codegen for comparing structs is incorrect") {
148120
// this would raise CompileException before the fix
149121
spark.range(10)

0 commit comments

Comments
 (0)