From 9314806cf03f7a4222a67f8b285c4225bdef3083 Mon Sep 17 00:00:00 2001 From: Takuya UESHIN Date: Tue, 10 Jun 2014 16:55:53 +0900 Subject: [PATCH 1/3] Fix NullPropagation to use exact type value. --- .../org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index 406ffd6801e98..f8358299e3f14 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -100,8 +100,8 @@ object ColumnPruning extends Rule[LogicalPlan] { object NullPropagation extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transform { case q: LogicalPlan => q transformExpressionsUp { - case e @ Count(Literal(null, _)) => Literal(0, e.dataType) - case e @ Sum(Literal(c, _)) if c == 0 => Literal(0, e.dataType) + case e @ Count(Literal(null, _)) => Literal(0L, e.dataType) + case e @ Sum(Literal(c, _)) if c == 0 => Cast(Literal(0), e.dataType) case e @ Average(Literal(c, _)) if c == 0 => Literal(0.0, e.dataType) case e @ IsNull(c) if c.nullable == false => Literal(false, BooleanType) case e @ IsNotNull(c) if c.nullable == false => Literal(true, BooleanType) From 830c20b63a8b65e17c1d6b3709b84ab1c93c0666 Mon Sep 17 00:00:00 2001 From: Takuya UESHIN Date: Tue, 10 Jun 2014 19:08:36 +0900 Subject: [PATCH 2/3] Add Cast to the result of transformation from Count. --- .../org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index f8358299e3f14..fa75176c5dba0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -100,7 +100,7 @@ object ColumnPruning extends Rule[LogicalPlan] { object NullPropagation extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transform { case q: LogicalPlan => q transformExpressionsUp { - case e @ Count(Literal(null, _)) => Literal(0L, e.dataType) + case e @ Count(Literal(null, _)) => Cast(Literal(0L), e.dataType) case e @ Sum(Literal(c, _)) if c == 0 => Cast(Literal(0), e.dataType) case e @ Average(Literal(c, _)) if c == 0 => Literal(0.0, e.dataType) case e @ IsNull(c) if c.nullable == false => Literal(false, BooleanType) From 65b6ff1012cf55d595037488d7e33f35f780e4dd Mon Sep 17 00:00:00 2001 From: Takuya UESHIN Date: Tue, 10 Jun 2014 19:10:32 +0900 Subject: [PATCH 3/3] Modify the literal value of the result of transformation from Sum to long value. --- .../org/apache/spark/sql/catalyst/optimizer/Optimizer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index fa75176c5dba0..f192b6f0e95af 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -101,7 +101,7 @@ object NullPropagation extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = plan transform { case q: LogicalPlan => q transformExpressionsUp { case e @ Count(Literal(null, _)) => Cast(Literal(0L), e.dataType) - case e @ Sum(Literal(c, _)) if c == 0 => Cast(Literal(0), e.dataType) + case e @ Sum(Literal(c, _)) if c == 0 => Cast(Literal(0L), e.dataType) case e @ Average(Literal(c, _)) if c == 0 => Literal(0.0, e.dataType) case e @ IsNull(c) if c.nullable == false => Literal(false, BooleanType) case e @ IsNotNull(c) if c.nullable == false => Literal(true, BooleanType)