From 1abf69155db9f44380d1726ed5c9b0d7a3b48d1d Mon Sep 17 00:00:00 2001 From: ptkool Date: Sun, 16 Apr 2017 08:33:59 -0400 Subject: [PATCH 1/4] Add optimization rules to apply Complementation Laws. --- .../spark/sql/catalyst/optimizer/expressions.scala | 5 +++++ .../optimizer/BooleanSimplificationSuite.scala | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala index 21d1cd593262..509f3d511ce4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala @@ -153,6 +153,11 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper { case TrueLiteral Or _ => TrueLiteral case _ Or TrueLiteral => TrueLiteral + case a And b if Not(a).semanticEquals(b) => FalseLiteral + case a Or b if Not(a).semanticEquals(b) => TrueLiteral + case a And b if a.semanticEquals(Not(b)) => FalseLiteral + case a Or b if a.semanticEquals(Not(b)) => TrueLiteral + case a And b if a.semanticEquals(b) => a case a Or b if a.semanticEquals(b) => a diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala index 1b9db0601492..621f60f288b3 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala @@ -42,6 +42,12 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper { val testRelation = LocalRelation('a.int, 'b.int, 'c.int, 'd.string) + private def checkCondition(input: Expression, expected: LogicalPlan): Unit = { + val plan = testRelation.where(input).analyze + val actual = Optimize.execute(plan) + comparePlans(actual, expected) + } + private def checkCondition(input: Expression, expected: Expression): Unit = { val plan = testRelation.where(input).analyze val actual = Optimize.execute(plan) @@ -138,6 +144,12 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper { checkCondition(!(('a || 'b) && ('c || 'd)), (!'a && !'b) || (!'c && !'d)) } + test("Complementation Laws") { + checkCondition('a && !'a, LocalRelation(testRelation.output, Seq.empty)) + + checkCondition('a || !'a, testRelation) + } + private val caseInsensitiveConf = new SimpleCatalystConf(false) private val caseInsensitiveAnalyzer = new Analyzer( new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry, caseInsensitiveConf), From b557165e61529159231b9d45e11b60b46eaed6c8 Mon Sep 17 00:00:00 2001 From: ptkool Date: Sun, 16 Apr 2017 08:44:18 -0400 Subject: [PATCH 2/4] Added additional tests. --- .../sql/catalyst/optimizer/BooleanSimplificationSuite.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala index 621f60f288b3..6ce41e0199bf 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala @@ -146,8 +146,10 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper { test("Complementation Laws") { checkCondition('a && !'a, LocalRelation(testRelation.output, Seq.empty)) + checkCondition(!'a && 'a, LocalRelation(testRelation.output, Seq.empty)) checkCondition('a || !'a, testRelation) + checkCondition(!'a || 'a, testRelation) } private val caseInsensitiveConf = new SimpleCatalystConf(false) From 688b2f0696f1d1d867e872f43506e52f95f46362 Mon Sep 17 00:00:00 2001 From: ptkool Date: Sun, 16 Apr 2017 08:50:26 -0400 Subject: [PATCH 3/4] Small correction to previous merge operation. --- .../optimizer/BooleanSimplificationSuite.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala index 6ce41e0199bf..b7e3fb4266e8 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala @@ -144,14 +144,6 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper { checkCondition(!(('a || 'b) && ('c || 'd)), (!'a && !'b) || (!'c && !'d)) } - test("Complementation Laws") { - checkCondition('a && !'a, LocalRelation(testRelation.output, Seq.empty)) - checkCondition(!'a && 'a, LocalRelation(testRelation.output, Seq.empty)) - - checkCondition('a || !'a, testRelation) - checkCondition(!'a || 'a, testRelation) - } - private val caseInsensitiveConf = new SimpleCatalystConf(false) private val caseInsensitiveAnalyzer = new Analyzer( new SessionCatalog(new InMemoryCatalog, EmptyFunctionRegistry, caseInsensitiveConf), @@ -174,4 +166,12 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper { testRelation.where('a > 2 || ('b > 3 && 'b < 5))) comparePlans(actual, expected) } + + test("Complementation Laws") { + checkCondition('a && !'a, LocalRelation(testRelation.output, Seq.empty)) + checkCondition(!'a && 'a, LocalRelation(testRelation.output, Seq.empty)) + + checkCondition('a || !'a, testRelation) + checkCondition(!'a || 'a, testRelation) + } } From 5317dd43109bde415fe1179b1d54e1620f26b8ba Mon Sep 17 00:00:00 2001 From: ptkool Date: Wed, 19 Apr 2017 16:42:19 -0400 Subject: [PATCH 4/4] Added new test relations with 1 row of data. --- .../optimizer/BooleanSimplificationSuite.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala index b7e3fb4266e8..1feeffd3e736 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.catalyst.optimizer +import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.SimpleCatalystConf import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.catalog.{InMemoryCatalog, SessionCatalog} @@ -42,8 +43,12 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper { val testRelation = LocalRelation('a.int, 'b.int, 'c.int, 'd.string) + val testRelationWithData = LocalRelation.fromExternalRows( + testRelation.output, Seq(Row(1, 2, 3, "abc")) + ) + private def checkCondition(input: Expression, expected: LogicalPlan): Unit = { - val plan = testRelation.where(input).analyze + val plan = testRelationWithData.where(input).analyze val actual = Optimize.execute(plan) comparePlans(actual, expected) } @@ -168,10 +173,10 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper { } test("Complementation Laws") { - checkCondition('a && !'a, LocalRelation(testRelation.output, Seq.empty)) - checkCondition(!'a && 'a, LocalRelation(testRelation.output, Seq.empty)) + checkCondition('a && !'a, testRelation) + checkCondition(!'a && 'a, testRelation) - checkCondition('a || !'a, testRelation) - checkCondition(!'a || 'a, testRelation) + checkCondition('a || !'a, testRelationWithData) + checkCondition(!'a || 'a, testRelationWithData) } }