diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala index 9130e77ea572..13c0766219a8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/filters.scala @@ -26,7 +26,18 @@ package org.apache.spark.sql.sources * * @since 1.3.0 */ -abstract class Filter +abstract class Filter { + /** + * List of columns that are referenced by this filter. + * @since 2.1.0 + */ + def references: Array[String] + + protected def findReferences(value: Any): Array[String] = value match { + case f: Filter => f.references + case _ => Array.empty + } +} /** * A filter that evaluates to `true` iff the attribute evaluates to a value @@ -34,7 +45,9 @@ abstract class Filter * * @since 1.3.0 */ -case class EqualTo(attribute: String, value: Any) extends Filter +case class EqualTo(attribute: String, value: Any) extends Filter { + override def references: Array[String] = Array(attribute) ++ findReferences(value) +} /** * Performs equality comparison, similar to [[EqualTo]]. However, this differs from [[EqualTo]] @@ -43,7 +56,9 @@ case class EqualTo(attribute: String, value: Any) extends Filter * * @since 1.5.0 */ -case class EqualNullSafe(attribute: String, value: Any) extends Filter +case class EqualNullSafe(attribute: String, value: Any) extends Filter { + override def references: Array[String] = Array(attribute) ++ findReferences(value) +} /** * A filter that evaluates to `true` iff the attribute evaluates to a value @@ -51,7 +66,9 @@ case class EqualNullSafe(attribute: String, value: Any) extends Filter * * @since 1.3.0 */ -case class GreaterThan(attribute: String, value: Any) extends Filter +case class GreaterThan(attribute: String, value: Any) extends Filter { + override def references: Array[String] = Array(attribute) ++ findReferences(value) +} /** * A filter that evaluates to `true` iff the attribute evaluates to a value @@ -59,7 +76,9 @@ case class GreaterThan(attribute: String, value: Any) extends Filter * * @since 1.3.0 */ -case class GreaterThanOrEqual(attribute: String, value: Any) extends Filter +case class GreaterThanOrEqual(attribute: String, value: Any) extends Filter { + override def references: Array[String] = Array(attribute) ++ findReferences(value) +} /** * A filter that evaluates to `true` iff the attribute evaluates to a value @@ -67,7 +86,9 @@ case class GreaterThanOrEqual(attribute: String, value: Any) extends Filter * * @since 1.3.0 */ -case class LessThan(attribute: String, value: Any) extends Filter +case class LessThan(attribute: String, value: Any) extends Filter { + override def references: Array[String] = Array(attribute) ++ findReferences(value) +} /** * A filter that evaluates to `true` iff the attribute evaluates to a value @@ -75,7 +96,9 @@ case class LessThan(attribute: String, value: Any) extends Filter * * @since 1.3.0 */ -case class LessThanOrEqual(attribute: String, value: Any) extends Filter +case class LessThanOrEqual(attribute: String, value: Any) extends Filter { + override def references: Array[String] = Array(attribute) ++ findReferences(value) +} /** * A filter that evaluates to `true` iff the attribute evaluates to one of the values in the array. @@ -99,6 +122,8 @@ case class In(attribute: String, values: Array[Any]) extends Filter { override def toString: String = { s"In($attribute, [${values.mkString(",")}]" } + + override def references: Array[String] = Array(attribute) ++ values.flatMap(findReferences) } /** @@ -106,35 +131,45 @@ case class In(attribute: String, values: Array[Any]) extends Filter { * * @since 1.3.0 */ -case class IsNull(attribute: String) extends Filter +case class IsNull(attribute: String) extends Filter { + override def references: Array[String] = Array(attribute) +} /** * A filter that evaluates to `true` iff the attribute evaluates to a non-null value. * * @since 1.3.0 */ -case class IsNotNull(attribute: String) extends Filter +case class IsNotNull(attribute: String) extends Filter { + override def references: Array[String] = Array(attribute) +} /** * A filter that evaluates to `true` iff both `left` or `right` evaluate to `true`. * * @since 1.3.0 */ -case class And(left: Filter, right: Filter) extends Filter +case class And(left: Filter, right: Filter) extends Filter { + override def references: Array[String] = left.references ++ right.references +} /** * A filter that evaluates to `true` iff at least one of `left` or `right` evaluates to `true`. * * @since 1.3.0 */ -case class Or(left: Filter, right: Filter) extends Filter +case class Or(left: Filter, right: Filter) extends Filter { + override def references: Array[String] = left.references ++ right.references +} /** * A filter that evaluates to `true` iff `child` is evaluated to `false`. * * @since 1.3.0 */ -case class Not(child: Filter) extends Filter +case class Not(child: Filter) extends Filter { + override def references: Array[String] = child.references +} /** * A filter that evaluates to `true` iff the attribute evaluates to @@ -142,7 +177,9 @@ case class Not(child: Filter) extends Filter * * @since 1.3.1 */ -case class StringStartsWith(attribute: String, value: String) extends Filter +case class StringStartsWith(attribute: String, value: String) extends Filter { + override def references: Array[String] = Array(attribute) +} /** * A filter that evaluates to `true` iff the attribute evaluates to @@ -150,7 +187,9 @@ case class StringStartsWith(attribute: String, value: String) extends Filter * * @since 1.3.1 */ -case class StringEndsWith(attribute: String, value: String) extends Filter +case class StringEndsWith(attribute: String, value: String) extends Filter { + override def references: Array[String] = Array(attribute) +} /** * A filter that evaluates to `true` iff the attribute evaluates to @@ -158,4 +197,6 @@ case class StringEndsWith(attribute: String, value: String) extends Filter * * @since 1.3.1 */ -case class StringContains(attribute: String, value: String) extends Filter +case class StringContains(attribute: String, value: String) extends Filter { + override def references: Array[String] = Array(attribute) +}