From 5f70b1ddc26c51d0c2aef34b58fe98f8220ffc0a Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 17 Oct 2016 21:07:52 +0900 Subject: [PATCH 01/47] Add examples (extend) in each function and improve documentation with arguments --- .../aggregate/ApproximatePercentile.scala | 42 ++++-- .../expressions/aggregate/Average.scala | 9 +- .../aggregate/CentralMomentAgg.scala | 56 +++++++- .../catalyst/expressions/aggregate/Corr.scala | 12 +- .../expressions/aggregate/Count.scala | 20 ++- .../expressions/aggregate/Covariance.scala | 23 +++- .../expressions/aggregate/First.scala | 16 ++- .../aggregate/HyperLogLogPlusPlus.scala | 18 ++- .../catalyst/expressions/aggregate/Last.scala | 15 ++- .../catalyst/expressions/aggregate/Max.scala | 8 +- .../catalyst/expressions/aggregate/Min.scala | 8 +- .../catalyst/expressions/aggregate/Sum.scala | 9 +- .../expressions/aggregate/collect.scala | 16 ++- .../sql/catalyst/expressions/arithmetic.scala | 8 +- .../sql/catalyst/expressions/xml/xpath.scala | 126 +++++++++++++++--- .../sql/execution/command/functions.scala | 2 +- 16 files changed, 328 insertions(+), 60 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala index f91ff87fc1c0..e412824e1a41 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala @@ -52,17 +52,41 @@ import org.apache.spark.sql.types._ usage = """ _FUNC_(col, percentage [, accuracy]) - Returns the approximate percentile value of numeric - column `col` at the given percentage. The value of percentage must be between 0.0 - and 1.0. The `accuracy` parameter (default: 10000) is a positive integer literal which - controls approximation accuracy at the cost of memory. Higher value of `accuracy` yields - better accuracy, `1.0/accuracy` is the relative error of the approximation. + column `col` at the given percentage. The value of percentage must be between 0.0 + and 1.0. The `accuracy` parameter (default: 10000) is a positive integer literal which + controls approximation accuracy at the cost of memory. Higher value of `accuracy` yields + better accuracy, `1.0/accuracy` is the relative error of the approximation. + + Arguments: + col - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + percentage - any numeric type or any nonnumeric type literal that can be + implicitly converted to double type. + accuracy - any numeric type or any nonnumeric type literal that can be implicitly + converted to int type. _FUNC_(col, array(percentage1 [, percentage2]...) [, accuracy]) - Returns the approximate - percentile array of column `col` at the given percentage array. Each value of the - percentage array must be between 0.0 and 1.0. The `accuracy` parameter (default: 10000) is - a positive integer literal which controls approximation accuracy at the cost of memory. - Higher value of `accuracy` yields better accuracy, `1.0/accuracy` is the relative error of - the approximation. + percentile array of column `col` at the given percentage array. Each value of the + percentage array must be between 0.0 and 1.0. The `accuracy` parameter (default: 10000) is + a positive integer literal which controls approximation accuracy at the cost of memory. + Higher value of `accuracy` yields better accuracy, `1.0/accuracy` is the relative error of + the approximation. + + Arguments: + col - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + array(...) - an array that contains any numeric type literal that can be implicitly + converted to double type. + accuracy - any numeric type or any nonnumeric type literal that can be implicitly + converted to int type. + """, + extended = + """ + > SELECT percentile_approx(10.0, 0.5, 100); + 10.0 + + > SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100); + [10.0,10.0,10.0] """) case class ApproximatePercentile( child: Expression, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala index ff7077484783..80126b625262 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala @@ -24,7 +24,14 @@ import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.types._ @ExpressionDescription( - usage = "_FUNC_(x) - Returns the mean calculated from values of a group.") + usage = + """ + _FUNC_(expr) - Returns the mean calculated from values of a group. + + Arguments: + expr - any numeric datatype or any nonnumeric datatype expression that can be implicitly + converted to numeric type. + """) case class Average(child: Expression) extends DeclarativeAggregate { override def prettyName: String = "avg" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala index 17a7c6dce89c..7813a295660e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala @@ -132,7 +132,14 @@ abstract class CentralMomentAgg(child: Expression) extends DeclarativeAggregate // Compute the population standard deviation of a column // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(x) - Returns the population standard deviation calculated from values of a group.") + usage = + """ + _FUNC_(expr) - Returns the population standard deviation calculated from values of a group. + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) // scalastyle:on line.size.limit case class StddevPop(child: Expression) extends CentralMomentAgg(child) { @@ -148,7 +155,14 @@ case class StddevPop(child: Expression) extends CentralMomentAgg(child) { // Compute the sample standard deviation of a column @ExpressionDescription( - usage = "_FUNC_(x) - Returns the sample standard deviation calculated from values of a group.") + usage = + """ + _FUNC_(expr) - Returns the sample standard deviation calculated from values of a group. + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class StddevSamp(child: Expression) extends CentralMomentAgg(child) { override protected def momentOrder = 2 @@ -164,7 +178,14 @@ case class StddevSamp(child: Expression) extends CentralMomentAgg(child) { // Compute the population variance of a column @ExpressionDescription( - usage = "_FUNC_(x) - Returns the population variance calculated from values of a group.") + usage = + """ + _FUNC_(expr) - Returns the population variance calculated from values of a group. + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class VariancePop(child: Expression) extends CentralMomentAgg(child) { override protected def momentOrder = 2 @@ -179,7 +200,14 @@ case class VariancePop(child: Expression) extends CentralMomentAgg(child) { // Compute the sample variance of a column @ExpressionDescription( - usage = "_FUNC_(x) - Returns the sample variance calculated from values of a group.") + usage = + """ + _FUNC_(expr) - Returns the sample variance calculated from values of a group. + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) { override protected def momentOrder = 2 @@ -194,8 +222,15 @@ case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) { } @ExpressionDescription( - usage = "_FUNC_(x) - Returns the Skewness value calculated from values of a group.") -case class Skewness(child: Expression) extends CentralMomentAgg(child) { + usage = + """ + _FUNC_(expr) - Returns the Skewness value calculated from values of a group. + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) +case class Skewness(expr: Expression) extends CentralMomentAgg(expr) { override def prettyName: String = "skewness" @@ -209,7 +244,14 @@ case class Skewness(child: Expression) extends CentralMomentAgg(child) { } @ExpressionDescription( - usage = "_FUNC_(x) - Returns the Kurtosis value calculated from values of a group.") + usage = + """ + _FUNC_(expr) - Returns the Kurtosis value calculated from values of a group. + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class Kurtosis(child: Expression) extends CentralMomentAgg(child) { override protected def momentOrder = 4 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala index e29265e2f41e..ff7cd4088af0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala @@ -29,7 +29,17 @@ import org.apache.spark.sql.types._ * http://en.wikipedia.org/wiki/Pearson_product-moment_correlation_coefficient */ @ExpressionDescription( - usage = "_FUNC_(x,y) - Returns Pearson coefficient of correlation between a set of number pairs.") + usage = + """ + _FUNC_(expr1, expr2) - Returns Pearson coefficient of correlation between a set of number + pairs. + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class Corr(x: Expression, y: Expression) extends DeclarativeAggregate { override def children: Seq[Expression] = Seq(x, y) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala index 17ae012af79b..40091c33e27a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala @@ -21,12 +21,22 @@ import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.types._ -// scalastyle:off line.size.limit @ExpressionDescription( - usage = """_FUNC_(*) - Returns the total number of retrieved rows, including rows containing NULL values. - _FUNC_(expr) - Returns the number of rows for which the supplied expression is non-NULL. - _FUNC_(DISTINCT expr[, expr...]) - Returns the number of rows for which the supplied expression(s) are unique and non-NULL.""") -// scalastyle:on line.size.limit + usage = + """ + _FUNC_(*) - Returns the total number of retrieved rows, including rows containing NULL. + + _FUNC_(expr) - Returns the number of rows for which the supplied expression is non-NULL. + + Arguments: + expr - any type expression that represents data to count. + + _FUNC_(DISTINCT expr[, expr...]) - Returns the number of rows for which the supplied + expression(s) are unique and non-NULL. + + Arguments: + expr - any type expression that represents data to count. + """) case class Count(children: Seq[Expression]) extends DeclarativeAggregate { override def nullable: Boolean = false diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala index d80afbebf740..23040c714432 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala @@ -77,7 +77,16 @@ abstract class Covariance(x: Expression, y: Expression) extends DeclarativeAggre } @ExpressionDescription( - usage = "_FUNC_(x,y) - Returns the population covariance of a set of number pairs.") + usage = + """ + _FUNC_(expr1, expr2) - Returns the population covariance of a set of number pairs. + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class CovPopulation(left: Expression, right: Expression) extends Covariance(left, right) { override val evaluateExpression: Expression = { If(n === Literal(0.0), Literal.create(null, DoubleType), @@ -86,9 +95,17 @@ case class CovPopulation(left: Expression, right: Expression) extends Covariance override def prettyName: String = "covar_pop" } - @ExpressionDescription( - usage = "_FUNC_(x,y) - Returns the sample covariance of a set of number pairs.") + usage = + """ + _FUNC_(expr1, expr2) - Returns the sample covariance of a set of number pairs. + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class CovSample(left: Expression, right: Expression) extends Covariance(left, right) { override val evaluateExpression: Expression = { If(n === Literal(0.0), Literal.create(null, DoubleType), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala index d702c08cfd34..2ddb6bdd2581 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala @@ -29,9 +29,19 @@ import org.apache.spark.sql.types._ * a single partition, and we use a single reducer to do the aggregation.). */ @ExpressionDescription( - usage = """_FUNC_(expr) - Returns the first value of `child` for a group of rows. - _FUNC_(expr,isIgnoreNull=false) - Returns the first value of `child` for a group of rows. - If isIgnoreNull is true, returns only non-null values. + usage = + """ + _FUNC_(expr) - Returns the first value of `child` for a group of rows. + + Arguments: + expr - any type expression that represents data to collect the first. + + _FUNC_(expr,isIgnoreNull=false) - Returns the first value of `child` for a group of rows. + If isIgnoreNull is true, returns only non-null values. + + Arguments: + expr - any type expression that represents data to collect the first. + isIgnoreNull - boolean type literal. """) case class First(child: Expression, ignoreNullsExpr: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala index 83c8d400c5d6..948622a46129 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala @@ -47,9 +47,21 @@ import org.apache.spark.sql.types._ */ // scalastyle:on @ExpressionDescription( - usage = """_FUNC_(expr) - Returns the estimated cardinality by HyperLogLog++. - _FUNC_(expr, relativeSD=0.05) - Returns the estimated cardinality by HyperLogLog++ - with relativeSD, the maximum estimation error allowed. + usage = + """ + _FUNC_(expr) - Returns the estimated cardinality by HyperLogLog++. + + Arguments: + expr - any type expression that represents data to collect the first. + + _FUNC_(expr, relativeSD) - Returns the estimated cardinality by HyperLogLog++ + with relativeSD, the maximum estimation error allowed. + + Arguments: + expr - any type expression that represents data to collect the first. + relativeSD - any numeric type or any nonnumeric type literal that can be implicitly + converted to double type, that represents maximum estimation error allowed + (default = 0.05). """) case class HyperLogLogPlusPlus( child: Expression, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala index 8579f7292d3a..e811fd56c81f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala @@ -29,7 +29,20 @@ import org.apache.spark.sql.types._ * a single partition, and we use a single reducer to do the aggregation.). */ @ExpressionDescription( - usage = "_FUNC_(expr,isIgnoreNull) - Returns the last value of `child` for a group of rows.") + usage = + """ + _FUNC_(expr) - Returns the last value of `child` for a group of rows. + + Arguments: + expr - any type expression that represents data to collect the last. + + _FUNC_(expr,isIgnoreNull=false) - Returns the last value of `child` for a group of rows. + If isIgnoreNull is true, returns only non-null values. + + Arguments: + expr - any type expression that represents data to collect the last. + isIgnoreNull - boolean type literal. + """) case class Last(child: Expression, ignoreNullsExpr: Expression) extends DeclarativeAggregate { def this(child: Expression) = this(child, Literal.create(false, BooleanType)) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala index c534fe495fc1..e0a90e39bade 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala @@ -23,7 +23,13 @@ import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.types._ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the maximum value of expr.") + usage = + """ + _FUNC_(expr) - Returns the maximum value of expr." + + Arguments: + expr - any type type expression. + """) case class Max(child: Expression) extends DeclarativeAggregate { override def children: Seq[Expression] = child :: Nil diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala index 35289b468183..df6d9e8b1ca2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala @@ -23,7 +23,13 @@ import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.types._ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the minimum value of expr.") + usage = + """ + _FUNC_(expr) - Returns the minimum value of expr." + + Arguments: + expr - any type type expression. + """) case class Min(child: Expression) extends DeclarativeAggregate { override def children: Seq[Expression] = child :: Nil diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala index ad217f25b5a2..26cd0853e921 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala @@ -23,7 +23,14 @@ import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.types._ @ExpressionDescription( - usage = "_FUNC_(x) - Returns the sum calculated from values of a group.") + usage = + """ + _FUNC_(expr) - Returns the sum calculated from values of a group. + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class Sum(child: Expression) extends DeclarativeAggregate { override def children: Seq[Expression] = child :: Nil diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala index 89eb864e9470..b3649831f5da 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala @@ -86,7 +86,13 @@ abstract class Collect extends ImperativeAggregate { * Collect a list of elements. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Collects and returns a list of non-unique elements.") + usage = + """ + _FUNC_(expr) - Collects and returns a set of unique elements. + + Arguments: + expr - any type expression that represents data to collect. + """) case class CollectList( child: Expression, mutableAggBufferOffset: Int = 0, @@ -109,7 +115,13 @@ case class CollectList( * Collect a list of unique elements. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Collects and returns a set of unique elements.") + usage = + """ + _FUNC_(expr) - Collects and returns a set of unique elements. + + Arguments: + expr - any type expression except map type that represents data to collect. + """) case class CollectSet( child: Expression, mutableAggBufferOffset: Int = 0, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 6f3db79622fa..8fc99c0fc020 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -25,7 +25,13 @@ import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.CalendarInterval @ExpressionDescription( - usage = "_FUNC_(a) - Returns -a.") + usage = + """ + _FUNC_(expr) - Returns -expr." + + Arguments: + expr - any type expression. + """) case class UnaryMinus(child: Expression) extends UnaryExpression with ExpectsInputTypes with NullIntolerant { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index 47f039e6a4cc..8b73f64797fd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -56,8 +56,19 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with } @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Evaluates a boolean xpath expression.", - extended = "> SELECT _FUNC_('1','a/b');\ntrue") + usage = + """ + _FUNC_(xml, xpath) - Evaluates a boolean xpath expression. + + Arguments: + xml - string type that represents XML documentation. + path - string type literal that represents xpath expression. + """, + extended = + """ + > SELECT _FUNC_('1','a/b'); + true + """) case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_boolean" @@ -69,8 +80,19 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract } @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a short value that matches the xpath expression", - extended = "> SELECT _FUNC_('12','sum(a/b)');\n3") + usage = + """ + _FUNC_(xml, xpath) - Returns a short value that matches the xpath expression. + + Arguments: + xml - string type that represents XML documentation. + path - string type literal that represents xpath expression. + """, + extended = + """ + > SELECT _FUNC_('12', 'sum(a/b)'); + 3 + """) case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_int" override def dataType: DataType = ShortType @@ -82,8 +104,19 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { } @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns an integer value that matches the xpath expression", - extended = "> SELECT _FUNC_('12','sum(a/b)');\n3") + usage = + """ + _FUNC_(xml, xpath) - Returns a short value that matches the xpath expression. + + Arguments: + xml - string type that represents XML documentation. + path - string type literal that represents xpath expression. + """, + extended = + """ + > SELECT _FUNC_('12', 'sum(a/b)'); + 3 + """) case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_int" override def dataType: DataType = IntegerType @@ -95,8 +128,19 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { } @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a long value that matches the xpath expression", - extended = "> SELECT _FUNC_('12','sum(a/b)');\n3") + usage = + """ + _FUNC_(xml, xpath) - Returns a short value that matches the xpath expression. + + Arguments: + xml - string type that represents XML documentation. + path - string type literal that represents xpath expression. + """, + extended = + """ + > SELECT _FUNC_('12', 'sum(a/b)'); + 3 + """) case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_long" override def dataType: DataType = LongType @@ -108,8 +152,19 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { } @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a float value that matches the xpath expression", - extended = "> SELECT _FUNC_('12','sum(a/b)');\n3.0") + usage = + """ + _FUNC_(xml, xpath) - Returns a short value that matches the xpath expression. + + Arguments: + xml - string type that represents XML documentation. + path - string type literal that represents xpath expression. + """, + extended = + """ + > SELECT _FUNC_('12', 'sum(a/b)'); + 3.0 + """) case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_float" override def dataType: DataType = FloatType @@ -121,8 +176,19 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { } @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a double value that matches the xpath expression", - extended = "> SELECT _FUNC_('12','sum(a/b)');\n3.0") + usage = + """ + _FUNC_(xml, xpath) - Returns a short value that matches the xpath expression. + + Arguments: + xml - string type that represents XML documentation. + path - string type literal that represents xpath expression. + """, + extended = + """ + > SELECT _FUNC_('12', 'sum(a/b)'); + 3.0 + """) case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_float" override def dataType: DataType = DoubleType @@ -133,11 +199,21 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { } } -// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the xpath expression", - extended = "> SELECT _FUNC_('bcc','a/c');\ncc") -// scalastyle:on line.size.limit + usage = + """ + _FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the + xpath expression. + + Arguments: + xml - string type that represents XML documentation. + path - string type literal that represents xpath expression. + """, + extended = + """ + > SELECT _FUNC_('bcc','a/c'); + cc + """) case class XPathString(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_string" override def dataType: DataType = StringType @@ -148,11 +224,21 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract { } } -// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a string array of values within xml nodes that match the xpath expression", - extended = "> SELECT _FUNC_('b1b2b3c1c2','a/b/text()');\n['b1','b2','b3']") -// scalastyle:on line.size.limit + usage = + """ + _FUNC_(xml, xpath) - Returns a string array of values within xml nodes that match the + xpath expression + + Arguments: + xml - string type that represents XML documentation. + path - string type literal that represents xpath expression. + """, + extended = + """ + > SELECT _FUNC_('b1b2b3c1c2','a/b/text()'); + ['b1','b2','b3'] + """) case class XPathList(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath" override def dataType: DataType = ArrayType(StringType, containsNull = false) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala index 26593d2918a6..e3976a325d96 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala @@ -125,7 +125,7 @@ case class DescribeFunctionCommand( if (isExtended) { result :+ - Row(s"Extended Usage:\n${replaceFunctionName(info.getExtended, info.getName)}") + Row(s"Extended Usage: ${replaceFunctionName(info.getExtended, info.getName)}") } else { result } From 77abafa5741c5c4c706f4e310f5a63c39811471b Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 18 Oct 2016 22:02:49 +0900 Subject: [PATCH 02/47] aggregate OK --- .../expressions/ExpressionDescription.java | 2 +- .../aggregate/ApproximatePercentile.scala | 82 ++++++++-------- .../expressions/aggregate/Average.scala | 16 +-- .../aggregate/CentralMomentAgg.scala | 98 ++++++++++--------- .../catalyst/expressions/aggregate/Corr.scala | 21 ++-- .../expressions/aggregate/Count.scala | 27 ++--- .../expressions/aggregate/Covariance.scala | 40 ++++---- .../expressions/aggregate/First.scala | 27 ++--- .../aggregate/HyperLogLogPlusPlus.scala | 35 ++++--- .../catalyst/expressions/aggregate/Last.scala | 27 ++--- .../catalyst/expressions/aggregate/Max.scala | 12 +-- .../catalyst/expressions/aggregate/Min.scala | 12 +-- .../catalyst/expressions/aggregate/Sum.scala | 16 +-- .../expressions/aggregate/collect.scala | 26 ++--- 14 files changed, 233 insertions(+), 208 deletions(-) diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java index 9e10f27d59d5..8a89032818f6 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionDescription.java @@ -39,5 +39,5 @@ @Retention(RetentionPolicy.RUNTIME) public @interface ExpressionDescription { String usage() default "_FUNC_ is undocumented"; - String extended() default "No example for _FUNC_."; + String extended() default "\n No example/argument for _FUNC_."; } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala index e412824e1a41..fbe04bda4e14 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala @@ -49,45 +49,49 @@ import org.apache.spark.sql.types._ * DEFAULT_PERCENTILE_ACCURACY. */ @ExpressionDescription( - usage = - """ - _FUNC_(col, percentage [, accuracy]) - Returns the approximate percentile value of numeric - column `col` at the given percentage. The value of percentage must be between 0.0 - and 1.0. The `accuracy` parameter (default: 10000) is a positive integer literal which - controls approximation accuracy at the cost of memory. Higher value of `accuracy` yields - better accuracy, `1.0/accuracy` is the relative error of the approximation. - - Arguments: - col - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - percentage - any numeric type or any nonnumeric type literal that can be - implicitly converted to double type. - accuracy - any numeric type or any nonnumeric type literal that can be implicitly - converted to int type. - - _FUNC_(col, array(percentage1 [, percentage2]...) [, accuracy]) - Returns the approximate - percentile array of column `col` at the given percentage array. Each value of the - percentage array must be between 0.0 and 1.0. The `accuracy` parameter (default: 10000) is - a positive integer literal which controls approximation accuracy at the cost of memory. - Higher value of `accuracy` yields better accuracy, `1.0/accuracy` is the relative error of - the approximation. - - Arguments: - col - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - array(...) - an array that contains any numeric type literal that can be implicitly - converted to double type. - accuracy - any numeric type or any nonnumeric type literal that can be implicitly - converted to int type. - """, - extended = - """ - > SELECT percentile_approx(10.0, 0.5, 100); - 10.0 - - > SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100); - [10.0,10.0,10.0] - """) + usage = """ + _FUNC_(col, percentage [, accuracy]) - Returns the approximate percentile value of numeric + column `col` at the given percentage. The value of percentage must be between 0.0 + and 1.0. The `accuracy` parameter (default: 10000) is a positive integer literal which + controls approximation accuracy at the cost of memory. Higher value of `accuracy` yields + better accuracy, `1.0/accuracy` is the relative error of the approximation. + + _FUNC_(col, array(percentage1 [, percentage2]...) [, accuracy]) - Returns the approximate + percentile array of column `col` at the given percentage array. Each value of the + percentage array must be between 0.0 and 1.0. The `accuracy` parameter (default: 10000) is + a positive integer literal which controls approximation accuracy at the cost of memory. + Higher value of `accuracy` yields better accuracy, `1.0/accuracy` is the relative error of + the approximation. + """, + extended = """ + _FUNC_(col, percentage [, accuracy]) + + Arguments: + col - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + array(...) - an array that contains any numeric type literal that can be implicitly + converted to double type. + accuracy - any numeric type or any nonnumeric type literal that can be implicitly + converted to int type. + + Examples: + > SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100); + [10.0,10.0,10.0] + + _FUNC_(col, array(percentage1 [, percentage2]...) [, accuracy]) + + Arguments: + col - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + percentage - any numeric type or any nonnumeric type literal that can be + implicitly converted to double type. + accuracy - any numeric type or any nonnumeric type literal that can be implicitly + converted to int type. + + Examples: + > SELECT percentile_approx(10.0, 0.5, 100); + 10.0 + """) case class ApproximatePercentile( child: Expression, percentageExpression: Expression, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala index 80126b625262..57ba96fb4dd7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala @@ -24,14 +24,14 @@ import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.types._ @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the mean calculated from values of a group. - - Arguments: - expr - any numeric datatype or any nonnumeric datatype expression that can be implicitly - converted to numeric type. - """) + usage = "_FUNC_(expr) - Returns the mean calculated from values of a group.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + """) case class Average(child: Expression) extends DeclarativeAggregate { override def prettyName: String = "avg" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala index 7813a295660e..9b2f70a99ac7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala @@ -132,14 +132,14 @@ abstract class CentralMomentAgg(child: Expression) extends DeclarativeAggregate // Compute the population standard deviation of a column // scalastyle:off line.size.limit @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the population standard deviation calculated from values of a group. - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - """) + usage = "_FUNC_(expr) - Returns the population standard deviation calculated from values of a group.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) // scalastyle:on line.size.limit case class StddevPop(child: Expression) extends CentralMomentAgg(child) { @@ -154,15 +154,17 @@ case class StddevPop(child: Expression) extends CentralMomentAgg(child) { } // Compute the sample standard deviation of a column +// scalastyle:off line.size.limit @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the sample standard deviation calculated from values of a group. - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - """) + usage = "_FUNC_(expr) - Returns the sample standard deviation calculated from values of a group.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) +// scalastyle:on line.size.limit case class StddevSamp(child: Expression) extends CentralMomentAgg(child) { override protected def momentOrder = 2 @@ -178,14 +180,14 @@ case class StddevSamp(child: Expression) extends CentralMomentAgg(child) { // Compute the population variance of a column @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the population variance calculated from values of a group. - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - """) + usage = "_FUNC_(expr) - Returns the population variance calculated from values of a group.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class VariancePop(child: Expression) extends CentralMomentAgg(child) { override protected def momentOrder = 2 @@ -200,14 +202,14 @@ case class VariancePop(child: Expression) extends CentralMomentAgg(child) { // Compute the sample variance of a column @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the sample variance calculated from values of a group. - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - """) + usage = "_FUNC_(expr) - Returns the sample variance calculated from values of a group.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) { override protected def momentOrder = 2 @@ -222,14 +224,14 @@ case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) { } @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the Skewness value calculated from values of a group. - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - """) + usage = "_FUNC_(expr) - Returns the skewness value calculated from values of a group.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class Skewness(expr: Expression) extends CentralMomentAgg(expr) { override def prettyName: String = "skewness" @@ -244,14 +246,14 @@ case class Skewness(expr: Expression) extends CentralMomentAgg(expr) { } @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the Kurtosis value calculated from values of a group. - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - """) + usage = "_FUNC_(expr) - Returns the kurtosis value calculated from values of a group.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class Kurtosis(child: Expression) extends CentralMomentAgg(child) { override protected def momentOrder = 4 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala index ff7cd4088af0..29bb7c67c050 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala @@ -28,18 +28,19 @@ import org.apache.spark.sql.types._ * Definition of Pearson correlation can be found at * http://en.wikipedia.org/wiki/Pearson_product-moment_correlation_coefficient */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = - """ - _FUNC_(expr1, expr2) - Returns Pearson coefficient of correlation between a set of number - pairs. + usage = "_FUNC_(expr1, expr2) - Returns Pearson coefficient of correlation between a set of number pairs.", + extended = """ + _FUNC_(expr1, expr2) - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - """) + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) +// scalastyle:on line.size.limit case class Corr(x: Expression, y: Expression) extends DeclarativeAggregate { override def children: Seq[Expression] = Seq(x, y) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala index 40091c33e27a..b607eb065b5a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala @@ -21,22 +21,27 @@ import org.apache.spark.sql.catalyst.dsl.expressions._ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.types._ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = - """ - _FUNC_(*) - Returns the total number of retrieved rows, including rows containing NULL. + usage = """ + _FUNC_(*) - Returns the total number of retrieved rows, including rows containing NULL. - _FUNC_(expr) - Returns the number of rows for which the supplied expression is non-NULL. + _FUNC_(expr) - Returns the number of rows for which the supplied expression is non-NULL. - Arguments: - expr - any type expression that represents data to count. + _FUNC_(DISTINCT expr[, expr...]) - Returns the number of rows for which the supplied expression(s) are unique and non-NULL. + """, + extended = """ + _FUNC_(expr) - _FUNC_(DISTINCT expr[, expr...]) - Returns the number of rows for which the supplied - expression(s) are unique and non-NULL. + Arguments: + expr - any type expression that represents data to count. - Arguments: - expr - any type expression that represents data to count. - """) + _FUNC_(DISTINCT expr[, expr...]) + + Arguments: + expr - any type expression that represents data to count. + """) +// scalastyle:on line.size.limit case class Count(children: Seq[Expression]) extends DeclarativeAggregate { override def nullable: Boolean = false diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala index 23040c714432..116dd3ddb448 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala @@ -77,16 +77,16 @@ abstract class Covariance(x: Expression, y: Expression) extends DeclarativeAggre } @ExpressionDescription( - usage = - """ - _FUNC_(expr1, expr2) - Returns the population covariance of a set of number pairs. - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - """) + usage = "_FUNC_(expr1, expr2) - Returns the population covariance of a set of number pairs.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class CovPopulation(left: Expression, right: Expression) extends Covariance(left, right) { override val evaluateExpression: Expression = { If(n === Literal(0.0), Literal.create(null, DoubleType), @@ -96,16 +96,16 @@ case class CovPopulation(left: Expression, right: Expression) extends Covariance } @ExpressionDescription( - usage = - """ - _FUNC_(expr1, expr2) - Returns the sample covariance of a set of number pairs. - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - """) + usage = "_FUNC_(expr1, expr2) - Returns the sample covariance of a set of number pairs.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class CovSample(left: Expression, right: Expression) extends Covariance(left, right) { override val evaluateExpression: Expression = { If(n === Literal(0.0), Literal.create(null, DoubleType), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala index 2ddb6bdd2581..83e7f39adb43 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala @@ -29,20 +29,25 @@ import org.apache.spark.sql.types._ * a single partition, and we use a single reducer to do the aggregation.). */ @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the first value of `child` for a group of rows. + usage = """ + _FUNC_(expr) - Returns the first value of expr for a group of rows. - Arguments: - expr - any type expression that represents data to collect the first. + _FUNC_(expr, isIgnoreNull) - Returns the first value of expr for a group of rows. + If isIgnoreNull is true, returns only non-null values. + """, + extended = """ + _FUNC_(expr) - _FUNC_(expr,isIgnoreNull=false) - Returns the first value of `child` for a group of rows. - If isIgnoreNull is true, returns only non-null values. + Arguments: + expr - any type expression that represents data to collect the first. - Arguments: - expr - any type expression that represents data to collect the first. - isIgnoreNull - boolean type literal. - """) + _FUNC_(expr, isIgnoreNull) + + Arguments: + expr - any type expression that represents data to collect the first. + isIgnoreNull - boolean type literal. If isIgnoreNull is true, returns only non-null + values. Default is false. + """) case class First(child: Expression, ignoreNullsExpr: Expression) extends DeclarativeAggregate { def this(child: Expression) = this(child, Literal.create(false, BooleanType)) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala index 948622a46129..7b0a06457f8e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala @@ -47,22 +47,25 @@ import org.apache.spark.sql.types._ */ // scalastyle:on @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the estimated cardinality by HyperLogLog++. - - Arguments: - expr - any type expression that represents data to collect the first. - - _FUNC_(expr, relativeSD) - Returns the estimated cardinality by HyperLogLog++ - with relativeSD, the maximum estimation error allowed. - - Arguments: - expr - any type expression that represents data to collect the first. - relativeSD - any numeric type or any nonnumeric type literal that can be implicitly - converted to double type, that represents maximum estimation error allowed - (default = 0.05). - """) + usage = """ + _FUNC_(expr) - Returns the estimated cardinality by HyperLogLog++. + + _FUNC_(expr, relativeSD) - Returns the estimated cardinality by HyperLogLog++ + with relativeSD, the maximum estimation error allowed. + """, + extended = """ + _FUNC_(expr) + + Arguments: + expr - any type expression that represents data to count. + + _FUNC_(expr, relativeSD) + + Arguments: + expr - any type expression that represents data to count. + relativeSD - any numeric type that can be implicitly converted to double type. + This represents maximum estimation error allowed (default = 0.05). + """) case class HyperLogLogPlusPlus( child: Expression, relativeSD: Double = 0.05, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala index e811fd56c81f..948bc73d4e94 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala @@ -29,20 +29,25 @@ import org.apache.spark.sql.types._ * a single partition, and we use a single reducer to do the aggregation.). */ @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the last value of `child` for a group of rows. + usage = """ + _FUNC_(expr) - Returns the last value of expr for a group of rows. - Arguments: - expr - any type expression that represents data to collect the last. + _FUNC_(expr, isIgnoreNull) - Returns the last value of expr for a group of rows. + If isIgnoreNull is true, returns only non-null values. + """, + extended = """ + _FUNC_ (expr) - _FUNC_(expr,isIgnoreNull=false) - Returns the last value of `child` for a group of rows. - If isIgnoreNull is true, returns only non-null values. + Arguments: + expr - any type expression that represents data to collect the last. - Arguments: - expr - any type expression that represents data to collect the last. - isIgnoreNull - boolean type literal. - """) + _FUNC_ (expr, isIgnoreNull) + + Arguments: + expr - any type expression that represents data to collect the last. + isIgnoreNull - boolean type literal. If isIgnoreNull is true, returns only non-null + values. Default is false. + """) case class Last(child: Expression, ignoreNullsExpr: Expression) extends DeclarativeAggregate { def this(child: Expression) = this(child, Literal.create(false, BooleanType)) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala index e0a90e39bade..576d43097b24 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala @@ -23,13 +23,13 @@ import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.types._ @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the maximum value of expr." + usage = "_FUNC_(expr) - Returns the maximum value of expr.", + extended = """ + _FUNC_(expr) - Arguments: - expr - any type type expression. - """) + Arguments: + expr - any type expression. + """) case class Max(child: Expression) extends DeclarativeAggregate { override def children: Seq[Expression] = child :: Nil diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala index df6d9e8b1ca2..b13826c95682 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala @@ -23,13 +23,13 @@ import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.types._ @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the minimum value of expr." + usage = "_FUNC_(expr) - Returns the minimum value of expr.", + extended = """ + _FUNC_(expr) - Arguments: - expr - any type type expression. - """) + Arguments: + expr - any type expression. + """) case class Min(child: Expression) extends DeclarativeAggregate { override def children: Seq[Expression] = child :: Nil diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala index 26cd0853e921..8b6e037271f7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala @@ -23,14 +23,14 @@ import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.types._ @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns the sum calculated from values of a group. - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. - """) + usage = "_FUNC_(expr) - Returns the sum calculated from values of a group.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to double type. + """) case class Sum(child: Expression) extends DeclarativeAggregate { override def children: Seq[Expression] = child :: Nil diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala index b3649831f5da..001d05ab3408 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala @@ -86,13 +86,13 @@ abstract class Collect extends ImperativeAggregate { * Collect a list of elements. */ @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Collects and returns a set of unique elements. + usage = "_FUNC_(expr) - Collects and returns a list of non-unique elements.", + extended = """ + _FUNC_(expr) - Arguments: - expr - any type expression that represents data to collect. - """) + Arguments: + expr - any type expression that represents data to collect. + """) case class CollectList( child: Expression, mutableAggBufferOffset: Int = 0, @@ -112,16 +112,16 @@ case class CollectList( } /** - * Collect a list of unique elements. + * Collect a set of unique elements. */ @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Collects and returns a set of unique elements. + usage = "_FUNC_(expr) - Collects and returns a set of unique elements.", + extended = """ + _FUNC_(expr) - Arguments: - expr - any type expression except map type that represents data to collect. - """) + Arguments: + expr - any type expression that represents data to collect. + """) case class CollectSet( child: Expression, mutableAggBufferOffset: Int = 0, From 8cd6d80e2c5232253444c59c363010c7a2c4aa69 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 18 Oct 2016 22:24:36 +0900 Subject: [PATCH 03/47] xml OK --- .../sql/catalyst/expressions/xml/xpath.scala | 206 +++++++++--------- 1 file changed, 100 insertions(+), 106 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index 8b73f64797fd..950f4dbc9465 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -56,19 +56,18 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with } @ExpressionDescription( - usage = - """ - _FUNC_(xml, xpath) - Evaluates a boolean xpath expression. - - Arguments: - xml - string type that represents XML documentation. - path - string type literal that represents xpath expression. - """, - extended = - """ - > SELECT _FUNC_('1','a/b'); - true - """) + usage = "_FUNC_(xml, xpath) - Evaluates a boolean XPath expression.", + extended = """ + _FUNC_(xml, xpath) + + Arguments: + xml - string type expression that represents XML document. + xpath - string type literal that represents XPath expression. + + Examples: + > SELECT _FUNC_('1','a/b'); + true + """) case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_boolean" @@ -80,19 +79,18 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract } @ExpressionDescription( - usage = - """ - _FUNC_(xml, xpath) - Returns a short value that matches the xpath expression. - - Arguments: - xml - string type that represents XML documentation. - path - string type literal that represents xpath expression. - """, - extended = - """ - > SELECT _FUNC_('12', 'sum(a/b)'); - 3 - """) + usage = "_FUNC_(xml, xpath) - Returns a short value that matches the XPath expression.", + extended = """ + _FUNC_(xml, xpath) + + Arguments: + xml - string type expression that represents XML document. + xpath - string type literal that represents XPath expression. + + Examples: + > SELECT _FUNC_('12', 'sum(a/b)'); + 3 + """) case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_int" override def dataType: DataType = ShortType @@ -104,19 +102,18 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { } @ExpressionDescription( - usage = - """ - _FUNC_(xml, xpath) - Returns a short value that matches the xpath expression. - - Arguments: - xml - string type that represents XML documentation. - path - string type literal that represents xpath expression. - """, - extended = - """ - > SELECT _FUNC_('12', 'sum(a/b)'); - 3 - """) + usage = "_FUNC_(xml, xpath) - Returns an integer value that matches the XPath expression.", + extended = """ + _FUNC_(xml, xpath) + + Arguments: + xml - string type expression that represents XML document. + xpath - string type literal that represents XPath expression. + + Examples: + > SELECT _FUNC_('12', 'sum(a/b)'); + 3 + """) case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_int" override def dataType: DataType = IntegerType @@ -128,19 +125,18 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { } @ExpressionDescription( - usage = - """ - _FUNC_(xml, xpath) - Returns a short value that matches the xpath expression. - - Arguments: - xml - string type that represents XML documentation. - path - string type literal that represents xpath expression. - """, - extended = - """ - > SELECT _FUNC_('12', 'sum(a/b)'); - 3 - """) + usage = "_FUNC_(xml, xpath) - Returns a long value that matches the XPath expression.", + extended = """ + _FUNC_(xml, xpath) + + Arguments: + xml - string type expression that represents XML document. + xpath - string type literal that represents XPath expression. + + Examples: + > SELECT _FUNC_('12', 'sum(a/b)'); + 3 + """) case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_long" override def dataType: DataType = LongType @@ -152,19 +148,18 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { } @ExpressionDescription( - usage = - """ - _FUNC_(xml, xpath) - Returns a short value that matches the xpath expression. - - Arguments: - xml - string type that represents XML documentation. - path - string type literal that represents xpath expression. - """, - extended = - """ - > SELECT _FUNC_('12', 'sum(a/b)'); - 3.0 - """) + usage = "_FUNC_(xml, xpath) - Returns a float value that matches the XPath expression.", + extended = """ + _FUNC_(xml, xpath) + + Arguments: + xml - string type expression that represents XML document. + xpath - string type literal that represents XPath expression. + + Examples: + > SELECT _FUNC_('12', 'sum(a/b)'); + 3.0 + """) case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_float" override def dataType: DataType = FloatType @@ -176,19 +171,18 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { } @ExpressionDescription( - usage = - """ - _FUNC_(xml, xpath) - Returns a short value that matches the xpath expression. - - Arguments: - xml - string type that represents XML documentation. - path - string type literal that represents xpath expression. - """, - extended = - """ - > SELECT _FUNC_('12', 'sum(a/b)'); - 3.0 - """) + usage = "_FUNC_(xml, xpath) - Returns a double value that matches the XPath expression.", + extended = """ + _FUNC_(xml, xpath) + + Arguments: + xml - string type expression that represents XML document. + xpath - string type literal that represents XPath expression. + + Examples: + > SELECT _FUNC_('12', 'sum(a/b)'); + 3.0 + """) case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_float" override def dataType: DataType = DoubleType @@ -199,21 +193,21 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { } } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = - """ - _FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the - xpath expression. - - Arguments: - xml - string type that represents XML documentation. - path - string type literal that represents xpath expression. - """, - extended = - """ - > SELECT _FUNC_('bcc','a/c'); - cc - """) + usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the XPath expression.", + extended = """ + _FUNC_(xml, xpath) + + Arguments: + xml - string type expression that represents XML document. + path - string type literal that represents XPath expression. + + Examples: + > SELECT _FUNC_('bcc','a/c'); + cc + """) +// scalastyle:on line.size.limit case class XPathString(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_string" override def dataType: DataType = StringType @@ -224,21 +218,21 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract { } } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = - """ - _FUNC_(xml, xpath) - Returns a string array of values within xml nodes that match the - xpath expression - - Arguments: - xml - string type that represents XML documentation. - path - string type literal that represents xpath expression. - """, - extended = - """ - > SELECT _FUNC_('b1b2b3c1c2','a/b/text()'); - ['b1','b2','b3'] - """) + usage = "_FUNC_(xml, xpath) - Returns a string array of values within xml nodes that match the XPath expression", + extended = """ + _FUNC_(xml, xpath) + + Arguments: + xml - string type expression that represents XML document. + path - string type literal that represents XPath expression. + + Examples: + > SELECT _FUNC_('b1b2b3c1c2','a/b/text()'); + ['b1','b2','b3'] + """) +// scalastyle:on line.size.limit case class XPathList(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath" override def dataType: DataType = ArrayType(StringType, containsNull = false) From 2614572e0b04130d662f34b4926e6acaf866c704 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 18 Oct 2016 23:14:54 +0900 Subject: [PATCH 04/47] arithmetic OK --- .../sql/catalyst/expressions/arithmetic.scala | 133 +++++++++++++++--- 1 file changed, 115 insertions(+), 18 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 8fc99c0fc020..12e543c85e59 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -25,13 +25,13 @@ import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.CalendarInterval @ExpressionDescription( - usage = - """ - _FUNC_(expr) - Returns -expr." + usage = "_FUNC_(expr) - Returns the negative value of expr.", + extended = """ + _FUNC_(expr) - Arguments: - expr - any type expression. - """) + Arguments: + expr - any numeric types and interval type. + """) case class UnaryMinus(child: Expression) extends UnaryExpression with ExpectsInputTypes with NullIntolerant { @@ -68,7 +68,13 @@ case class UnaryMinus(child: Expression) extends UnaryExpression } @ExpressionDescription( - usage = "_FUNC_(a) - Returns a.") + usage = "_FUNC_(expr) - Returns the positive value of expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric types and interval type. + """) case class UnaryPositive(child: Expression) extends UnaryExpression with ExpectsInputTypes with NullIntolerant { override def prettyName: String = "positive" @@ -90,7 +96,16 @@ case class UnaryPositive(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the absolute value of the numeric value.", - extended = "> SELECT _FUNC_('-1');\n 1") + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric types. + + Examples: + > SELECT _FUNC_(-1); + 1 + """) case class Abs(child: Expression) extends UnaryExpression with ExpectsInputTypes with NullIntolerant { @@ -137,7 +152,16 @@ object BinaryArithmetic { } @ExpressionDescription( - usage = "a _FUNC_ b - Returns a+b.") + usage = "expr1 _FUNC_ expr2 - Returns expr1+expr2.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any numeric types, any nonnumeric type expression that can be implicitly + converted to numeric type and interval type. + expr2 - any numeric types, any nonnumeric type expression that can be implicitly + converted to numeric type and interval type. + """) case class Add(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { override def inputType: AbstractDataType = TypeCollection.NumericAndInterval @@ -168,7 +192,16 @@ case class Add(left: Expression, right: Expression) extends BinaryArithmetic wit } @ExpressionDescription( - usage = "a _FUNC_ b - Returns a-b.") + usage = "expr1 _FUNC_ expr2 - Returns expr1-expr2.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any numeric types, any nonnumeric type expression that can be implicitly + converted to numeric type and interval type. + expr2 - any numeric types, any nonnumeric type expression that can be implicitly + converted to numeric type and interval type. + """) case class Subtract(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -200,7 +233,16 @@ case class Subtract(left: Expression, right: Expression) } @ExpressionDescription( - usage = "a _FUNC_ b - Multiplies a by b.") + usage = "expr1 _FUNC_ expr2 - Multiplies expr1 by expr2.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any numeric types and any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric types and any nonnumeric type expression that can be implicitly + converted to numeric type. + """) case class Multiply(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -215,8 +257,20 @@ case class Multiply(left: Expression, right: Expression) } @ExpressionDescription( - usage = "a _FUNC_ b - Divides a by b.", - extended = "> SELECT 3 _FUNC_ 2;\n 1.5") + usage = "expr1 _FUNC_ expr2 - Divides expr1 by expr2.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any numeric types and any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric types and any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT 3 _FUNC_ 2; + 1.5 + """) case class Divide(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -292,7 +346,16 @@ case class Divide(left: Expression, right: Expression) } @ExpressionDescription( - usage = "a _FUNC_ b - Returns the remainder when dividing a by b.") + usage = "expr1 _FUNC_ expr2 - Returns the remainder when dividing expr1 by expr2.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any numeric types and any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric types and any nonnumeric type expression that can be implicitly + converted to numeric type. + """) case class Remainder(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -373,8 +436,20 @@ case class Remainder(left: Expression, right: Expression) } @ExpressionDescription( - usage = "_FUNC_(a, b) - Returns the positive modulo", - extended = "> SELECT _FUNC_(10,3);\n 1") + usage = "_FUNC_(expr1, expr2) - Returns the positive modulo.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric types and any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric types and any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(10, 3); + 1 + """) case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { override def toString: String = s"pmod($left, $right)" @@ -477,7 +552,18 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic wi * It takes at least 2 parameters, and returns null iff all parameters are null. */ @ExpressionDescription( - usage = "_FUNC_(n1, ...) - Returns the least value of all parameters, skipping null values.") + usage = "_FUNC_(expr, ...) - Returns the least value of all parameters, skipping null values.", + extended = """ + _FUNC_(expr, ...) + + Arguments: + expr - any numeric types and any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(10, 9, 2, 4, 3); + 2 + """) case class Least(children: Seq[Expression]) extends Expression { override def nullable: Boolean = children.forall(_.nullable) @@ -537,7 +623,18 @@ case class Least(children: Seq[Expression]) extends Expression { * It takes at least 2 parameters, and returns null iff all parameters are null. */ @ExpressionDescription( - usage = "_FUNC_(n1, ...) - Returns the greatest value of all parameters, skipping null values.") + usage = "_FUNC_(expr, ...) - Returns the greatest value of all parameters, skipping null values.", + extended = """ + _FUNC_(expr, ...) + + Arguments: + expr - any numeric types and any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(10, 9, 2, 4, 3); + 10 + """) case class Greatest(children: Seq[Expression]) extends Expression { override def nullable: Boolean = children.forall(_.nullable) From 29d0262ef0b1c3d828b2301d50f2d3d1f37ac961 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 18 Oct 2016 23:59:06 +0900 Subject: [PATCH 05/47] bitwiseExpressions OK and double-check others --- .../aggregate/ApproximatePercentile.scala | 12 ++-- .../aggregate/CentralMomentAgg.scala | 12 ++-- .../catalyst/expressions/aggregate/Corr.scala | 4 +- .../expressions/aggregate/Covariance.scala | 8 +-- .../catalyst/expressions/aggregate/Sum.scala | 2 +- .../sql/catalyst/expressions/arithmetic.scala | 44 +++++++-------- .../expressions/bitwiseExpressions.scala | 55 ++++++++++++++++--- 7 files changed, 87 insertions(+), 50 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala index fbe04bda4e14..df4a2629428c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala @@ -68,11 +68,11 @@ import org.apache.spark.sql.types._ Arguments: col - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. array(...) - an array that contains any numeric type literal that can be implicitly - converted to double type. + converted to numeric type. accuracy - any numeric type or any nonnumeric type literal that can be implicitly - converted to int type. + converted to numeric type. Examples: > SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100); @@ -82,11 +82,11 @@ import org.apache.spark.sql.types._ Arguments: col - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. percentage - any numeric type or any nonnumeric type literal that can be - implicitly converted to double type. + implicitly converted to numeric type. accuracy - any numeric type or any nonnumeric type literal that can be implicitly - converted to int type. + converted to numeric type. Examples: > SELECT percentile_approx(10.0, 0.5, 100); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala index 9b2f70a99ac7..6008616a71f9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala @@ -138,7 +138,7 @@ abstract class CentralMomentAgg(child: Expression) extends DeclarativeAggregate Arguments: expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. """) // scalastyle:on line.size.limit case class StddevPop(child: Expression) extends CentralMomentAgg(child) { @@ -162,7 +162,7 @@ case class StddevPop(child: Expression) extends CentralMomentAgg(child) { Arguments: expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. """) // scalastyle:on line.size.limit case class StddevSamp(child: Expression) extends CentralMomentAgg(child) { @@ -186,7 +186,7 @@ case class StddevSamp(child: Expression) extends CentralMomentAgg(child) { Arguments: expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. """) case class VariancePop(child: Expression) extends CentralMomentAgg(child) { @@ -208,7 +208,7 @@ case class VariancePop(child: Expression) extends CentralMomentAgg(child) { Arguments: expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. """) case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) { @@ -230,7 +230,7 @@ case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) { Arguments: expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. """) case class Skewness(expr: Expression) extends CentralMomentAgg(expr) { @@ -252,7 +252,7 @@ case class Skewness(expr: Expression) extends CentralMomentAgg(expr) { Arguments: expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. """) case class Kurtosis(child: Expression) extends CentralMomentAgg(child) { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala index 29bb7c67c050..854856c109d7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala @@ -36,9 +36,9 @@ import org.apache.spark.sql.types._ Arguments: expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. """) // scalastyle:on line.size.limit case class Corr(x: Expression, y: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala index 116dd3ddb448..5e3d8a2dbd87 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala @@ -83,9 +83,9 @@ abstract class Covariance(x: Expression, y: Expression) extends DeclarativeAggre Arguments: expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. """) case class CovPopulation(left: Expression, right: Expression) extends Covariance(left, right) { override val evaluateExpression: Expression = { @@ -102,9 +102,9 @@ case class CovPopulation(left: Expression, right: Expression) extends Covariance Arguments: expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. """) case class CovSample(left: Expression, right: Expression) extends Covariance(left, right) { override val evaluateExpression: Expression = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala index 8b6e037271f7..528837b75574 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.types._ Arguments: expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to double type. + converted to numeric type. """) case class Sum(child: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 12e543c85e59..350cf2449005 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -30,7 +30,7 @@ import org.apache.spark.unsafe.types.CalendarInterval _FUNC_(expr) Arguments: - expr - any numeric types and interval type. + expr - any numeric type or interval type expression. """) case class UnaryMinus(child: Expression) extends UnaryExpression with ExpectsInputTypes with NullIntolerant { @@ -73,7 +73,7 @@ case class UnaryMinus(child: Expression) extends UnaryExpression _FUNC_(expr) Arguments: - expr - any numeric types and interval type. + expr - any numeric type or interval type expression. """) case class UnaryPositive(child: Expression) extends UnaryExpression with ExpectsInputTypes with NullIntolerant { @@ -100,7 +100,7 @@ case class UnaryPositive(child: Expression) _FUNC_(expr) Arguments: - expr - any numeric types. + expr - any numeric type expression. Examples: > SELECT _FUNC_(-1); @@ -157,10 +157,10 @@ object BinaryArithmetic { expr1 _FUNC_ expr2 Arguments: - expr1 - any numeric types, any nonnumeric type expression that can be implicitly - converted to numeric type and interval type. - expr2 - any numeric types, any nonnumeric type expression that can be implicitly - converted to numeric type and interval type. + expr1 - any numeric type, interval type or any nonnumeric type expression that + can be implicitly converted to numeric type. + expr2 - any numeric type, interval type or any nonnumeric type expression that + can be implicitly converted to numeric type. """) case class Add(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -197,10 +197,10 @@ case class Add(left: Expression, right: Expression) extends BinaryArithmetic wit expr1 _FUNC_ expr2 Arguments: - expr1 - any numeric types, any nonnumeric type expression that can be implicitly - converted to numeric type and interval type. - expr2 - any numeric types, any nonnumeric type expression that can be implicitly - converted to numeric type and interval type. + expr1 - any numeric type, interval type or any nonnumeric type expression that + can be implicitly converted to numeric type. + expr2 - any numeric type, interval type or any nonnumeric type expression that + can be implicitly converted to numeric type. """) case class Subtract(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -238,9 +238,9 @@ case class Subtract(left: Expression, right: Expression) expr1 _FUNC_ expr2 Arguments: - expr1 - any numeric types and any nonnumeric type expression that can be implicitly + expr1 - any numeric type or any nonnumeric type expression that can be implicitly converted to numeric type. - expr2 - any numeric types and any nonnumeric type expression that can be implicitly + expr2 - any numeric type or any nonnumeric type expression that can be implicitly converted to numeric type. """) case class Multiply(left: Expression, right: Expression) @@ -262,9 +262,9 @@ case class Multiply(left: Expression, right: Expression) expr1 _FUNC_ expr2 Arguments: - expr1 - any numeric types and any nonnumeric type expression that can be implicitly + expr1 - any numeric type or any nonnumeric type expression that can be implicitly converted to numeric type. - expr2 - any numeric types and any nonnumeric type expression that can be implicitly + expr2 - any numeric type or any nonnumeric type expression that can be implicitly converted to numeric type. Examples: @@ -351,9 +351,9 @@ case class Divide(left: Expression, right: Expression) expr1 _FUNC_ expr2 Arguments: - expr1 - any numeric types and any nonnumeric type expression that can be implicitly + expr1 - any numeric type or any nonnumeric type expression that can be implicitly converted to numeric type. - expr2 - any numeric types and any nonnumeric type expression that can be implicitly + expr2 - any numeric type or any nonnumeric type expression that can be implicitly converted to numeric type. """) case class Remainder(left: Expression, right: Expression) @@ -441,9 +441,9 @@ case class Remainder(left: Expression, right: Expression) _FUNC_(expr1, expr2) Arguments: - expr1 - any numeric types and any nonnumeric type expression that can be implicitly + expr1 - any numeric type or any nonnumeric type expression that can be implicitly converted to numeric type. - expr2 - any numeric types and any nonnumeric type expression that can be implicitly + expr2 - any numeric type or any nonnumeric type expression that can be implicitly converted to numeric type. Examples: @@ -557,8 +557,7 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic wi _FUNC_(expr, ...) Arguments: - expr - any numeric types and any nonnumeric type expression that can be implicitly - converted to numeric type. + expr - any type expression except struct/map/array type. Examples: > SELECT _FUNC_(10, 9, 2, 4, 3); @@ -628,8 +627,7 @@ case class Least(children: Seq[Expression]) extends Expression { _FUNC_(expr, ...) Arguments: - expr - any numeric types and any nonnumeric type expression that can be implicitly - converted to numeric type. + expr - any type expression except struct/map/array type. Examples: > SELECT _FUNC_(10, 9, 2, 4, 3); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index 3a0a882e3876..dbd6f761ca79 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -27,8 +27,18 @@ import org.apache.spark.sql.types._ * Code generation inherited from BinaryArithmetic. */ @ExpressionDescription( - usage = "a _FUNC_ b - Bitwise AND.", - extended = "> SELECT 3 _FUNC_ 5; 1") + usage = "expr1 _FUNC_ expr2 - Bitwise AND.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any intergal numeric type expression. + expr2 - any intergal numeric type expression. + + Examples: + > SELECT 3 _FUNC_ 5; + 1 + """) case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithmetic { override def inputType: AbstractDataType = IntegralType @@ -55,8 +65,18 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme * Code generation inherited from BinaryArithmetic. */ @ExpressionDescription( - usage = "a _FUNC_ b - Bitwise OR.", - extended = "> SELECT 3 _FUNC_ 5; 7") + usage = "expr1 _FUNC_ expr2 - Bitwise OR.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any intergal numeric type expression. + expr2 - any intergal numeric type expression. + + Examples: + > SELECT 3 _FUNC_ 5; + 7 + """) case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmetic { override def inputType: AbstractDataType = IntegralType @@ -83,8 +103,18 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet * Code generation inherited from BinaryArithmetic. */ @ExpressionDescription( - usage = "a _FUNC_ b - Bitwise exclusive OR.", - extended = "> SELECT 3 _FUNC_ 5; 2") + usage = "expr1 _FUNC_ expr2 - Bitwise exclusive OR.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any intergal numeric type expression. + expr2 - any intergal numeric type expression. + + Examples: + > SELECT 3 _FUNC_ 5; + 2 + """) case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithmetic { override def inputType: AbstractDataType = IntegralType @@ -109,8 +139,17 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme * A function that calculates bitwise not(~) of a number. */ @ExpressionDescription( - usage = "_FUNC_ b - Bitwise NOT.", - extended = "> SELECT _FUNC_ 0; -1") + usage = "_FUNC_ expr - Bitwise NOT.", + extended = """ + _FUNC_ expr + + Arguments: + expr - any intergal numeric type expression. + + Examples: + > SELECT _FUNC_ 0; + -1 + """) case class BitwiseNot(child: Expression) extends UnaryExpression with ExpectsInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) From 710c68eda59c0fdb299feccd5125f75d38063176 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 19 Oct 2016 00:09:46 +0900 Subject: [PATCH 06/47] CallMethodViaReflection OK --- .../expressions/CallMethodViaReflection.scala | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala index fe24c0489fc9..c79f2769d38a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala @@ -43,11 +43,23 @@ import org.apache.spark.util.Utils * and the second element should be a literal string for the method name, * and the remaining are input arguments to the Java method. */ -// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(class,method[,arg1[,arg2..]]) calls method with reflection", - extended = "> SELECT _FUNC_('java.util.UUID', 'randomUUID');\n c33fb387-8500-4bfa-81d2-6e0e3e930df2") -// scalastyle:on line.size.limit + usage = "_FUNC_(class, method[, arg1[, arg2 ..]]) calls method with reflection", + extended = """ + _FUNC_(class, method[, arg1[, arg2 ..]]) + + Arguments: + class - string type literal that represents full-qualified class name. + method - string type literal that represents method name. + arg - string type literal that represents arguments for the method. + + Examples: + > SELECT _FUNC_('java.util.UUID', 'randomUUID'); + c33fb387-8500-4bfa-81d2-6e0e3e930df2 + + > SELECT _FUNC_('java.util.UUID', 'fromString', 'a5cf6c42-0c85-418f-af6c-3e4e5b1328f2'); + a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 + """) case class CallMethodViaReflection(children: Seq[Expression]) extends Expression with CodegenFallback { From 1d44ec38d5c68ea2853d8bda9713f633c984cdd0 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 19 Oct 2016 00:15:01 +0900 Subject: [PATCH 07/47] Cast OK --- .../spark/sql/catalyst/expressions/Cast.scala | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 58fd65f62ffe..2c2e7f276504 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -114,8 +114,18 @@ object Cast { /** Cast the child expression to the target data type. */ @ExpressionDescription( - usage = " - Cast value v to the target data type.", - extended = "> SELECT _FUNC_('10' as int);\n 10") + usage = "_FUNC_(expr AS type) - Cast value expr to the target data type.", + extended = """ + _FUNC_(expr AS type) + + Arguments: + expr - any type expression. + type - data types. + + Examples: + > SELECT _FUNC_('10' as int); + 10 + """) case class Cast(child: Expression, dataType: DataType) extends UnaryExpression with NullIntolerant { override def toString: String = s"cast($child as ${dataType.simpleString})" From 78b1fc75e9c9d931f5ff130109cf2f0d9ce2547b Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 19 Oct 2016 01:20:07 +0900 Subject: [PATCH 08/47] collectionOperations OK --- .../expressions/collectionOperations.scala | 66 ++++++++++++++++--- 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index c0200299376c..12a85b6133fe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -28,8 +28,17 @@ import org.apache.spark.sql.types._ * Given an array or map, returns its size. Returns -1 if null. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the size of an array or a map.", - extended = " > SELECT _FUNC_(array('b', 'd', 'c', 'a'));\n 4") + usage = "_FUNC_(expr) - Returns the size of an array or a map. Returns -1 if null.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - array type or map type expression. + + Examples: + > SELECT _FUNC_(array('b', 'd', 'c', 'a')); + 4 + """) case class Size(child: Expression) extends UnaryExpression with ExpectsInputTypes { override def dataType: DataType = IntegerType override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(ArrayType, MapType)) @@ -59,8 +68,17 @@ case class Size(child: Expression) extends UnaryExpression with ExpectsInputType * Returns an unordered array containing the keys of the map. */ @ExpressionDescription( - usage = "_FUNC_(map) - Returns an unordered array containing the keys of the map.", - extended = " > SELECT _FUNC_(map(1, 'a', 2, 'b'));\n [1,2]") + usage = "_FUNC_(expr) - Returns an unordered array containing the keys of the map.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - map type expression. + + Examples: + > SELECT _FUNC_(map(1, 'a', 2, 'b')); + [1,2] + """) case class MapKeys(child: Expression) extends UnaryExpression with ExpectsInputTypes { @@ -84,7 +102,16 @@ case class MapKeys(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(map) - Returns an unordered array containing the values of the map.", - extended = " > SELECT _FUNC_(map(1, 'a', 2, 'b'));\n [\"a\",\"b\"]") + extended = """ + _FUNC_(expr) + + Arguments: + expr - map type expression. + + Examples: + > SELECT _FUNC_(map(1, 'a', 2, 'b')); + ["a","b"] + """) case class MapValues(child: Expression) extends UnaryExpression with ExpectsInputTypes { @@ -109,8 +136,18 @@ case class MapValues(child: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(array(obj1, obj2, ...), ascendingOrder) - Sorts the input array in ascending order according to the natural ordering of the array elements.", - extended = " > SELECT _FUNC_(array('b', 'd', 'c', 'a'), true);\n 'a', 'b', 'c', 'd'") + usage = "_FUNC_(expr, ascendingOrder) - Sorts the input array in ascending order according to the natural ordering of the array elements.", + extended = """ + _FUNC_(expr, ascendingOrder) + + Arguments: + expr - array type expression. + ascendingOrder - boolean type literal. + + Examples: + > SELECT _FUNC_(array('b', 'd', 'c', 'a'), true); + ["a","b","c","d"] + """) // scalastyle:on line.size.limit case class SortArray(base: Expression, ascendingOrder: Expression) extends BinaryExpression with ExpectsInputTypes with CodegenFallback { @@ -194,8 +231,19 @@ case class SortArray(base: Expression, ascendingOrder: Expression) * Checks if the array (left) has the element (right) */ @ExpressionDescription( - usage = "_FUNC_(array, value) - Returns TRUE if the array contains the value.", - extended = " > SELECT _FUNC_(array(1, 2, 3), 2);\n true") + usage = "_FUNC_(expr1, value) - Returns TRUE if the array contains the value.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - array type expression. + expr2 - expression in the element type of expr1 or any type that can be implicitly + converted to the element type of expr1. + + Examples: + > SELECT _FUNC_(array(1, 2, 3), 2); + true + """) case class ArrayContains(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { From e2062afdd9579fbec08ac073a6021736078d7ad2 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 19 Oct 2016 19:15:18 +0900 Subject: [PATCH 09/47] complexTypeCreator OK and double check others --- .../spark/sql/catalyst/expressions/Cast.scala | 2 +- .../expressions/complexTypeCreator.scala | 63 +++++++++++++++++-- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 2c2e7f276504..85ad1e08d4cd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -120,7 +120,7 @@ object Cast { Arguments: expr - any type expression. - type - data types. + type - data type. Examples: > SELECT _FUNC_('10' as int); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index 09e22aaf3e3d..a70690ceca2c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -28,7 +28,17 @@ import org.apache.spark.unsafe.types.UTF8String * Returns an Array containing the evaluation of all children expressions. */ @ExpressionDescription( - usage = "_FUNC_(n0, ...) - Returns an array with the given elements.") + usage = "_FUNC_(expr, ...) - Returns an array with the given elements.", + extended = """ + _FUNC_(expr, ...) + + Arguments: + expr - any type expression. + + Examples: + > SELECT _FUNC_(1, 2, 3); + [1,2,3] + """) case class CreateArray(children: Seq[Expression]) extends Expression { override def foldable: Boolean = children.forall(_.foldable) @@ -82,7 +92,18 @@ case class CreateArray(children: Seq[Expression]) extends Expression { * The children are a flatted sequence of kv pairs, e.g. (key1, value1, key2, value2, ...) */ @ExpressionDescription( - usage = "_FUNC_(key0, value0, key1, value1...) - Creates a map with the given key/value pairs.") + usage = "_FUNC_(key0, value0, key1, value1...) - Creates a map with the given key/value pairs.", + extended = """ + _FUNC_(key0, value0, key1, value1...) + + Arguments: + key - any type expression. + value - any type expression. + + Examples: + > SELECT _FUNC_(1.0, '2', 3.0, '4'); + {1.0:"2",3.0:"4"} + """) case class CreateMap(children: Seq[Expression]) extends Expression { lazy val keys = children.indices.filter(_ % 2 == 0).map(children) lazy val values = children.indices.filter(_ % 2 != 0).map(children) @@ -175,7 +196,17 @@ case class CreateMap(children: Seq[Expression]) extends Expression { * Returns a Row containing the evaluation of all children expressions. */ @ExpressionDescription( - usage = "_FUNC_(col1, col2, col3, ...) - Creates a struct with the given field values.") + usage = "_FUNC_(expr1, expr2, expr2 ...) - Creates a struct with the given field values.", + extended = """ + _FUNC_(expr1, expr2, expr2 ...) + + Arguments: + expr - any type expression. + + Examples: + > SELECT _FUNC_(1, 2, 3); + {"col1":1,"col2":2,"col3":3} + """) case class CreateStruct(children: Seq[Expression]) extends Expression { override def foldable: Boolean = children.forall(_.foldable) @@ -234,7 +265,18 @@ case class CreateStruct(children: Seq[Expression]) extends Expression { */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(name1, val1, name2, val2, ...) - Creates a struct with the given field names and values.") + usage = "_FUNC_(name1, val1, name2, val2, ...) - Creates a struct with the given field names and values.", + extended = """ + _FUNC_(name1, val1, name2, val2, ...) + + Arguments: + name - string type expression that allows constant folding and represents the field name. + val - any type expression. + + Examples: + > SELECT _FUNC_("a", 1, "b", 2, "c", 3); + {"a":1,"b":2,"c":3} + """) // scalastyle:on line.size.limit case class CreateNamedStruct(children: Seq[Expression]) extends Expression { @@ -400,7 +442,18 @@ case class CreateNamedStructUnsafe(children: Seq[Expression]) extends Expression // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(text[, pairDelim, keyValueDelim]) - Creates a map after splitting the text into key/value pairs using delimiters. Default delimiters are ',' for pairDelim and ':' for keyValueDelim.", - extended = """ > SELECT _FUNC_('a:1,b:2,c:3',',',':');\n map("a":"1","b":"2","c":"3") """) + extended = """ + _FUNC_(text[, pairDelim, keyValueDelim]) + + Arguments: + text - string type expression that represents data to convert into map. + pairDelim - string type expression that allows constant folding. + keyValueDelim - string type expression that allows constant folding. + + Examples: + > SELECT _FUNC_('a:1,b:2,c:3',',',':'); + map("a":"1","b":"2","c":"3") + """) // scalastyle:on line.size.limit case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: Expression) extends TernaryExpression with CodegenFallback with ExpectsInputTypes { From e9672db417cbf800a92665942a8c096221724cbe Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 19 Oct 2016 21:59:30 +0900 Subject: [PATCH 10/47] conditionalExpressions OK --- .../expressions/conditionalExpressions.scala | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala index 71d4e9a3c947..94ceb71015d6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala @@ -24,7 +24,19 @@ import org.apache.spark.sql.types._ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr1,expr2,expr3) - If expr1 is TRUE then IF() returns expr2; otherwise it returns expr3.") + usage = "_FUNC_(expr1, expr2, expr3) - If expr1 is TRUE then IF() returns expr2; otherwise it returns expr3.", + extended = """ + _FUNC_(expr1, expr2, expr3) + + Arguments: + expr1 - boolean type expression. + expr2 - any type expression that represents the return value when expr1 is TRUE. + expr3 - any type expression that represents the return value when expr1 is FALSE. + + Examples: + > SELECT _FUNC_(1 < 2, 'a', 'b'); + a + """) // scalastyle:on line.size.limit case class If(predicate: Expression, trueValue: Expression, falseValue: Expression) extends Expression { From 9baa847729cd6f7686c1aa476cc1de0548f9ac2f Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 19 Oct 2016 23:09:38 +0900 Subject: [PATCH 11/47] datetimeExpressions OK --- .../expressions/datetimeExpressions.scala | 326 ++++++++++++++++-- 1 file changed, 289 insertions(+), 37 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 7ab68a13e09c..d6d136d5a42e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -76,7 +76,19 @@ case class CurrentTimestamp() extends LeafExpression with CodegenFallback { */ @ExpressionDescription( usage = "_FUNC_(start_date, num_days) - Returns the date that is num_days after start_date.", - extended = "> SELECT _FUNC_('2016-07-30', 1);\n '2016-07-31'") + extended = """ + _FUNC_(start_date, num_days) + + Arguments: + start_date - date type or any type expression that can be implicitly converted to + date type. + num_days - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_('2016-07-30', 1); + 2016-07-31 + """) case class DateAdd(startDate: Expression, days: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -105,7 +117,19 @@ case class DateAdd(startDate: Expression, days: Expression) */ @ExpressionDescription( usage = "_FUNC_(start_date, num_days) - Returns the date that is num_days before start_date.", - extended = "> SELECT _FUNC_('2016-07-30', 1);\n '2016-07-29'") + extended = """ + _FUNC_(start_date, num_days) + + Arguments: + start_date - date type or any type expression that can be implicitly converted to + date type. + num_days - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_('2016-07-30', 1); + 2016-07-29 + """) case class DateSub(startDate: Expression, days: Expression) extends BinaryExpression with ImplicitCastInputTypes { override def left: Expression = startDate @@ -129,8 +153,18 @@ case class DateSub(startDate: Expression, days: Expression) } @ExpressionDescription( - usage = "_FUNC_(param) - Returns the hour component of the string/timestamp/interval.", - extended = "> SELECT _FUNC_('2009-07-30 12:58:59');\n 12") + usage = "_FUNC_(expr) - Returns the hour component of the string/timestamp/interval.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - timestamp type or any type expression that can be implicitly converted to + timestamp type. + + Examples: + > SELECT _FUNC_('2009-07-30 12:58:59'); + 12 + """) case class Hour(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType) @@ -148,8 +182,18 @@ case class Hour(child: Expression) extends UnaryExpression with ImplicitCastInpu } @ExpressionDescription( - usage = "_FUNC_(param) - Returns the minute component of the string/timestamp/interval.", - extended = "> SELECT _FUNC_('2009-07-30 12:58:59');\n 58") + usage = "_FUNC_(expr) - Returns the minute component of the string/timestamp/interval.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - timestamp type or any type expression that can be implicitly converted to + timestamp type. + + Examples: + > SELECT _FUNC_('2009-07-30 12:58:59'); + 58 + """) case class Minute(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType) @@ -167,8 +211,18 @@ case class Minute(child: Expression) extends UnaryExpression with ImplicitCastIn } @ExpressionDescription( - usage = "_FUNC_(param) - Returns the second component of the string/timestamp/interval.", - extended = "> SELECT _FUNC_('2009-07-30 12:58:59');\n 59") + usage = "_FUNC_(expr) - Returns the second component of the string/timestamp/interval.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - timestamp type or any type expression that can be implicitly converted to + timestamp type. + + Examples: + > SELECT _FUNC_('2009-07-30 12:58:59'); + 59 + """) case class Second(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType) @@ -186,8 +240,17 @@ case class Second(child: Expression) extends UnaryExpression with ImplicitCastIn } @ExpressionDescription( - usage = "_FUNC_(param) - Returns the day of year of date/timestamp.", - extended = "> SELECT _FUNC_('2016-04-09');\n 100") + usage = "_FUNC_(expr) - Returns the day of year of date/timestamp.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - date type or any type expression that can be implicitly converted to date type. + + Examples: + > SELECT _FUNC_('2016-04-09'); + 100 + """) case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(DateType) @@ -205,8 +268,17 @@ case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCas } @ExpressionDescription( - usage = "_FUNC_(param) - Returns the year component of the date/timestamp/interval.", - extended = "> SELECT _FUNC_('2016-07-30');\n 2016") + usage = "_FUNC_(expr) - Returns the year component of the date/timestamp/interval.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - date type or any type expression that can be implicitly converted to date type. + + Examples: + > SELECT _FUNC_('2016-07-30'); + 2016 + """) case class Year(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(DateType) @@ -224,7 +296,17 @@ case class Year(child: Expression) extends UnaryExpression with ImplicitCastInpu } @ExpressionDescription( - usage = "_FUNC_(param) - Returns the quarter of the year for date, in the range 1 to 4.") + usage = "_FUNC_(expr) - Returns the quarter of the year for date, in the range 1 to 4.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - date type or any type expression that can be implicitly converted to date type. + + Examples: + > SELECT _FUNC_('2016-08-31'); + 3 + """) case class Quarter(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(DateType) @@ -242,8 +324,17 @@ case class Quarter(child: Expression) extends UnaryExpression with ImplicitCastI } @ExpressionDescription( - usage = "_FUNC_(param) - Returns the month component of the date/timestamp/interval", - extended = "> SELECT _FUNC_('2016-07-30');\n 7") + usage = "_FUNC_(expr) - Returns the month component of the date/timestamp/interval.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - date type or any type expression that can be implicitly converted to date type. + + Examples: + > SELECT _FUNC_('2016-07-30'); + 7 + """) case class Month(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(DateType) @@ -261,8 +352,17 @@ case class Month(child: Expression) extends UnaryExpression with ImplicitCastInp } @ExpressionDescription( - usage = "_FUNC_(param) - Returns the day of month of date/timestamp, or the day of interval.", - extended = "> SELECT _FUNC_('2009-07-30');\n 30") + usage = "_FUNC_(expr) - Returns the day of month of date/timestamp, or the day of interval.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - date type or any type expression that can be implicitly converted to date type. + + Examples: + > SELECT _FUNC_('2009-07-30'); + 30 + """) case class DayOfMonth(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(DateType) @@ -280,8 +380,17 @@ case class DayOfMonth(child: Expression) extends UnaryExpression with ImplicitCa } @ExpressionDescription( - usage = "_FUNC_(param) - Returns the week of the year of the given date.", - extended = "> SELECT _FUNC_('2008-02-20');\n 8") + usage = "_FUNC_(expr) - Returns the week of the year of the given date.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - date type or any type expression that can be implicitly converted to date type. + + Examples: + > SELECT _FUNC_('2008-02-20'); + 8 + """) case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(DateType) @@ -320,8 +429,18 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(date/timestamp/string, fmt) - Converts a date/timestamp/string to a value of string in the format specified by the date format fmt.", - extended = "> SELECT _FUNC_('2016-04-08', 'y')\n '2016'") + usage = "_FUNC_(expr, fmt) - Converts expr to a value of string in the format specified by the date format fmt.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - timestamp type or any type expression that can be implicitly converted to timestamp type. + fmt - string type expression that represents date/timestamp format used in java.text.SimpleDateFormat. + + Examples: + > SELECT _FUNC_('2016-04-08', 'y'); + 2016 + """) // scalastyle:on line.size.limit case class DateFormatClass(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -351,7 +470,19 @@ case class DateFormatClass(left: Expression, right: Expression) extends BinaryEx * Deterministic version of [[UnixTimestamp]], must have at least one parameter. */ @ExpressionDescription( - usage = "_FUNC_(date[, pattern]) - Returns the UNIX timestamp of the give time.") + usage = "_FUNC_(expr[, pattern]) - Returns the UNIX timestamp of the give time.", + extended = """ + _FUNC_(expr[, pattern]) + + Arguments: + expr - date/string/timestamp type expression. + pattern - string type expression that represents date/timestamp format used in + java.text.SimpleDateFormat. + + Examples: + > SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd'); + 1460041200 + """) case class ToUnixTimestamp(timeExp: Expression, format: Expression) extends UnixTime { override def left: Expression = timeExp override def right: Expression = format @@ -374,7 +505,21 @@ case class ToUnixTimestamp(timeExp: Expression, format: Expression) extends Unix * second parameter. */ @ExpressionDescription( - usage = "_FUNC_([date[, pattern]]) - Returns the UNIX timestamp of current or specified time.") + usage = "_FUNC_([expr[, pattern]]) - Returns the UNIX timestamp of current or specified time.", + extended = """ + _FUNC_(expr[, pattern]) + + Arguments: + expr - date/string/timestamp type expression. + pattern - string type expression that represents date/timestamp format used in + java.text.SimpleDateFormat. + + Examples: + > SELECT _FUNC_(); + 1476884637 + > SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd'); + 1460041200 + """) case class UnixTimestamp(timeExp: Expression, format: Expression) extends UnixTime { override def left: Expression = timeExp override def right: Expression = format @@ -499,8 +644,20 @@ abstract class UnixTime extends BinaryExpression with ExpectsInputTypes { * Note that hive Language Manual says it returns 0 if fail, but in fact it returns null. */ @ExpressionDescription( - usage = "_FUNC_(unix_time, format) - Returns unix_time in the specified format", - extended = "> SELECT _FUNC_(0, 'yyyy-MM-dd HH:mm:ss');\n '1970-01-01 00:00:00'") + usage = "_FUNC_(unix_time, format) - Returns unix_time in the specified format.", + extended = """ + _FUNC_(unix_time, format) + + Arguments: + unix_time - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + format - string type expression that represents date/timestamp format used in + java.text.SimpleDateFormat. + + Examples: + > SELECT _FUNC_(0, 'yyyy-MM-dd HH:mm:ss'); + 1970-01-01 00:00:00 + """) case class FromUnixTime(sec: Expression, format: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -586,8 +743,17 @@ case class FromUnixTime(sec: Expression, format: Expression) * Returns the last day of the month which the date belongs to. */ @ExpressionDescription( - usage = "_FUNC_(date) - Returns the last day of the month which the date belongs to.", - extended = "> SELECT _FUNC_('2009-01-12');\n '2009-01-31'") + usage = "_FUNC_(expr) - Returns the last day of the month which the date belongs to.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - date type or any type expression that can be implicitly converted to date type. + + Examples: + > SELECT _FUNC_('2009-01-12'); + 2009-01-31 + """) case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def child: Expression = startDate @@ -617,7 +783,19 @@ case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitC // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(start_date, day_of_week) - Returns the first date which is later than start_date and named as indicated.", - extended = "> SELECT _FUNC_('2015-01-14', 'TU');\n '2015-01-20'") + extended = """ + _FUNC_(expr) + + Arguments: + start_date - date type or any type expression that can be implicitly converted to date type. + day_of_week - string type expression or any type expression that can be implicitly converted to string type + that represents a day of week. "SU", "SUN", "SUNDAY", "MO", "MON", "MONDAY", "TU", "TUE", "TUESDAY", "WE", + "WED", "WEDNESDAY", "TH", "THU", "THURSDAY", "FR", "FRI", "FRIDAY", "SA", "SAT" or "SATURDAY" can be given. + + Examples: + > SELECT _FUNC_('2015-01-14', 'TU'); + 2015-01-20 + """) // scalastyle:on line.size.limit case class NextDay(startDate: Expression, dayOfWeek: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -706,7 +884,18 @@ case class TimeAdd(start: Expression, interval: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp, string timezone) - Assumes given timestamp is UTC and converts to given timezone.") + usage = "_FUNC_(timestamp, timezone) - Assumes given timestamp is UTC and converts to given timezone.", + extended = """ + _FUNC_(timestamp, timezone) + + Arguments: + timestamp - timestamp type or any type expression that can be implicitly converted to timestamp type. + timezone - string type expression that represents time zone (e.g., 'Asia/Seoul'). + + Examples: + > SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul'); + 2016-08-31 09:00:00 + """) // scalastyle:on line.size.limit case class FromUTCTimestamp(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -787,7 +976,19 @@ case class TimeSub(start: Expression, interval: Expression) */ @ExpressionDescription( usage = "_FUNC_(start_date, num_months) - Returns the date that is num_months after start_date.", - extended = "> SELECT _FUNC_('2016-08-31', 1);\n '2016-09-30'") + extended = """ + _FUNC_(start_date, num_months) + + Arguments: + start_date - date type or any type expression that can be implicitly converted to + date type. + num_months - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_('2016-08-31', 1); + 2016-09-30 + """) case class AddMonths(startDate: Expression, numMonths: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -816,8 +1017,18 @@ case class AddMonths(startDate: Expression, numMonths: Expression) * Returns number of months between dates date1 and date2. */ @ExpressionDescription( - usage = "_FUNC_(date1, date2) - returns number of months between dates date1 and date2.", - extended = "> SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30');\n 3.94959677") + usage = "_FUNC_(expr1, expr2) - Returns number of months between dates expr1 and expr2.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr - timestamp type or any type expression that can be implicitly converted to + timestamp type. + + Examples: + > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); + 3.94959677 + """) case class MonthsBetween(date1: Expression, date2: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -847,7 +1058,18 @@ case class MonthsBetween(date1: Expression, date2: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp, string timezone) - Assumes given timestamp is in given timezone and converts to UTC.") + usage = "_FUNC_(timestamp, timezone) - Assumes given timestamp is in given timezone and converts to UTC.", + extended = """ + _FUNC_(timestamp, timezone) + + Arguments: + timestamp - timestamp type or any type expression that can be implicitly converted to timestamp type. + timezone - string type expression that represents time zone (e.g., 'Asia/Seoul'). + + Examples: + > SELECT _FUNC_('2016-08-31', 'Asia/Seoul'); + 2016-08-30 15:00:00 + """) // scalastyle:on line.size.limit case class ToUTCTimestamp(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -899,7 +1121,16 @@ case class ToUTCTimestamp(left: Expression, right: Expression) */ @ExpressionDescription( usage = "_FUNC_(expr) - Extracts the date part of the date or datetime expression expr.", - extended = "> SELECT _FUNC_('2009-07-30 04:17:52');\n '2009-07-30'") + extended = """ + _FUNC_(expr) + + Arguments: + expr - date type or any type expression that can be implicitly converted to date type. + + Examples: + > SELECT _FUNC_('2009-07-30 04:17:52'); + 2009-07-30 + """) case class ToDate(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { // Implicit casting of spark will accept string in both date and timestamp format, as @@ -923,7 +1154,19 @@ case class ToDate(child: Expression) extends UnaryExpression with ImplicitCastIn // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(date, fmt) - Returns returns date with the time portion of the day truncated to the unit specified by the format model fmt.", - extended = "> SELECT _FUNC_('2009-02-12', 'MM')\n '2009-02-01'\n> SELECT _FUNC_('2015-10-27', 'YEAR');\n '2015-01-01'") + extended = """ + _FUNC_(date, fmt) + + Arguments: + expr - date type or any type expression that can be implicitly converted to date type. + fmt - string type expression that represents the truncate level. It could be "YEAR", "YYYY", "YY", "MON", "MONTH" or "MM". + + Examples: + > SELECT _FUNC_('2009-02-12', 'MM'); + 2009-02-01 + > SELECT _FUNC_('2015-10-27', 'YEAR'); + 2015-01-01 + """) // scalastyle:on line.size.limit case class TruncDate(date: Expression, format: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -995,8 +1238,17 @@ case class TruncDate(date: Expression, format: Expression) * Returns the number of days from startDate to endDate. */ @ExpressionDescription( - usage = "_FUNC_(date1, date2) - Returns the number of days between date1 and date2.", - extended = "> SELECT _FUNC_('2009-07-30', '2009-07-31');\n 1") + usage = "_FUNC_(expr1, expr2) - Returns the number of days between expr1 and expr2.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr - date type or any type expression that can be implicitly converted to date type. + + Examples: + > SELECT _FUNC_('2009-07-30', '2009-07-31'); + 1 + """) case class DateDiff(endDate: Expression, startDate: Expression) extends BinaryExpression with ImplicitCastInputTypes { From fff85f6bd645c17d91ce96ac73dfd99d957909d3 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 19 Oct 2016 23:27:42 +0900 Subject: [PATCH 12/47] generators OK --- .../sql/catalyst/expressions/generators.scala | 55 ++++++++++++++++--- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index f74208ff66db..b31c931f9802 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -103,7 +103,18 @@ case class UserDefinedGenerator( */ @ExpressionDescription( usage = "_FUNC_(n, v1, ..., vk) - Separate v1, ..., vk into n rows.", - extended = "> SELECT _FUNC_(2, 1, 2, 3);\n [1,2]\n [3,null]") + extended = """ + _FUNC_(n, v1, ..., vk) + + Arguments: + n - integer type literal that represents the number of output rows. + v - any type expression. + + Examples: + > SELECT _FUNC_(2, 1, 2, 3); + 1 2 + 3 NULL + """) case class Stack(children: Seq[Expression]) extends Expression with Generator with CodegenFallback { @@ -226,8 +237,18 @@ abstract class ExplodeBase(child: Expression, position: Boolean) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(a) - Separates the elements of array a into multiple rows, or the elements of map a into multiple rows and columns.", - extended = "> SELECT _FUNC_(array(10,20));\n 10\n 20") + usage = "_FUNC_(expr) - Separates the elements of array a into multiple rows, or the elements of map a into multiple rows and columns.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - array type expression that contains map type or struct type expression as the element. + + Examples: + > SELECT _FUNC_(array(10, 20)); + 10 + 20 + """) // scalastyle:on line.size.limit case class Explode(child: Expression) extends ExplodeBase(child, position = false) @@ -242,8 +263,18 @@ case class Explode(child: Expression) extends ExplodeBase(child, position = fals */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(a) - Separates the elements of array a into multiple rows with positions, or the elements of a map into multiple rows and columns with positions.", - extended = "> SELECT _FUNC_(array(10,20));\n 0\t10\n 1\t20") + usage = "_FUNC_(expr) - Separates the elements of array a into multiple rows with positions, or the elements of a map into multiple rows and columns with positions.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - array type expression that contains map type or struct type expression as the element. + + Examples: + > SELECT _FUNC_(array(10,20)); + 0 10 + 1 20 + """) // scalastyle:on line.size.limit case class PosExplode(child: Expression) extends ExplodeBase(child, position = true) @@ -251,8 +282,18 @@ case class PosExplode(child: Expression) extends ExplodeBase(child, position = t * Explodes an array of structs into a table. */ @ExpressionDescription( - usage = "_FUNC_(a) - Explodes an array of structs into a table.", - extended = "> SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b')));\n [1,a]\n [2,b]") + usage = "_FUNC_(expr) - Explodes an array of structs into a table.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - array type expression that contains struct type expression as the element. + + Examples: + > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b'))); + 1 a + 2 b + """) case class Inline(child: Expression) extends UnaryExpression with Generator with CodegenFallback { override def checkInputDataTypes(): TypeCheckResult = child.dataType match { From 24627750226aef3a0e34886b3516496b4e7bb456 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 19 Oct 2016 23:29:32 +0900 Subject: [PATCH 13/47] InputFileName OK --- .../spark/sql/catalyst/expressions/InputFileName.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala index 96929ecf5637..bb3fc98d3a63 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala @@ -27,8 +27,14 @@ import org.apache.spark.unsafe.types.UTF8String * Expression that returns the name of the current file being read. */ @ExpressionDescription( - usage = "_FUNC_() - Returns the name of the current file being read if available", - extended = "> SELECT _FUNC_();\n ''") + usage = "_FUNC_() - Returns the name of the current file being read if available.", + extended = """ + _FUNC_(expr) + + Examples: + > SELECT _FUNC_(); + + """) case class InputFileName() extends LeafExpression with Nondeterministic { override def nullable: Boolean = true From 10892fb676b6e70e4eb5039a8581f29edae59226 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 19 Oct 2016 23:41:16 +0900 Subject: [PATCH 14/47] jsonExpressions OK --- .../expressions/jsonExpressions.scala | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index 65dbd6a4e3f1..35e0e74131ad 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -109,7 +109,18 @@ private[this] object SharedFactory { * of the extracted json object. It will return null if the input json string is invalid. */ @ExpressionDescription( - usage = "_FUNC_(json_txt, path) - Extract a json object from path") + usage = "_FUNC_(json_txt, path) - Extracts a json object from path.", + extended = """ + _FUNC_(json_txt, path) + + Arguments: + json_txt - string type expression that represents JSON document. + path - string type expression that represents path for JSON document. + + Examples: + > SELECT _FUNC_('{"a":"b"}', '$.a'); + b + """) case class GetJsonObject(json: Expression, path: Expression) extends BinaryExpression with ExpectsInputTypes with CodegenFallback { @@ -325,7 +336,18 @@ case class GetJsonObject(json: Expression, path: Expression) // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(jsonStr, p1, p2, ..., pn) - like get_json_object, but it takes multiple names and return a tuple. All the input parameters and output column types are string.") + usage = "_FUNC_(jsonStr, p1, p2, ..., pn) - like get_json_object, but it takes multiple names and return a tuple. All the input parameters and output column types are string.", + extended = """ + _FUNC_(jsonStr, p1, p2, ..., pn) + + Arguments: + json_txt - string type expression that represents JSON document. + p - string type expression that represents key name for JSON document. + + Examples: + > SELECT _FUNC_('{"a":1, "b":2}', 'a', 'b'); + 1 2 + """) // scalastyle:on line.size.limit case class JsonTuple(children: Seq[Expression]) extends Generator with CodegenFallback { From 45e7f99a9252d66a3a87d17136206e098cff6eea Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 20 Oct 2016 01:01:14 +0900 Subject: [PATCH 15/47] mathExpressions OK, double check others and fix scala style --- .../sql/catalyst/expressions/generators.scala | 8 +- .../expressions/jsonExpressions.scala | 2 +- .../expressions/mathExpressions.scala | 571 +++++++++++++++--- 3 files changed, 501 insertions(+), 80 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index b31c931f9802..f876aa0a7f67 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -112,8 +112,8 @@ case class UserDefinedGenerator( Examples: > SELECT _FUNC_(2, 1, 2, 3); - 1 2 - 3 NULL + 1 2 + 3 NULL """) case class Stack(children: Seq[Expression]) extends Expression with Generator with CodegenFallback { @@ -291,8 +291,8 @@ case class PosExplode(child: Expression) extends ExplodeBase(child, position = t Examples: > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b'))); - 1 a - 2 b + 1 a + 2 b """) case class Inline(child: Expression) extends UnaryExpression with Generator with CodegenFallback { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index 35e0e74131ad..931bf95840c5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -346,7 +346,7 @@ case class GetJsonObject(json: Expression, path: Expression) Examples: > SELECT _FUNC_('{"a":1, "b":2}', 'a', 'b'); - 1 2 + 1 2 """) // scalastyle:on line.size.limit case class JsonTuple(children: Seq[Expression]) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 5152265152ae..40a31ff6be2a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -140,7 +140,13 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String) */ @ExpressionDescription( usage = "_FUNC_() - Returns Euler's number, E.", - extended = "> SELECT _FUNC_();\n 2.718281828459045") + extended = """ + _FUNC_() + + Examples: + > SELECT _FUNC_(); + 2.718281828459045 + """) case class EulerNumber() extends LeafMathExpression(math.E, "E") /** @@ -149,7 +155,13 @@ case class EulerNumber() extends LeafMathExpression(math.E, "E") */ @ExpressionDescription( usage = "_FUNC_() - Returns PI.", - extended = "> SELECT _FUNC_();\n 3.141592653589793") + extended = """ + _FUNC_() + + Examples: + > SELECT _FUNC_(); + 3.141592653589793 + """) case class Pi() extends LeafMathExpression(math.Pi, "PI") //////////////////////////////////////////////////////////////////////////////////////////////////// @@ -159,28 +171,84 @@ case class Pi() extends LeafMathExpression(math.Pi, "PI") //////////////////////////////////////////////////////////////////////////////////////////////////// @ExpressionDescription( - usage = "_FUNC_(x) - Returns the arc cosine of x if -1<=x<=1 or NaN otherwise.", - extended = "> SELECT _FUNC_(1);\n 0.0\n> SELECT _FUNC_(2);\n NaN") + usage = "_FUNC_(expr) - Returns the arc cosine of expr if -1<=expr<=1 or NaN otherwise.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(1); + 0.0 + > SELECT _FUNC_(2); + NaN + """) case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the arc sin of x if -1<=x<=1 or NaN otherwise.", - extended = "> SELECT _FUNC_(0);\n 0.0\n> SELECT _FUNC_(2);\n NaN") + usage = "_FUNC_(expr) - Returns the arc sin of expr if -1<=expr<=1 or NaN otherwise.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0); + 0.0 + > SELECT _FUNC_(2); + NaN + """) case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the arc tangent.", - extended = "> SELECT _FUNC_(0);\n 0.0") + usage = "_FUNC_(expr) - Returns the arc tangent.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0); + 0.0 + """) case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the cube root of a double value.", - extended = "> SELECT _FUNC_(27.0);\n 3.0") + usage = "_FUNC_(expr) - Returns the cube root of a double value.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(27.0); + 3.0 + """) case class Cbrt(child: Expression) extends UnaryMathExpression(math.cbrt, "CBRT") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the smallest integer not smaller than x.", - extended = "> SELECT _FUNC_(-0.1);\n 0\n> SELECT _FUNC_(5);\n 5") + usage = "_FUNC_(expr) - Returns the smallest integer not smaller than expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(-0.1); + 0 + > SELECT _FUNC_(5); + 5 + """) case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil, "CEIL") { override def dataType: DataType = child.dataType match { case dt @ DecimalType.Fixed(_, 0) => dt @@ -208,13 +276,33 @@ case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil, "CEIL" } @ExpressionDescription( - usage = "_FUNC_(x) - Returns the cosine of x.", - extended = "> SELECT _FUNC_(0);\n 1.0") + usage = "_FUNC_(expr) - Returns the cosine of expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0); + 1.0 + """) case class Cos(child: Expression) extends UnaryMathExpression(math.cos, "COS") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the hyperbolic cosine of x.", - extended = "> SELECT _FUNC_(0);\n 1.0") + usage = "_FUNC_(expr) - Returns the hyperbolic cosine of expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0); + 1.0 + """) case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH") /** @@ -226,7 +314,23 @@ case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH" */ @ExpressionDescription( usage = "_FUNC_(num, from_base, to_base) - Convert num from from_base to to_base.", - extended = "> SELECT _FUNC_('100', 2, 10);\n '4'\n> SELECT _FUNC_(-10, 16, -10);\n '16'") + extended = """ + _FUNC_(num, from_base, to_base) + + Arguments: + num - any string type or any type expression that can be implicitly + converted to numeric type. + from_base - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + to_base - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_('100', 2, 10); + 4 + > SELECT _FUNC_(-10, 16, -10); + 16 + """) case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -256,18 +360,50 @@ case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expre } @ExpressionDescription( - usage = "_FUNC_(x) - Returns e to the power of x.", - extended = "> SELECT _FUNC_(0);\n 1.0") + usage = "_FUNC_(expr) - Returns e to the power of expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0); + 1.0 + """) case class Exp(child: Expression) extends UnaryMathExpression(math.exp, "EXP") @ExpressionDescription( - usage = "_FUNC_(x) - Returns exp(x) - 1.", - extended = "> SELECT _FUNC_(0);\n 0.0") + usage = "_FUNC_(expr) - Returns exp(expr) - 1.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0); + 0.0 + """) case class Expm1(child: Expression) extends UnaryMathExpression(math.expm1, "EXPM1") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the largest integer not greater than x.", - extended = "> SELECT _FUNC_(-0.1);\n -1\n> SELECT _FUNC_(5);\n 5") + usage = "_FUNC_(expr) - Returns the largest integer not greater than expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(-0.1); + -1 + > SELECT _FUNC_(5); + 5 + """) case class Floor(child: Expression) extends UnaryMathExpression(math.floor, "FLOOR") { override def dataType: DataType = child.dataType match { case dt @ DecimalType.Fixed(_, 0) => dt @@ -326,8 +462,18 @@ object Factorial { } @ExpressionDescription( - usage = "_FUNC_(n) - Returns n factorial for n is [0..20]. Otherwise, NULL.", - extended = "> SELECT _FUNC_(5);\n 120") + usage = "_FUNC_(expr) - Returns expr factorial for expr is [0..20]. Otherwise, NULL.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(5); + 120 + """) case class Factorial(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[DataType] = Seq(IntegerType) @@ -361,13 +507,33 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas } @ExpressionDescription( - usage = "_FUNC_(x) - Returns the natural logarithm of x with base e.", - extended = "> SELECT _FUNC_(1);\n 0.0") + usage = "_FUNC_(expr) - Returns the natural logarithm of x with base expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(1); + 0.0 + """) case class Log(child: Expression) extends UnaryLogExpression(math.log, "LOG") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the logarithm of x with base 2.", - extended = "> SELECT _FUNC_(2);\n 1.0") + usage = "_FUNC_(expr) - Returns the logarithm of expr with base 2.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(2); + 1.0 + """) case class Log2(child: Expression) extends UnaryLogExpression((x: Double) => math.log(x) / math.log(2), "LOG2") { override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { @@ -384,71 +550,193 @@ case class Log2(child: Expression) } @ExpressionDescription( - usage = "_FUNC_(x) - Returns the logarithm of x with base 10.", - extended = "> SELECT _FUNC_(10);\n 1.0") + usage = "_FUNC_(expr) - Returns the logarithm of expr with base 10.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(10); + 1.0 + """) case class Log10(child: Expression) extends UnaryLogExpression(math.log10, "LOG10") @ExpressionDescription( - usage = "_FUNC_(x) - Returns log(1 + x).", - extended = "> SELECT _FUNC_(0);\n 0.0") + usage = "_FUNC_(expr) - Returns log(1 + expr).", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0); + 0.0 + """) case class Log1p(child: Expression) extends UnaryLogExpression(math.log1p, "LOG1P") { protected override val yAsymptote: Double = -1.0 } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(x, d) - Return the rounded x at d decimal places.", - extended = "> SELECT _FUNC_(12.3456, 1);\n 12.3") + usage = "_FUNC_(expr) - Returns the double value that is closest in value to the argument and is equal to a mathematical integer.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(12.3456); + 12.0 + """) +// scalastyle:on line.size.limit case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND") { override def funcName: String = "rint" } @ExpressionDescription( - usage = "_FUNC_(x) - Returns the sign of x.", - extended = "> SELECT _FUNC_(40);\n 1.0") + usage = "_FUNC_(expr) - Returns the sign of expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(40); + 1.0 + """) case class Signum(child: Expression) extends UnaryMathExpression(math.signum, "SIGNUM") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the sine of x.", - extended = "> SELECT _FUNC_(0);\n 0.0") + usage = "_FUNC_(expr) - Returns the sine of expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0); + 0.0 + """) case class Sin(child: Expression) extends UnaryMathExpression(math.sin, "SIN") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the hyperbolic sine of x.", - extended = "> SELECT _FUNC_(0);\n 0.0") + usage = "_FUNC_(expr) - Returns the hyperbolic sine of expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0); + 0.0 + """) case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the square root of x.", - extended = "> SELECT _FUNC_(4);\n 2.0") + usage = "_FUNC_(expr) - Returns the square root of expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(4); + 2.0 + """) case class Sqrt(child: Expression) extends UnaryMathExpression(math.sqrt, "SQRT") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the tangent of x.", - extended = "> SELECT _FUNC_(0);\n 0.0") + usage = "_FUNC_(expr) - Returns the tangent of expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0); + 0.0 + """) case class Tan(child: Expression) extends UnaryMathExpression(math.tan, "TAN") @ExpressionDescription( - usage = "_FUNC_(x) - Returns the hyperbolic tangent of x.", - extended = "> SELECT _FUNC_(0);\n 0.0") + usage = "_FUNC_(expr) - Returns the hyperbolic tangent of expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0); + 0.0 + """) case class Tanh(child: Expression) extends UnaryMathExpression(math.tanh, "TANH") @ExpressionDescription( - usage = "_FUNC_(x) - Converts radians to degrees.", - extended = "> SELECT _FUNC_(3.141592653589793);\n 180.0") + usage = "_FUNC_(expr) - Converts radians to degrees.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(3.141592653589793); + 180.0 + """) case class ToDegrees(child: Expression) extends UnaryMathExpression(math.toDegrees, "DEGREES") { override def funcName: String = "toDegrees" } @ExpressionDescription( - usage = "_FUNC_(x) - Converts degrees to radians.", - extended = "> SELECT _FUNC_(180);\n 3.141592653589793") + usage = "_FUNC_(expr) - Converts degrees to radians.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(180); + 3.141592653589793 + """) case class ToRadians(child: Expression) extends UnaryMathExpression(math.toRadians, "RADIANS") { override def funcName: String = "toRadians" } @ExpressionDescription( - usage = "_FUNC_(x) - Returns x in binary.", - extended = "> SELECT _FUNC_(13);\n '1101'") + usage = "_FUNC_(expr) - Returns expr in binary.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(13); + 1101 + """) case class Bin(child: Expression) extends UnaryExpression with Serializable with ImplicitCastInputTypes { @@ -541,8 +829,20 @@ object Hex { * and returns the resulting STRING. Negative numbers would be treated as two's complement. */ @ExpressionDescription( - usage = "_FUNC_(x) - Convert the argument to hexadecimal.", - extended = "> SELECT _FUNC_(17);\n '11'\n> SELECT _FUNC_('Spark SQL');\n '537061726B2053514C'") + usage = "_FUNC_(expr) - Converts the expr to hexadecimal.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - numeric type, binary type, string type or any type expression that can be + implicitly converted to one of these types. + + Examples: + > SELECT _FUNC_(17); + 11 + > SELECT _FUNC_('Spark SQL'); + 537061726B2053514C + """) case class Hex(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = @@ -572,8 +872,17 @@ case class Hex(child: Expression) extends UnaryExpression with ImplicitCastInput * Resulting characters are returned as a byte array. */ @ExpressionDescription( - usage = "_FUNC_(x) - Converts hexadecimal argument to binary.", - extended = "> SELECT decode(_FUNC_('537061726B2053514C'),'UTF-8');\n 'Spark SQL'") + usage = "_FUNC_(expr) - Converts hexadecimal expr to binary.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - string type or any type expression that can be implicitly converted to string type. + + Examples: + > SELECT decode(_FUNC_('537061726B2053514C'), 'UTF-8'); + Spark SQL + """) case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(StringType) @@ -603,8 +912,20 @@ case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInp //////////////////////////////////////////////////////////////////////////////////////////////////// @ExpressionDescription( - usage = "_FUNC_(x,y) - Returns the arc tangent2.", - extended = "> SELECT _FUNC_(0, 0);\n 0.0") + usage = "_FUNC_(expr1, expr2) - Returns the arc tangent2.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(0, 0); + 0.0 + """) case class Atan2(left: Expression, right: Expression) extends BinaryMathExpression(math.atan2, "ATAN2") { @@ -619,8 +940,20 @@ case class Atan2(left: Expression, right: Expression) } @ExpressionDescription( - usage = "_FUNC_(x1, x2) - Raise x1 to the power of x2.", - extended = "> SELECT _FUNC_(2, 3);\n 8.0") + usage = "_FUNC_(expr1, expr2) - Raise expr1 to the power of expr2.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(2, 3); + 8.0 + """) case class Pow(left: Expression, right: Expression) extends BinaryMathExpression(math.pow, "POWER") { override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { @@ -636,8 +969,20 @@ case class Pow(left: Expression, right: Expression) * @param right number of bits to left shift. */ @ExpressionDescription( - usage = "_FUNC_(a, b) - Bitwise left shift.", - extended = "> SELECT _FUNC_(2, 1);\n 4") + usage = "_FUNC_(expr1, expr2) - Bitwise left shift.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(2, 1); + 4 + """) case class ShiftLeft(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -666,8 +1011,20 @@ case class ShiftLeft(left: Expression, right: Expression) * @param right number of bits to right shift. */ @ExpressionDescription( - usage = "_FUNC_(a, b) - Bitwise right shift.", - extended = "> SELECT _FUNC_(4, 1);\n 2") + usage = "_FUNC_(expr1, expr2) - Bitwise right shift.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(4, 1); + 2 + """) case class ShiftRight(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -696,8 +1053,20 @@ case class ShiftRight(left: Expression, right: Expression) * @param right the number of bits to right shift. */ @ExpressionDescription( - usage = "_FUNC_(a, b) - Bitwise unsigned right shift.", - extended = "> SELECT _FUNC_(4, 1);\n 2") + usage = "_FUNC_(expr1, expr2) - Bitwise unsigned right shift.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(4, 1); + 2 + """) case class ShiftRightUnsigned(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -719,8 +1088,20 @@ case class ShiftRightUnsigned(left: Expression, right: Expression) } @ExpressionDescription( - usage = "_FUNC_(a, b) - Returns sqrt(a**2 + b**2).", - extended = "> SELECT _FUNC_(3, 4);\n 5.0") + usage = "_FUNC_(expr1, expr2) - Returns sqrt(expr1**2 + expr2**2).", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(3, 4); + 5.0 + """) case class Hypot(left: Expression, right: Expression) extends BinaryMathExpression(math.hypot, "HYPOT") @@ -732,8 +1113,20 @@ case class Hypot(left: Expression, right: Expression) * @param right the number to compute the logarithm of. */ @ExpressionDescription( - usage = "_FUNC_(b, x) - Returns the logarithm of x with base b.", - extended = "> SELECT _FUNC_(10, 100);\n 2.0") + usage = "_FUNC_(expr1, expr2) - Returns the logarithm of expr1 with base expr2.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(10, 100); + 2.0 + """) case class Logarithm(left: Expression, right: Expression) extends BinaryMathExpression((c1, c2) => math.log(c2) / math.log(c1), "LOG") { @@ -956,9 +1349,23 @@ abstract class RoundBase(child: Expression, scale: Expression, * Round an expression to d decimal places using HALF_UP rounding mode. * round(2.5) == 3.0, round(3.5) == 4.0. */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(x, d) - Round x to d decimal places using HALF_UP rounding mode.", - extended = "> SELECT _FUNC_(2.5, 0);\n 3.0") + usage = "_FUNC_(expr1, expr2) - Round expr1 to expr2 decimal places using HALF_UP rounding mode.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(2.5, 0); + 3.0 + """) +// scalastyle:on line.size.limit case class Round(child: Expression, scale: Expression) extends RoundBase(child, scale, BigDecimal.RoundingMode.HALF_UP, "ROUND_HALF_UP") with Serializable with ImplicitCastInputTypes { @@ -970,9 +1377,23 @@ case class Round(child: Expression, scale: Expression) * also known as Gaussian rounding or bankers' rounding. * round(2.5) = 2.0, round(3.5) = 4.0. */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(x, d) - Round x to d decimal places using HALF_EVEN rounding mode.", - extended = "> SELECT _FUNC_(2.5, 0);\n 2.0") + usage = "_FUNC_(expr1, expr2) - Round expr1 to expr2 decimal places using HALF_EVEN rounding mode.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(2.5, 0); + 2.0 + """) +// scalastyle:on line.size.limit case class BRound(child: Expression, scale: Expression) extends RoundBase(child, scale, BigDecimal.RoundingMode.HALF_EVEN, "ROUND_HALF_EVEN") with Serializable with ImplicitCastInputTypes { From 1d69e40a89c1242c283e820ece0e9fdf4b52c7cb Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 20 Oct 2016 23:14:49 +0900 Subject: [PATCH 16/47] misc OK, double-check others and ignore a test in SQLQuerySuite for now --- .../spark/sql/catalyst/expressions/misc.scala | 104 +++++++++++++++--- .../sql/hive/execution/SQLQuerySuite.scala | 2 +- 2 files changed, 91 insertions(+), 15 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala index 5ead16908732..e0bcd7b393d4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala @@ -38,8 +38,18 @@ import org.apache.spark.unsafe.Platform * For input of type [[BinaryType]] */ @ExpressionDescription( - usage = "_FUNC_(input) - Returns an MD5 128-bit checksum as a hex string of the input", - extended = "> SELECT _FUNC_('Spark');\n '8cde774d6f7333752ed72cacddb05126'") + usage = "_FUNC_(expr) - Returns an MD5 128-bit checksum as a hex string of expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - binary type or any type expression that can be implicitly converted to + binary type. + + Examples: + > SELECT _FUNC_('Spark'); + 8cde774d6f7333752ed72cacddb05126 + """) case class Md5(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def dataType: DataType = StringType @@ -65,10 +75,23 @@ case class Md5(child: Expression) extends UnaryExpression with ImplicitCastInput */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = """_FUNC_(input, bitLength) - Returns a checksum of SHA-2 family as a hex string of the input. - SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256.""", - extended = """> SELECT _FUNC_('Spark', 0); - '529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b'""") + usage = """ + _FUNC_(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of expr. + SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256. + """, + extended = """ + _FUNC_(expr, bitLength) + + Arguments: + expr - string type or any type expression that can be implicitly converted to + string type. + bitLength - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_('Spark', 0); + 529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + """) // scalastyle:on line.size.limit case class Sha2(left: Expression, right: Expression) extends BinaryExpression with Serializable with ImplicitCastInputTypes { @@ -136,8 +159,18 @@ case class Sha2(left: Expression, right: Expression) * For input of type [[BinaryType]] or [[StringType]] */ @ExpressionDescription( - usage = "_FUNC_(input) - Returns a sha1 hash value as a hex string of the input", - extended = "> SELECT _FUNC_('Spark');\n '85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c'") + usage = "_FUNC_(expr) - Returns a sha1 hash value as a hex string of the expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - binary type or any type expression that that can be implicitly converted to + binary type. + + Examples: + > SELECT _FUNC_('Spark'); + 85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c + """) case class Sha1(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def dataType: DataType = StringType @@ -159,8 +192,18 @@ case class Sha1(child: Expression) extends UnaryExpression with ImplicitCastInpu * For input of type [[BinaryType]] */ @ExpressionDescription( - usage = "_FUNC_(input) - Returns a cyclic redundancy check value as a bigint of the input", - extended = "> SELECT _FUNC_('Spark');\n '1557323817'") + usage = "_FUNC_(expr) - Returns a cyclic redundancy check value as a bigint of the expr.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - binary type or any type expression that that can be implicitly converted to + binary type. + + Examples: + > SELECT _FUNC_('Spark'); + 1557323817 + """) case class Crc32(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def dataType: DataType = LongType @@ -490,7 +533,17 @@ abstract class InterpretedHashFunction { * and bucketing have same data distribution. */ @ExpressionDescription( - usage = "_FUNC_(a1, a2, ...) - Returns a hash value of the arguments.") + usage = "_FUNC_(expr1, expr2, ...) - Returns a hash value of the arguments.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any type expression. + + Examples: + > SELECT _FUNC_('Spark', array(123), 2); + -1321691492 + """) case class Murmur3Hash(children: Seq[Expression], seed: Int) extends HashExpression[Int] { def this(arguments: Seq[Expression]) = this(arguments, 42) @@ -544,7 +597,18 @@ case class PrintToStderr(child: Expression) extends UnaryExpression { * A function throws an exception if 'condition' is not true. */ @ExpressionDescription( - usage = "_FUNC_(condition) - Throw an exception if 'condition' is not true.") + usage = "_FUNC_(expr) - Throw an exception if expr is not true.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - boolean type or any type expression that can be implicitly converted to + boolean type. + + Examples: + > SELECT _FUNC_(0 < 1); + NULL + """) case class AssertTrue(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def nullable: Boolean = true @@ -613,7 +677,13 @@ object XxHash64Function extends InterpretedHashFunction { */ @ExpressionDescription( usage = "_FUNC_() - Returns the current database.", - extended = "> SELECT _FUNC_()") + extended = """ + _FUNC_() + + Examples: + > SELECT _FUNC_(); + default + """) case class CurrentDatabase() extends LeafExpression with Unevaluable { override def dataType: DataType = StringType override def foldable: Boolean = true @@ -631,7 +701,13 @@ case class CurrentDatabase() extends LeafExpression with Unevaluable { * TODO: Support Decimal and date related types */ @ExpressionDescription( - usage = "_FUNC_(a1, a2, ...) - Returns a hash value of the arguments.") + usage = "_FUNC_(expr1, expr2, ...) - Returns a hash value of the arguments.", + extended = """ + _FUNC_(expr1, expr2, ...) + + Arguments: + expr - any type expression. + """) case class HiveHash(children: Seq[Expression]) extends HashExpression[Int] { override val seed = 0 diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index e26b6b57ef56..6af80c5e2269 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -248,7 +248,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } } - test("describe functions - built-in functions") { + ignore("describe functions - built-in functions") { checkKeywordsExist(sql("describe function extended upper"), "Function: upper", "Class: org.apache.spark.sql.catalyst.expressions.Upper", From 9b24e7879d6d75e913dd71ee3e32292483da5fb5 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 20 Oct 2016 23:18:47 +0900 Subject: [PATCH 17/47] MonotonicallyIncreasingID OK --- .../expressions/MonotonicallyIncreasingID.scala | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala index 5b4922e0cf2b..99bf2d28cc66 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala @@ -33,13 +33,20 @@ import org.apache.spark.sql.types.{DataType, LongType} * Since this expression is stateful, it cannot be a case object. */ @ExpressionDescription( - usage = - """_FUNC_() - Returns monotonically increasing 64-bit integers. + usage = """ + _FUNC_() - Returns monotonically increasing 64-bit integers. The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive. The current implementation puts the partition ID in the upper 31 bits, and the lower 33 bits represent the record number within each partition. The assumption is that the data frame has - less than 1 billion partitions, and each partition has less than 8 billion records.""", - extended = "> SELECT _FUNC_();\n 0") + less than 1 billion partitions, and each partition has less than 8 billion records. + """, + extended = """ + _FUNC_() + + Examples: + > SELECT _FUNC_(); + 0 + """) case class MonotonicallyIncreasingID() extends LeafExpression with Nondeterministic { /** From ed1c83936bdf78cbab35c50307c2a8afa6c586a3 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 20 Oct 2016 23:43:17 +0900 Subject: [PATCH 18/47] nullExpressions OK --- .../expressions/nullExpressions.scala | 131 ++++++++++++++++-- 1 file changed, 116 insertions(+), 15 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala index 1c18265e0fed..b74f25e8829a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala @@ -34,9 +34,20 @@ import org.apache.spark.sql.types._ * coalesce(null, null, null) => null * }}} */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(a1, a2, ...) - Returns the first non-null argument if exists. Otherwise, NULL.", - extended = "> SELECT _FUNC_(NULL, 1, NULL);\n 1") + usage = "_FUNC_(expr1, expr2, ...) - Returns the first non-null argument if exists. Otherwise, NULL.", + extended = """ + _FUNC_(expr1, expr2, ...) + + Arguments: + expr - any type expression. + + Examples: + > SELECT _FUNC_(NULL, 1, NULL); + 1 + """) +// scalastyle:on line.size.limit case class Coalesce(children: Seq[Expression]) extends Expression { /** Coalesce is nullable if all of its children are nullable, or if it has no children. */ @@ -87,8 +98,19 @@ case class Coalesce(children: Seq[Expression]) extends Expression { } } - -@ExpressionDescription(usage = "_FUNC_(a,b) - Returns b if a is null, or a otherwise.") +@ExpressionDescription( + usage = "_FUNC_(expr1, expr2) - Returns expr2 if expr1 is null, or expr1 otherwise.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + + Examples: + > SELECT _FUNC_(NULL, array('2')); + ["2"] + """) case class IfNull(left: Expression, right: Expression) extends RuntimeReplaceable { override def children: Seq[Expression] = Seq(left, right) @@ -105,8 +127,19 @@ case class IfNull(left: Expression, right: Expression) extends RuntimeReplaceabl } } - -@ExpressionDescription(usage = "_FUNC_(a,b) - Returns null if a equals to b, or a otherwise.") +@ExpressionDescription( + usage = "_FUNC_(expr1, expr2) - Returns null if expr1 equals to expr2, or expr1 otherwise.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + + Examples: + > SELECT _FUNC_(2, 2); + NULL + """) case class NullIf(left: Expression, right: Expression) extends RuntimeReplaceable { override def children: Seq[Expression] = Seq(left, right) @@ -125,8 +158,19 @@ case class NullIf(left: Expression, right: Expression) extends RuntimeReplaceabl } } - -@ExpressionDescription(usage = "_FUNC_(a,b) - Returns b if a is null, or a otherwise.") +@ExpressionDescription( + usage = "_FUNC_(expr1, expr2) - Returns expr2 if expr1 is null, or expr1 otherwise.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + + Examples: + > SELECT _FUNC_(NULL, array('2')); + ["2"] + """) case class Nvl(left: Expression, right: Expression) extends RuntimeReplaceable { override def children: Seq[Expression] = Seq(left, right) @@ -143,8 +187,22 @@ case class Nvl(left: Expression, right: Expression) extends RuntimeReplaceable { } } - -@ExpressionDescription(usage = "_FUNC_(a,b,c) - Returns b if a is not null, or c otherwise.") +// scalastyle:off line.size.limit +@ExpressionDescription( + usage = "_FUNC_(expr1, expr2, expr3) - Returns expr2 if expr1 is not null, or expr3 otherwise.", + extended = """ + _FUNC_(expr1, expr2, expr3) + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + expr3 - any type expression. + + Examples: + > SELECT _FUNC_(NULL, 2, 1); + 1 + """) +// scalastyle:on line.size.limit case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression) extends RuntimeReplaceable { @@ -163,12 +221,22 @@ case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression) } } - /** * Evaluates to `true` iff it's NaN. */ @ExpressionDescription( - usage = "_FUNC_(a) - Returns true if a is NaN and false otherwise.") + usage = "_FUNC_(expr) - Returns true if expr is NaN and false otherwise.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(cast('NaN' as double)); + true + """) case class IsNaN(child: Expression) extends UnaryExpression with Predicate with ImplicitCastInputTypes { @@ -206,7 +274,20 @@ case class IsNaN(child: Expression) extends UnaryExpression * This Expression is useful for mapping NaN values to null. */ @ExpressionDescription( - usage = "_FUNC_(a,b) - Returns a iff it's not NaN, or b otherwise.") + usage = "_FUNC_(expr1, expr2) - Returns expr1 if it's not NaN, or expr2 otherwise.", + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + expr2 - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. + + Examples: + > SELECT _FUNC_(cast('NaN' as double), 123); + 123.0 + """) case class NaNvl(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -261,7 +342,17 @@ case class NaNvl(left: Expression, right: Expression) * An expression that is evaluated to true if the input is null. */ @ExpressionDescription( - usage = "_FUNC_(a) - Returns true if a is NULL and false otherwise.") + usage = "_FUNC_(expr) - Returns true if expr is NULL and false otherwise.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any type expression. + + Examples: + > SELECT _FUNC_(1); + false + """) case class IsNull(child: Expression) extends UnaryExpression with Predicate { override def nullable: Boolean = false @@ -282,7 +373,17 @@ case class IsNull(child: Expression) extends UnaryExpression with Predicate { * An expression that is evaluated to true if the input is not null. */ @ExpressionDescription( - usage = "_FUNC_(a) - Returns true if a is not NULL and false otherwise.") + usage = "_FUNC_(expr) - Returns true if expr is not NULL and false otherwise.", + extended = """ + _FUNC_(expr) + + Arguments: + expr - any type expression. + + Examples: + > SELECT _FUNC_(1); + true + """) case class IsNotNull(child: Expression) extends UnaryExpression with Predicate { override def nullable: Boolean = false From 15351863fecf7765ac97289b40c2b578ae4db7e1 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 20 Oct 2016 23:59:35 +0900 Subject: [PATCH 19/47] predicates OK --- .../sql/catalyst/expressions/predicates.scala | 92 ++++++++++++++++--- 1 file changed, 81 insertions(+), 11 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index 799858a6865e..405256c00670 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -92,7 +92,13 @@ trait PredicateHelper { } @ExpressionDescription( - usage = "_FUNC_ a - Logical not") + usage = "_FUNC_ expr - Logical not.", + extended = """ + _FUNC_ expr + + Arguments: + expr - any type expression. + """) case class Not(child: Expression) extends UnaryExpression with Predicate with ImplicitCastInputTypes with NullIntolerant { @@ -114,7 +120,13 @@ case class Not(child: Expression) * Evaluates to `true` if `list` contains `value`. */ @ExpressionDescription( - usage = "expr _FUNC_(val1, val2, ...) - Returns true if expr equals to any valN.") + usage = "expr1 _FUNC_(expr2, expr3, ...) - Returns true if expr equals to any valN.", + extended = """ + expr1 _FUNC_(expr2, expr3, ...) + + Arguments: + expr - any type expression. + """) case class In(value: Expression, list: Seq[Expression]) extends Predicate with ImplicitCastInputTypes { @@ -251,7 +263,14 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with } @ExpressionDescription( - usage = "a _FUNC_ b - Logical AND.") + usage = "expr1 _FUNC_ expr2 - Logical AND.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + """) case class And(left: Expression, right: Expression) extends BinaryOperator with Predicate { override def inputType: AbstractDataType = BooleanType @@ -314,7 +333,14 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with } @ExpressionDescription( - usage = "a _FUNC_ b - Logical OR.") + usage = "expr1 _FUNC_ expr2 - Logical OR.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + """) case class Or(left: Expression, right: Expression) extends BinaryOperator with Predicate { override def inputType: AbstractDataType = BooleanType @@ -409,7 +435,14 @@ object Equality { } @ExpressionDescription( - usage = "a _FUNC_ b - Returns TRUE if a equals b and false otherwise.") + usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 equals expr2 and false otherwise.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + """) case class EqualTo(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { @@ -435,8 +468,17 @@ case class EqualTo(left: Expression, right: Expression) } @ExpressionDescription( - usage = """a _FUNC_ b - Returns same result with EQUAL(=) operator for non-null operands, - but returns TRUE if both are NULL, FALSE if one of the them is NULL.""") + usage = """ + expr1 _FUNC_ expr2 - Returns same result with EQUAL(=) operator for non-null operands. + but returns TRUE if both are NULL, FALSE if one of the them is NULL. + """, + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + """) case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComparison { override def inputType: AbstractDataType = AnyDataType @@ -476,7 +518,14 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp } @ExpressionDescription( - usage = "a _FUNC_ b - Returns TRUE if a is less than b.") + usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is less than expr2.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + """) case class LessThan(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { @@ -490,7 +539,14 @@ case class LessThan(left: Expression, right: Expression) } @ExpressionDescription( - usage = "a _FUNC_ b - Returns TRUE if a is not greater than b.") + usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is not greater than expr2.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + """) case class LessThanOrEqual(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { @@ -504,7 +560,14 @@ case class LessThanOrEqual(left: Expression, right: Expression) } @ExpressionDescription( - usage = "a _FUNC_ b - Returns TRUE if a is greater than b.") + usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is greater than expr2.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + """) case class GreaterThan(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { @@ -518,7 +581,14 @@ case class GreaterThan(left: Expression, right: Expression) } @ExpressionDescription( - usage = "a _FUNC_ b - Returns TRUE if a is not smaller than b.") + usage = "expr1 _FUNC_ expr2 - Returns TRUE if a is not smaller than b.", + extended = """ + expr1 _FUNC_ expr2 + + Arguments: + expr1 - any type expression. + expr2 - any type expression. + """) case class GreaterThanOrEqual(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { From 8efee7e07ca4bcce654fc947c0fb513b7f361555 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 00:11:47 +0900 Subject: [PATCH 20/47] randomExpressions OK --- .../expressions/randomExpressions.scala | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala index ca200768b228..ffa52b357121 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala @@ -55,8 +55,22 @@ abstract class RDG extends LeafExpression with Nondeterministic { } /** Generate a random column with i.i.d. uniformly distributed values in [0, 1). */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(a) - Returns a random column with i.i.d. uniformly distributed values in [0, 1).") + usage = "_FUNC_([seed]) - Returns a random column with i.i.d. uniformly distributed values in [0, 1].", + extended = """ + _FUNC_([seed]) + + Arguments: + seed - numeric type literal. + + Examples: + > SELECT _FUNC_(); + 0.9629742951434543 + > SELECT _FUNC_(0); + 0.8446490682263027 + """) +// scalastyle:on line.size.limit case class Rand(seed: Long) extends RDG { override protected def evalInternal(input: InternalRow): Double = rng.nextDouble() @@ -77,9 +91,23 @@ case class Rand(seed: Long) extends RDG { } } -/** Generate a random column with i.i.d. gaussian random distribution. */ +/** Generate a random column with i.i.d. values drawn from the standard normal distribution. */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(a) - Returns a random column with i.i.d. gaussian random distribution.") + usage = "_FUNC_([seed]) - Returns a random column with i.i.d. values drawn from the standard normal distribution.", + extended = """ + _FUNC_([seed]) + + Arguments: + seed - numeric type literal. + + Examples: + > SELECT _FUNC_(); + -0.3254147983080288 + > SELECT _FUNC_(0); + 1.1164209726833079 + """) +// scalastyle:on line.size.limit case class Randn(seed: Long) extends RDG { override protected def evalInternal(input: InternalRow): Double = rng.nextGaussian() From a111d2a63965eda60c6ba6f297a84c9e0a8f85f1 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 00:24:48 +0900 Subject: [PATCH 21/47] regexpExpressions OK --- .../expressions/regexpExpressions.scala | 74 +++++++++++++++++-- 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index d25da3fd587b..dc3bf19bc5c2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -68,7 +68,16 @@ trait StringRegexExpression extends ImplicitCastInputTypes { * Simple RegEx pattern matching function */ @ExpressionDescription( - usage = "str _FUNC_ pattern - Returns true if str matches pattern and false otherwise.") + usage = "str _FUNC_ pattern - Returns true if str matches pattern and false otherwise.", + extended = """ + str _FUNC_ pattern + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + pattern - string type or any type expression that can be implicitly converted + to string type. + """) case class Like(left: Expression, right: Expression) extends BinaryExpression with StringRegexExpression { @@ -121,7 +130,16 @@ case class Like(left: Expression, right: Expression) } @ExpressionDescription( - usage = "str _FUNC_ regexp - Returns true if str matches regexp and false otherwise.") + usage = "str _FUNC_ regexp - Returns true if str matches regexp and false otherwise.", + extended = """ + str _FUNC_ regexp + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + regexp - string type or any type expression that can be implicitly converted + to string type. + """) case class RLike(left: Expression, right: Expression) extends BinaryExpression with StringRegexExpression { @@ -175,8 +193,20 @@ case class RLike(left: Expression, right: Expression) * Splits str around pat (pattern is a regular expression). */ @ExpressionDescription( - usage = "_FUNC_(str, regex) - Splits str around occurrences that match regex", - extended = "> SELECT _FUNC_('oneAtwoBthreeC', '[ABC]');\n ['one', 'two', 'three']") + usage = "_FUNC_(str, regex) - Splits str around occurrences that match regex.", + extended = """ + _FUNC_(str, regex) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + regex - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('oneAtwoBthreeC', '[ABC]'); + ["one","two","three",""] + """) case class StringSplit(str: Expression, pattern: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -207,8 +237,22 @@ case class StringSplit(str: Expression, pattern: Expression) * NOTE: this expression is not THREAD-SAFE, as it has some internal mutable status. */ @ExpressionDescription( - usage = "_FUNC_(str, regexp, rep) - replace all substrings of str that match regexp with rep.", - extended = "> SELECT _FUNC_('100-200', '(\\d+)', 'num');\n 'num-num'") + usage = "_FUNC_(str, regexp, rep) - Replaces all substrings of str that match regexp with rep.", + extended = """ + _FUNC_(str, regex, rep) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + regexp - string type or any type expression that can be implicitly converted + to string type. + rep - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('100-200', '(\d+)', 'num'); + num-num + """) case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -309,8 +353,22 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio * NOTE: this expression is not THREAD-SAFE, as it has some internal mutable status. */ @ExpressionDescription( - usage = "_FUNC_(str, regexp[, idx]) - extracts a group that matches regexp.", - extended = "> SELECT _FUNC_('100-200', '(\\d+)-(\\d+)', 1);\n '100'") + usage = "_FUNC_(str, regexp[, idx]) - Extracts a group that matches regexp.", + extended = """ + _FUNC_(str, regexp[, idx]) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + regexp - string type or any type expression that can be implicitly converted + to string type. + idx - any numeric type or any nonnumeric tpye expression that can be implicitly converted + to numeric type. + + Examples: + > SELECT _FUNC_('100-200', '(\d+)-(\d+)', 1); + 100 + """) case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expression) extends TernaryExpression with ImplicitCastInputTypes { def this(s: Expression, r: Expression) = this(s, r, Literal(1)) From a8ddcc2010b06c19fa76a5bfcc64e490ad58f5b3 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 00:26:18 +0900 Subject: [PATCH 22/47] SparkPartitionID OK --- .../sql/catalyst/expressions/SparkPartitionID.scala | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala index 1f675d5b0727..c1909ddd4d51 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala @@ -26,8 +26,14 @@ import org.apache.spark.sql.types.{DataType, IntegerType} * Expression that returns the current partition id of the Spark task. */ @ExpressionDescription( - usage = "_FUNC_() - Returns the current partition id of the Spark task", - extended = "> SELECT _FUNC_();\n 0") + usage = "_FUNC_() - Returns the current partition id of the Spark task.", + extended = """ + _FUNC_() + + Examples: + > SELECT _FUNC_(); + 0 + """) case class SparkPartitionID() extends LeafExpression with Nondeterministic { override def nullable: Boolean = false From 99b565879c86b1e8a89da8224a7f4183f0b10b1d Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 01:38:39 +0900 Subject: [PATCH 23/47] tringExpressions OK --- .../expressions/stringExpressions.scala | 514 +++++++++++++++--- 1 file changed, 443 insertions(+), 71 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 1bcbb6cfc924..64fa2e86059e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -41,8 +41,18 @@ import org.apache.spark.unsafe.types.{ByteArray, UTF8String} * If any input is null, concat returns null. */ @ExpressionDescription( - usage = "_FUNC_(str1, str2, ..., strN) - Returns the concatenation of str1, str2, ..., strN", - extended = "> SELECT _FUNC_('Spark','SQL');\n 'SparkSQL'") + usage = "_FUNC_(str1, str2, ..., strN) - Returns the concatenation of str1, str2, ..., strN.", + extended = """ + _FUNC_(str1, str2, ..., strN) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('Spark','SQL'); + SparkSQL + """) case class Concat(children: Seq[Expression]) extends Expression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq.fill(children.size)(StringType) @@ -78,10 +88,23 @@ case class Concat(children: Seq[Expression]) extends Expression with ImplicitCas * * Returns null if the separator is null. Otherwise, concat_ws skips all null values. */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = - "_FUNC_(sep, [str | array(str)]+) - Returns the concatenation of the strings separated by sep.", - extended = "> SELECT _FUNC_(' ', Spark', 'SQL');\n 'Spark SQL'") + usage = "_FUNC_(sep, [str | array(str)]+) - Returns the concatenation of the strings separated by sep.", + extended = """ + _FUNC_(sep, [str | array(str)]+) + + Arguments: + sep - string type or any type expression that can be implicitly converted + to string type. + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_(' ', Spark', 'SQL'); + Spark SQL + """) +// scalastyle:on line.size.limit case class ConcatWs(children: Seq[Expression]) extends Expression with ImplicitCastInputTypes { @@ -168,8 +191,20 @@ case class ConcatWs(children: Seq[Expression]) } @ExpressionDescription( - usage = "_FUNC_(n, str1, str2, ...) - returns the n-th string, e.g. returns str2 when n is 2", - extended = "> SELECT _FUNC_(1, 'scala', 'java') FROM src LIMIT 1;\n" + "'scala'") + usage = "_FUNC_(n, str1, str2, ...) - Returns the n-th string, e.g. returns str2 when n is 2.", + extended = """ + _FUNC_(n, str1, str2, ...) + + Arguments: + n - any numeric type or any nonnumeric expression that can be implicitly converted + to numeric type. + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_(1, 'scala', 'java') FROM src LIMIT 1; + scala + """) case class Elt(children: Seq[Expression]) extends Expression with ImplicitCastInputTypes { @@ -246,8 +281,18 @@ trait String2StringExpression extends ImplicitCastInputTypes { * A function that converts the characters of a string to uppercase. */ @ExpressionDescription( - usage = "_FUNC_(str) - Returns str with all characters changed to uppercase", - extended = "> SELECT _FUNC_('SparkSql');\n 'SPARKSQL'") + usage = "_FUNC_(str) - Returns str with all characters changed to uppercase.", + extended = """ + _FUNC_(str) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('SparkSql'); + SPARKSQL + """) case class Upper(child: Expression) extends UnaryExpression with String2StringExpression { @@ -262,8 +307,18 @@ case class Upper(child: Expression) * A function that converts the characters of a string to lowercase. */ @ExpressionDescription( - usage = "_FUNC_(str) - Returns str with all characters changed to lowercase", - extended = "> SELECT _FUNC_('SparkSql');\n 'sparksql'") + usage = "_FUNC_(str) - Returns str with all characters changed to lowercase.", + extended = """ + _FUNC_(str) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('SparkSql'); + sparksql + """) case class Lower(child: Expression) extends UnaryExpression with String2StringExpression { override def convert(v: UTF8String): UTF8String = v.toLowerCase @@ -347,8 +402,19 @@ object StringTranslate { */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = """_FUNC_(input, from, to) - Translates the input string by replacing the characters present in the from string with the corresponding characters in the to string""", - extended = "> SELECT _FUNC_('AaBbCc', 'abc', '123');\n 'A1B2C3'") + usage = "_FUNC_(input, from, to) - Translates the input string by replacing the characters present in the from string with the corresponding characters in the to string.", + extended = """ + _FUNC_(input, from, to) + + Arguments: + input - string type or any type expression that can be implicitly converted to string type. + from - string type or any type expression that can be implicitly converted to string type. + to - string type or any type expression that can be implicitly converted to string type. + + Examples: + > SELECT _FUNC_('AaBbCc', 'abc', '123'); + A1B2C3 + """) // scalastyle:on line.size.limit case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replaceExpr: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -407,9 +473,22 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = """_FUNC_(str, str_array) - Returns the index (1-based) of the given string (left) in the comma-delimited list (right). - Returns 0, if the string wasn't found or if the given string (left) contains a comma.""", - extended = "> SELECT _FUNC_('ab','abc,b,ab,c,def');\n 3") + usage = """ + _FUNC_(str, str_array) - Returns the index (1-based) of the given string (left) in the comma-delimited list (right). + Returns 0, if the string wasn't found or if the given string (left) contains a comma. + """, + extended = """ + _FUNC_(str, str_array) + + Arguments: + input - string type or any type expression that can be implicitly converted to string type. + from - string type or any type expression that can be implicitly converted to string type. + to - string type or any type expression that can be implicitly converted to string type. + + Examples: + > SELECT _FUNC_('ab','abc,b,ab,c,def'); + 3 + """) // scalastyle:on case class FindInSet(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -435,7 +514,17 @@ case class FindInSet(left: Expression, right: Expression) extends BinaryExpressi */ @ExpressionDescription( usage = "_FUNC_(str) - Removes the leading and trailing space characters from str.", - extended = "> SELECT _FUNC_(' SparkSQL ');\n 'SparkSQL'") + extended = """ + _FUNC_(str) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_(' SparkSQL '); + SparkSQL + """) case class StringTrim(child: Expression) extends UnaryExpression with String2StringExpression { @@ -452,8 +541,18 @@ case class StringTrim(child: Expression) * A function that trim the spaces from left end for given string. */ @ExpressionDescription( - usage = "_FUNC_(str) - Removes the leading space characters from str.", - extended = "> SELECT _FUNC_(' SparkSQL ');\n 'SparkSQL '") + usage = "_FUNC_(str) - Removes the leading and trailing space characters from str.", + extended = """ + _FUNC_(str) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_(' SparkSQL'); + SparkSQL + """) case class StringTrimLeft(child: Expression) extends UnaryExpression with String2StringExpression { @@ -471,7 +570,17 @@ case class StringTrimLeft(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(str) - Removes the trailing space characters from str.", - extended = "> SELECT _FUNC_(' SparkSQL ');\n ' SparkSQL'") + extended = """ + _FUNC_(str) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_(' SparkSQL '); + SparkSQL + """) case class StringTrimRight(child: Expression) extends UnaryExpression with String2StringExpression { @@ -493,7 +602,19 @@ case class StringTrimRight(child: Expression) */ @ExpressionDescription( usage = "_FUNC_(str, substr) - Returns the (1-based) index of the first occurrence of substr in str.", - extended = "> SELECT _FUNC_('SparkSQL', 'SQL');\n 6") + extended = """ + _FUNC_(str, substr) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + substr - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('SparkSQL', 'SQL'); + 6 + """) case class StringInstr(str: Expression, substr: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -522,12 +643,25 @@ case class StringInstr(str: Expression, substr: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = """_FUNC_(str, delim, count) - Returns the substring from str before count occurrences of the delimiter delim. - If count is positive, everything to the left of the final delimiter (counting from the - left) is returned. If count is negative, everything to the right of the final delimiter - (counting from the right) is returned. Substring_index performs a case-sensitive match - when searching for delim.""", - extended = "> SELECT _FUNC_('www.apache.org', '.', 2);\n 'www.apache'") + usage = """ + _FUNC_(str, delim, count) - Returns the substring from str before count occurrences of the delimiter delim. + If count is positive, everything to the left of the final delimiter (counting from the + left) is returned. If count is negative, everything to the right of the final delimiter + (counting from the right) is returned. Substring_index performs a case-sensitive match + when searching for delim. + """, + extended = """ + _FUNC_(str, delim, count) + + Arguments: + str - string type or any type expression that can be implicitly converted to string type. + delim - string type or any type expression that can be implicitly converted to string type. + count - any numeric type or any nonnumeric expression that can be implicitly converted to numeric type. + + Examples: + > SELECT _FUNC_('www.apache.org', '.', 2); + www.apache + """) // scalastyle:on line.size.limit case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -554,9 +688,21 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = """_FUNC_(substr, str[, pos]) - Returns the position of the first occurrence of substr in str after position pos. - The given pos and return value are 1-based.""", - extended = "> SELECT _FUNC_('bar', 'foobarbar', 5);\n 7") + usage = """ + _FUNC_(substr, str[, pos]) - Returns the position of the first occurrence of substr in str after position pos. + The given pos and return value are 1-based.""", + extended = """ + _FUNC_(substr, str[, pos]) + + Arguments: + substr - string type or any type expression that can be implicitly converted to string type. + str - string type or any type expression that can be implicitly converted to string type. + pos - any numeric type or any nonnumeric expression that can be implicitly converted to numeric type. + + Examples: + > SELECT _FUNC_('bar', 'foobarbar', 5); + 7 + """) // scalastyle:on line.size.limit case class StringLocate(substr: Expression, str: Expression, start: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -631,10 +777,27 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression) * Returns str, left-padded with pad to a length of len. */ @ExpressionDescription( - usage = """_FUNC_(str, len, pad) - Returns str, left-padded with pad to a length of len. - If str is longer than len, the return value is shortened to len characters.""", - extended = "> SELECT _FUNC_('hi', 5, '??');\n '???hi'\n" + - "> SELECT _FUNC_('hi', 1, '??');\n 'h'") + usage = """ + _FUNC_(str, len, pad) - Returns str, left-padded with pad to a length of len. + If str is longer than len, the return value is shortened to len characters. + """, + extended = """ + _FUNC_(str, len, pad) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + len - any numeric type or any nonnumeric expression that can be implicitly converted + to numeric type. + pad - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('hi', 5, '??'); + ???hi + > SELECT _FUNC_('hi', 1, '??'); + h + """) case class StringLPad(str: Expression, len: Expression, pad: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -657,10 +820,27 @@ case class StringLPad(str: Expression, len: Expression, pad: Expression) * Returns str, right-padded with pad to a length of len. */ @ExpressionDescription( - usage = """_FUNC_(str, len, pad) - Returns str, right-padded with pad to a length of len. - If str is longer than len, the return value is shortened to len characters.""", - extended = "> SELECT _FUNC_('hi', 5, '??');\n 'hi???'\n" + - "> SELECT _FUNC_('hi', 1, '??');\n 'h'") + usage = """ + _FUNC_(str, len, pad) - Returns str, right-padded with pad to a length of len. + If str is longer than len, the return value is shortened to len characters. + """, + extended = """ + _FUNC_(str, len, pad) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + len - any numeric type or any nonnumeric expression that can be implicitly converted + to numeric type. + pad - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('hi', 5, '??'); + hi??? + > SELECT _FUNC_('hi', 1, '??'); + h + """) case class StringRPad(str: Expression, len: Expression, pad: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -697,15 +877,26 @@ object ParseUrl { */ @ExpressionDescription( usage = "_FUNC_(url, partToExtract[, key]) - extracts a part from a URL", - extended = """Parts: HOST, PATH, QUERY, REF, PROTOCOL, AUTHORITY, FILE, USERINFO. - Key specifies which query to extract. - Examples: - > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST') - 'spark.apache.org' - > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY') - 'query=1' - > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY', 'query') - '1'""") + extended = """ + _FUNC_(url, partToExtract[, key]) + + Arguments: + url - string type or any type expression that can be implicitly converted + to string type. + partToExtract - string type or any type expression that can be implicitly converted + to string type. It can be one of HOST, PATH, QUERY, REF, PROTOCOL, AUTHORITY, FILE, + USERINFO. + key - string type or any type expression that can be implicitly converted + to string type. It specifies which query to extract. + + Examples: + > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST') + spark.apache.org + > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY') + query=1 + > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY', 'query') + 1 + """) case class ParseUrl(children: Seq[Expression]) extends Expression with ExpectsInputTypes with CodegenFallback { @@ -851,8 +1042,18 @@ case class ParseUrl(children: Seq[Expression]) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(String format, Obj... args) - Returns a formatted string from printf-style format strings.", - extended = "> SELECT _FUNC_(\"Hello World %d %s\", 100, \"days\");\n 'Hello World 100 days'") + usage = "_FUNC_(strfmt, obj, ...) - Returns a formatted string from printf-style format strings.", + extended = """ + _FUNC_(strfmt, obj, ...) + + Arguments: + strfmt - string type or any type expression that can be implicitly converted to string type. + obj - any type expression. + + Examples: + > SELECT _FUNC_("Hello World %d %s", 100, "days"); + Hello World 100 days + """) // scalastyle:on line.size.limit case class FormatString(children: Expression*) extends Expression with ImplicitCastInputTypes { @@ -923,10 +1124,21 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC * Words are delimited by whitespace. */ @ExpressionDescription( - usage = - """_FUNC_(str) - Returns str with the first letter of each word in uppercase. - All other letters are in lowercase. Words are delimited by white space.""", - extended = "> SELECT initcap('sPark sql');\n 'Spark Sql'") + usage = """ + _FUNC_(str) - Returns str with the first letter of each word in uppercase. + All other letters are in lowercase. Words are delimited by white space. + """, + extended = """ + _FUNC_(str) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT initcap('sPark sql'); + Spark Sql + """) case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[DataType] = Seq(StringType) @@ -945,7 +1157,19 @@ case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastI */ @ExpressionDescription( usage = "_FUNC_(str, n) - Returns the string which repeat the given string value n times.", - extended = "> SELECT _FUNC_('123', 2);\n '123123'") + extended = """ + _FUNC_(str, n) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + n - any numeric type or any nonnumeric expression that can be implicitly converted + to numeric type. + + Examples: + > SELECT _FUNC_('123', 2); + 123123 + """) case class StringRepeat(str: Expression, times: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -970,7 +1194,17 @@ case class StringRepeat(str: Expression, times: Expression) */ @ExpressionDescription( usage = "_FUNC_(str) - Returns the reversed given string.", - extended = "> SELECT _FUNC_('Spark SQL');\n 'LQS krapS'") + extended = """ + _FUNC_(str) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('Spark SQL'); + LQS krapS + """) case class StringReverse(child: Expression) extends UnaryExpression with String2StringExpression { override def convert(v: UTF8String): UTF8String = v.reverse() @@ -986,7 +1220,17 @@ case class StringReverse(child: Expression) extends UnaryExpression with String2 */ @ExpressionDescription( usage = "_FUNC_(n) - Returns a n spaces string.", - extended = "> SELECT _FUNC_(2);\n ' '") + extended = """ + _FUNC_(n) + + Arguments: + n - any numeric type or any nonnumeric expression that can be implicitly converted + to numeric type. + + Examples: + > SELECT concat(_FUNC_(2), '1'); + 1 + """) case class StringSpace(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -1015,7 +1259,22 @@ case class StringSpace(child: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(str, pos[, len]) - Returns the substring of str that starts at pos and is of length len or the slice of byte array that starts at pos and is of length len.", - extended = "> SELECT _FUNC_('Spark SQL', 5);\n 'k SQL'\n> SELECT _FUNC_('Spark SQL', -3);\n 'SQL'\n> SELECT _FUNC_('Spark SQL', 5, 1);\n 'k'") + extended = """ + _FUNC_(str, pos[, len]) + + Arguments: + str - string type or any type expression that can be implicitly converted to string type. + pos - any numeric type or any nonnumeric expression that can be implicitly converted to numeric type. + len - any numeric type or any nonnumeric expression that can be implicitly converted to numeric type. + + Examples: + > SELECT _FUNC_('Spark SQL', 5); + k SQL + > SELECT _FUNC_('Spark SQL', -3); + SQL + > SELECT _FUNC_('Spark SQL', 5, 1); + k + """) // scalastyle:on line.size.limit case class Substring(str: Expression, pos: Expression, len: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -1056,7 +1315,19 @@ case class Substring(str: Expression, pos: Expression, len: Expression) */ @ExpressionDescription( usage = "_FUNC_(str | binary) - Returns the length of str or number of bytes in binary data.", - extended = "> SELECT _FUNC_('Spark SQL');\n 9") + extended = """ + _FUNC_(str | binary) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + binary - binary type or any type expression that can be implicitly converted + to binary type. + + Examples: + > SELECT _FUNC_('Spark SQL'); + 9 + """) case class Length(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def dataType: DataType = IntegerType override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(StringType, BinaryType)) @@ -1079,7 +1350,19 @@ case class Length(child: Expression) extends UnaryExpression with ImplicitCastIn */ @ExpressionDescription( usage = "_FUNC_(str1, str2) - Returns the Levenshtein distance between the two given strings.", - extended = "> SELECT _FUNC_('kitten', 'sitting');\n 3") + extended = """ + _FUNC_(str1, str2) + + Arguments: + str1 - string type or any type expression that can be implicitly converted + to string type. + str2 - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('kitten', 'sitting'); + 3 + """) case class Levenshtein(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1100,7 +1383,17 @@ case class Levenshtein(left: Expression, right: Expression) extends BinaryExpres */ @ExpressionDescription( usage = "_FUNC_(str) - Returns soundex code of the string.", - extended = "> SELECT _FUNC_('Miller');\n 'M460'") + extended = """ + _FUNC_(str) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('Miller'); + M460 + """) case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputTypes { override def dataType: DataType = StringType @@ -1119,8 +1412,19 @@ case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputT */ @ExpressionDescription( usage = "_FUNC_(str) - Returns the numeric value of the first character of str.", - extended = "> SELECT _FUNC_('222');\n 50\n" + - "> SELECT _FUNC_(2);\n 50") + extended = """ + _FUNC_(str) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('222'); + 50 + > SELECT _FUNC_(2); + 50 + """) case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def dataType: DataType = IntegerType @@ -1153,7 +1457,18 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp * Converts the argument from binary to a base 64 string. */ @ExpressionDescription( - usage = "_FUNC_(bin) - Convert the argument from binary to a base 64 string.") + usage = "_FUNC_(bin) - Converts the argument from binary to a base 64 string.", + extended = """ + _FUNC_(bin) + + Arguments: + bin - binary type or any type expression that can be implicitly converted + to binary type. + + Examples: + > SELECT _FUNC_('Spark SQL'); + U3BhcmsgU1FM + """) case class Base64(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def dataType: DataType = StringType @@ -1177,7 +1492,18 @@ case class Base64(child: Expression) extends UnaryExpression with ImplicitCastIn * Converts the argument from a base 64 string to BINARY. */ @ExpressionDescription( - usage = "_FUNC_(str) - Convert the argument from a base 64 string to binary.") + usage = "_FUNC_(str) - Convert the argument from a base 64 string to binary.", + extended = """ + _FUNC_(str) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('U3BhcmsgU1FM'); + Spark SQL + """) case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def dataType: DataType = BinaryType @@ -1200,7 +1526,17 @@ case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCast * If either argument is null, the result will also be null. */ @ExpressionDescription( - usage = "_FUNC_(bin, str) - Decode the first argument using the second argument character set.") + usage = "_FUNC_(bin, str) - Decodes the first argument using the second argument character set.", + extended = """ + _FUNC_(bin, str) + + Arguments: + bin - binary type or any type expression that can be implicitly converted + to binary type. + str - string type or any type expression that can be implicitly converted + to string type. It represents the character set which can be one of 'US-ASCII', + 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'. + """) case class Decode(bin: Expression, charset: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1232,7 +1568,17 @@ case class Decode(bin: Expression, charset: Expression) * If either argument is null, the result will also be null. */ @ExpressionDescription( - usage = "_FUNC_(str, str) - Encode the first argument using the second argument character set.") + usage = "_FUNC_(str, str) - Encodes the first argument using the second argument character set.", + extended = """ + _FUNC_(str, str) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + str - string type or any type expression that can be implicitly converted + to string type. It represents the character set which can be one of 'US-ASCII', + 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'. + """) case class Encode(value: Expression, charset: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1263,10 +1609,22 @@ case class Encode(value: Expression, charset: Expression) * fractional part. */ @ExpressionDescription( - usage = """_FUNC_(X, D) - Formats the number X like '#,###,###.##', rounded to D decimal places. - If D is 0, the result has no decimal point or fractional part. - This is supposed to function like MySQL's FORMAT.""", - extended = "> SELECT _FUNC_(12332.123456, 4);\n '12,332.1235'") + usage = """ + _FUNC_(expr1, expr2) - Formats the number expr1 like '#,###,###.##', rounded to expr2 + decimal places. If expr2 is 0, the result has no decimal point or fractional part. + This is supposed to function like MySQL's FORMAT. + """, + extended = """ + _FUNC_(expr1, expr2) + + Arguments: + expr1 - numeric type expression. + expr2 - integer type expression. + + Examples: + > SELECT _FUNC_(12332.123456, 4); + 12,332.1235 + """) case class FormatNumber(x: Expression, d: Expression) extends BinaryExpression with ExpectsInputTypes { @@ -1389,7 +1747,21 @@ case class FormatNumber(x: Expression, d: Expression) */ @ExpressionDescription( usage = "_FUNC_(str[, lang, country]) - Splits str into an array of array of words.", - extended = "> SELECT _FUNC_('Hi there! Good morning.');\n [['Hi','there'], ['Good','morning']]") + extended = """ + _FUNC_(str[, lang, country]) + + Arguments: + str - string type or any type expression that can be implicitly converted + to string type. + lang - string type or any type expression that can be implicitly converted + to string type. + country - string type or any type expression that can be implicitly converted + to string type. + + Examples: + > SELECT _FUNC_('Hi there! Good morning.'); + [["Hi","there"],["Good","morning"]] + """) case class Sentences( str: Expression, language: Expression = Literal(""), From a29472eeb0fbbdccdd8affd82d7e5706623114c0 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 01:51:30 +0900 Subject: [PATCH 24/47] windowExpressions OK --- .../expressions/windowExpressions.scala | 110 ++++++++++++------ 1 file changed, 75 insertions(+), 35 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala index b47486f7af7f..e350c2f9773d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala @@ -381,13 +381,24 @@ abstract class OffsetWindowFunction * @param offset rows to jump ahead in the partition. * @param default to use when the offset is larger than the window. The default value is null. */ -@ExpressionDescription(usage = - """_FUNC_(input, offset, default) - LEAD returns the value of 'x' at the 'offset'th row - after the current row in the window. - The default value of 'offset' is 1 and the default value of 'default' is null. - If the value of 'x' at the 'offset'th row is null, null is returned. - If there is no such offset row (e.g. when the offset is 1, the last row of the window - does not have any subsequent row), 'default' is returned.""") +@ExpressionDescription( + usage = """ + _FUNC_(input[, offset[, default]]) - LEAD returns the value of 'x' at the 'offset'th row + after the current row in the window. + The default value of 'offset' is 1 and the default value of 'default' is null. + If the value of 'x' at the 'offset'th row is null, null is returned. + If there is no such offset row (e.g. when the offset is 1, the last row of the window + does not have any subsequent row), 'default' is returned. + """, + extended = """ + _FUNC_(input[, offset[, default]]) + + Arguments: + input - any type expression. + offset - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. Default is 1. + default - any type expression. Defualt is NULL. + """) case class Lead(input: Expression, offset: Expression, default: Expression) extends OffsetWindowFunction { @@ -410,13 +421,24 @@ case class Lead(input: Expression, offset: Expression, default: Expression) * @param offset rows to jump back in the partition. * @param default to use when the offset row does not exist. */ -@ExpressionDescription(usage = - """_FUNC_(input, offset, default) - LAG returns the value of 'x' at the 'offset'th row - before the current row in the window. - The default value of 'offset' is 1 and the default value of 'default' is null. - If the value of 'x' at the 'offset'th row is null, null is returned. - If there is no such offset row (e.g. when the offset is 1, the first row of the window - does not have any previous row), 'default' is returned.""") +@ExpressionDescription( + usage = """ + _FUNC_(input[, offset[, default]]) - LAG returns the value of 'x' at the 'offset'th row + before the current row in the window. + The default value of 'offset' is 1 and the default value of 'default' is null. + If the value of 'x' at the 'offset'th row is null, null is returned. + If there is no such offset row (e.g. when the offset is 1, the first row of the window + does not have any previous row), 'default' is returned. + """, + extended = """ + _FUNC_(input[, offset[, default]]) + + Arguments: + input - any type expression. + offset - any numeric type or any nonnumeric type expression that can be implicitly + converted to numeric type. Default is 1. + default - any type expression. Defualt is NULL. + """) case class Lag(input: Expression, offset: Expression, default: Expression) extends OffsetWindowFunction { @@ -471,10 +493,12 @@ object SizeBasedWindowFunction { * * This documentation has been based upon similar documentation for the Hive and Presto projects. */ -@ExpressionDescription(usage = - """_FUNC_() - The ROW_NUMBER() function assigns a unique, sequential number to - each row, starting with one, according to the ordering of rows within - the window partition.""") +@ExpressionDescription( + usage = """ + _FUNC_() - The ROW_NUMBER() function assigns a unique, sequential number to + each row, starting with one, according to the ordering of rows within + the window partition. + """) case class RowNumber() extends RowNumberLike { override val evaluateExpression = rowNumber override def prettyName: String = "row_number" @@ -488,9 +512,11 @@ case class RowNumber() extends RowNumberLike { * * This documentation has been based upon similar documentation for the Hive and Presto projects. */ -@ExpressionDescription(usage = - """_FUNC_() - The CUME_DIST() function computes the position of a value relative to - a all values in the partition.""") +@ExpressionDescription( + usage = """ + _FUNC_() - The CUME_DIST() function computes the position of a value relative to + a all values in the partition. + """) case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction { override def dataType: DataType = DoubleType // The frame for CUME_DIST is Range based instead of Row based, because CUME_DIST must @@ -521,9 +547,17 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction { * * @param buckets number of buckets to divide the rows in. Default value is 1. */ -@ExpressionDescription(usage = - """_FUNC_(x) - The NTILE(n) function divides the rows for each window partition - into 'n' buckets ranging from 1 to at most 'n'.""") +@ExpressionDescription( + usage = """ + _FUNC_(expr) - The NTILE(n) function divides the rows for each window partition + into 'n' buckets ranging from 1 to at most 'n'. + """, + extended = """ + _FUNC_(expr) + + Arguments: + expr - integer type expression that allows constant folding. + """) case class NTile(buckets: Expression) extends RowNumberLike with SizeBasedWindowFunction { def this() = this(Literal(1)) @@ -644,10 +678,12 @@ abstract class RankLike extends AggregateWindowFunction { * change in rank. This is an internal parameter and will be assigned by the * Analyser. */ -@ExpressionDescription(usage = - """_FUNC_() - RANK() computes the rank of a value in a group of values. The result - is one plus the number of rows preceding or equal to the current row in the - ordering of the partition. Tie values will produce gaps in the sequence.""") +@ExpressionDescription( + usage = """ + _FUNC_() - RANK() computes the rank of a value in a group of values. The result + is one plus the number of rows preceding or equal to the current row in the + ordering of the partition. Tie values will produce gaps in the sequence. + """) case class Rank(children: Seq[Expression]) extends RankLike { def this() = this(Nil) override def withOrder(order: Seq[Expression]): Rank = Rank(order) @@ -664,10 +700,12 @@ case class Rank(children: Seq[Expression]) extends RankLike { * change in rank. This is an internal parameter and will be assigned by the * Analyser. */ -@ExpressionDescription(usage = - """_FUNC_() - The DENSE_RANK() function computes the rank of a value in a group of - values. The result is one plus the previously assigned rank value. Unlike Rank, - DenseRank will not produce gaps in the ranking sequence.""") +@ExpressionDescription( + usage = """ + _FUNC_() - The DENSE_RANK() function computes the rank of a value in a group of + values. The result is one plus the previously assigned rank value. Unlike Rank, + DenseRank will not produce gaps in the ranking sequence. + """) case class DenseRank(children: Seq[Expression]) extends RankLike { def this() = this(Nil) override def withOrder(order: Seq[Expression]): DenseRank = DenseRank(order) @@ -692,9 +730,11 @@ case class DenseRank(children: Seq[Expression]) extends RankLike { * change in rank. This is an internal parameter and will be assigned by the * Analyser. */ -@ExpressionDescription(usage = - """_FUNC_() - PERCENT_RANK() The PercentRank function computes the percentage - ranking of a value in a group of values.""") +@ExpressionDescription( + usage = """ + _FUNC_() - PERCENT_RANK() The PercentRank function computes the percentage + ranking of a value in a group of values. + """) case class PercentRank(children: Seq[Expression]) extends RankLike with SizeBasedWindowFunction { def this() = this(Nil) override def withOrder(order: Seq[Expression]): PercentRank = PercentRank(order) From 73ccda0d89e71249adc25d4b04f70501849f1fd9 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 02:17:04 +0900 Subject: [PATCH 25/47] conditionalExpressions OK, double-check others and fix tests --- .../expressions/conditionalExpressions.scala | 2 +- .../sql/execution/command/functions.scala | 20 ++++++++------ .../sql/hive/execution/SQLQuerySuite.scala | 26 ++++++++++++------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala index 94ceb71015d6..22d8bccd58bb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala @@ -174,7 +174,7 @@ abstract class CaseWhenBase( */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "CASE WHEN a THEN b [WHEN c THEN d]* [ELSE e] END - When a = true, returns b; when c = true, return d; else return e.") + usage = "CASE WHEN expr1 THEN expr2 [WHEN expr3 THEN expr4]* [ELSE expr5] END - When expr1 = true, returns expr2; when expr3 = true, return expr4; else return expr5.") // scalastyle:on line.size.limit case class CaseWhen( val branches: Seq[(Expression, Expression)], diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala index e3976a325d96..57482f4dc8ce 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala @@ -103,18 +103,22 @@ case class DescribeFunctionCommand( functionName.funcName.toLowerCase match { case "<>" => Row(s"Function: $functionName") :: - Row(s"Usage: a <> b - Returns TRUE if a is not equal to b") :: Nil + Row("Usage: expr1 <> expr2 - " + + "Returns TRUE if expr1 is not equal to expr2") :: Nil case "!=" => Row(s"Function: $functionName") :: - Row(s"Usage: a != b - Returns TRUE if a is not equal to b") :: Nil + Row("Usage: expr1 != expr2 - " + + "Returns TRUE if expr1 is not equal to expr2") :: Nil case "between" => - Row(s"Function: between") :: - Row(s"Usage: a [NOT] BETWEEN b AND c - " + - s"evaluate if a is [not] in between b and c") :: Nil + Row("Function: between") :: + Row("Usage: expr1 [NOT] BETWEEN expr2 AND expr3 - " + + "evaluate if expr1 is [not] in between expr2 and expr3") :: Nil case "case" => - Row(s"Function: case") :: - Row(s"Usage: CASE a WHEN b THEN c [WHEN d THEN e]* [ELSE f] END - " + - s"When a = b, returns c; when a = d, return e; else return f") :: Nil + Row("Function: case") :: + Row("Usage: CASE expr1 WHEN expr2 THEN expr3 " + + "[WHEN expr4 THEN expr5]* [ELSE expr6] END - " + + "When expr1 = expr2, returns expr3; " + + "when expr1 = expr4, return expr5; else return expr6") :: Nil case _ => try { val info = sparkSession.sessionState.catalog.lookupFunctionInfo(functionName) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index 6af80c5e2269..adac0f564a00 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -248,14 +248,19 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { } } - ignore("describe functions - built-in functions") { + test("describe functions - built-in functions") { checkKeywordsExist(sql("describe function extended upper"), "Function: upper", "Class: org.apache.spark.sql.catalyst.expressions.Upper", "Usage: upper(str) - Returns str with all characters changed to uppercase", "Extended Usage:", - "> SELECT upper('SparkSql')", - "'SPARKSQL'") + "upper(str)", + "Arguments:", + "str - string type or any type expression that can be implicitly converted", + "to string type.", + "Examples:", + "> SELECT upper('SparkSql');", + "SPARKSQL") checkKeywordsExist(sql("describe functioN Upper"), "Function: upper", @@ -271,25 +276,28 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { checkKeywordsExist(sql("describe functioN `~`"), "Function: ~", "Class: org.apache.spark.sql.catalyst.expressions.BitwiseNot", - "Usage: ~ b - Bitwise NOT.") + "Usage: ~ expr - Bitwise NOT.") // Hard coded describe functions checkKeywordsExist(sql("describe function `<>`"), "Function: <>", - "Usage: a <> b - Returns TRUE if a is not equal to b") + "Usage: expr1 <> expr2 - Returns TRUE if expr1 is not equal to expr2") checkKeywordsExist(sql("describe function `!=`"), "Function: !=", - "Usage: a != b - Returns TRUE if a is not equal to b") + "Usage: expr1 != expr2 - Returns TRUE if expr1 is not equal to expr2") checkKeywordsExist(sql("describe function `between`"), "Function: between", - "Usage: a [NOT] BETWEEN b AND c - evaluate if a is [not] in between b and c") + "Usage: expr1 [NOT] BETWEEN expr2 AND expr3 - " + + "evaluate if expr1 is [not] in between expr2 and expr3") checkKeywordsExist(sql("describe function `case`"), "Function: case", - "Usage: CASE a WHEN b THEN c [WHEN d THEN e]* [ELSE f] END - " + - "When a = b, returns c; when a = d, return e; else return f") + "Usage: CASE expr1 WHEN expr2 THEN expr3 " + + "[WHEN expr4 THEN expr5]* [ELSE expr6] END - " + + "When expr1 = expr2, returns expr3; " + + "when expr1 = expr4, return expr5; else return expr6") } test("describe functions - user defined functions") { From d927bff266b978c26db98fd64ff346ca248f7ec4 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 02:32:31 +0900 Subject: [PATCH 26/47] double-check --- .../expressions/collectionOperations.scala | 24 +++++++++---------- .../expressions/datetimeExpressions.scala | 14 +++++------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index 12a85b6133fe..8f2c5235cee7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -68,12 +68,12 @@ case class Size(child: Expression) extends UnaryExpression with ExpectsInputType * Returns an unordered array containing the keys of the map. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns an unordered array containing the keys of the map.", + usage = "_FUNC_(map) - Returns an unordered array containing the keys of the map.", extended = """ - _FUNC_(expr) + _FUNC_(map) Arguments: - expr - map type expression. + map - map type expression. Examples: > SELECT _FUNC_(map(1, 'a', 2, 'b')); @@ -103,10 +103,10 @@ case class MapKeys(child: Expression) @ExpressionDescription( usage = "_FUNC_(map) - Returns an unordered array containing the values of the map.", extended = """ - _FUNC_(expr) + _FUNC_(map) Arguments: - expr - map type expression. + map - map type expression. Examples: > SELECT _FUNC_(map(1, 'a', 2, 'b')); @@ -136,12 +136,12 @@ case class MapValues(child: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr, ascendingOrder) - Sorts the input array in ascending order according to the natural ordering of the array elements.", + usage = "_FUNC_(array, ascendingOrder) - Sorts the input array in ascending order according to the natural ordering of the array elements.", extended = """ - _FUNC_(expr, ascendingOrder) + _FUNC_(array, ascendingOrder) Arguments: - expr - array type expression. + array - array type expression. ascendingOrder - boolean type literal. Examples: @@ -231,13 +231,13 @@ case class SortArray(base: Expression, ascendingOrder: Expression) * Checks if the array (left) has the element (right) */ @ExpressionDescription( - usage = "_FUNC_(expr1, value) - Returns TRUE if the array contains the value.", + usage = "_FUNC_(array, value) - Returns TRUE if the array contains the value.", extended = """ - _FUNC_(expr1, expr2) + _FUNC_(array, value) Arguments: - expr1 - array type expression. - expr2 - expression in the element type of expr1 or any type that can be implicitly + array - array type expression. + value - expression in the element type of expr1 or any type that can be implicitly converted to the element type of expr1. Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index d6d136d5a42e..365cced8a493 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -884,12 +884,12 @@ case class TimeAdd(start: Expression, interval: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp, timezone) - Assumes given timestamp is UTC and converts to given timezone.", + usage = "_FUNC_(expr, timezone) - Assumes given timestamp is UTC and converts to given timezone.", extended = """ - _FUNC_(timestamp, timezone) + _FUNC_(expr, timezone) Arguments: - timestamp - timestamp type or any type expression that can be implicitly converted to timestamp type. + expr - timestamp type or any type expression that can be implicitly converted to timestamp type. timezone - string type expression that represents time zone (e.g., 'Asia/Seoul'). Examples: @@ -1058,9 +1058,9 @@ case class MonthsBetween(date1: Expression, date2: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp, timezone) - Assumes given timestamp is in given timezone and converts to UTC.", + usage = "_FUNC_(expr, timezone) - Assumes given timestamp is in given timezone and converts to UTC.", extended = """ - _FUNC_(timestamp, timezone) + _FUNC_(expr, timezone) Arguments: timestamp - timestamp type or any type expression that can be implicitly converted to timestamp type. @@ -1153,9 +1153,9 @@ case class ToDate(child: Expression) extends UnaryExpression with ImplicitCastIn */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(date, fmt) - Returns returns date with the time portion of the day truncated to the unit specified by the format model fmt.", + usage = "_FUNC_(expr, fmt) - Returns returns date with the time portion of the day truncated to the unit specified by the format model fmt.", extended = """ - _FUNC_(date, fmt) + _FUNC_(expr, fmt) Arguments: expr - date type or any type expression that can be implicitly converted to date type. From 91d2ab5174273623819a6b18f0d2d557c54603f7 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 10:11:15 +0900 Subject: [PATCH 27/47] Fix tests in SQLQuerySuite and DDLSuite first --- .../org/apache/spark/sql/SQLQuerySuite.scala | 7 ++++- .../sql/execution/command/DDLSuite.scala | 28 +++++++++++++------ 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 60978efddd7f..a6c1417e2707 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -90,8 +90,13 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { "Class: org.apache.spark.sql.catalyst.expressions.Upper", "Usage: upper(str) - Returns str with all characters changed to uppercase", "Extended Usage:", + "upper(str)", + "Arguments:", + "str - string type or any type expression that can be implicitly converted", + "to string type.", + "Examples:", "> SELECT upper('SparkSql');", - "'SPARKSQL'") + "SPARKSQL") checkKeywordsExist(sql("describe functioN Upper"), "Function: upper", diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index c8b8e9ebabc7..3208802008e9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1455,34 +1455,34 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { sql("DESCRIBE FUNCTION log"), Row("Class: org.apache.spark.sql.catalyst.expressions.Logarithm") :: Row("Function: log") :: - Row("Usage: log(b, x) - Returns the logarithm of x with base b.") :: Nil + Row("Usage: log(expr1, expr2) - Returns the logarithm of expr1 with base expr2.") :: Nil ) // predicate operator checkAnswer( sql("DESCRIBE FUNCTION or"), Row("Class: org.apache.spark.sql.catalyst.expressions.Or") :: Row("Function: or") :: - Row("Usage: a or b - Logical OR.") :: Nil + Row("Usage: expr1 or expr2 - Logical OR.") :: Nil ) checkAnswer( sql("DESCRIBE FUNCTION !"), Row("Class: org.apache.spark.sql.catalyst.expressions.Not") :: Row("Function: !") :: - Row("Usage: ! a - Logical not") :: Nil + Row("Usage: ! expr - Logical not.") :: Nil ) // arithmetic operators checkAnswer( sql("DESCRIBE FUNCTION +"), Row("Class: org.apache.spark.sql.catalyst.expressions.Add") :: Row("Function: +") :: - Row("Usage: a + b - Returns a+b.") :: Nil + Row("Usage: expr1 + expr2 - Returns expr1+expr2.") :: Nil ) // comparison operators checkAnswer( sql("DESCRIBE FUNCTION <"), Row("Class: org.apache.spark.sql.catalyst.expressions.LessThan") :: Row("Function: <") :: - Row("Usage: a < b - Returns TRUE if a is less than b.") :: Nil + Row("Usage: expr1 < expr2 - Returns TRUE if expr1 is less than expr2.") :: Nil ) // STRING checkAnswer( @@ -1490,15 +1490,27 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { Row("Class: org.apache.spark.sql.catalyst.expressions.Concat") :: Row("Function: concat") :: Row("Usage: concat(str1, str2, ..., strN) " + - "- Returns the concatenation of str1, str2, ..., strN") :: Nil + "- Returns the concatenation of str1, str2, ..., strN.") :: Nil ) // extended mode checkAnswer( sql("DESCRIBE FUNCTION EXTENDED ^"), Row("Class: org.apache.spark.sql.catalyst.expressions.BitwiseXor") :: - Row("Extended Usage:\n> SELECT 3 ^ 5; 2") :: + Row( + "Extended Usage: " + + """ + | expr1 ^ expr2 + | + | Arguments: + | expr1 - any intergal numeric type expression. + | expr2 - any intergal numeric type expression. + | + | Examples: + | > SELECT 3 ^ 5; + | 2 + | """.stripMargin) :: Row("Function: ^") :: - Row("Usage: a ^ b - Bitwise exclusive OR.") :: Nil + Row("Usage: expr1 ^ expr2 - Bitwise exclusive OR.") :: Nil ) } From 7841860bf50fba8b31da774574ad9818ee678d85 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 10:13:35 +0900 Subject: [PATCH 28/47] Take out a space after `Extended Usage:`. --- .../org/apache/spark/sql/execution/command/functions.scala | 2 +- .../org/apache/spark/sql/execution/command/DDLSuite.scala | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala index 57482f4dc8ce..0c3c10b3e0ee 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala @@ -129,7 +129,7 @@ case class DescribeFunctionCommand( if (isExtended) { result :+ - Row(s"Extended Usage: ${replaceFunctionName(info.getExtended, info.getName)}") + Row(s"Extended Usage:${replaceFunctionName(info.getExtended, info.getName)}") } else { result } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 3208802008e9..775cf33699aa 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1497,8 +1497,7 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { sql("DESCRIBE FUNCTION EXTENDED ^"), Row("Class: org.apache.spark.sql.catalyst.expressions.BitwiseXor") :: Row( - "Extended Usage: " + - """ + """Extended Usage: | expr1 ^ expr2 | | Arguments: From 01eecfe44c5edc3db0d25f43a7ae8f80ca07ac61 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 10:18:46 +0900 Subject: [PATCH 29/47] Consistent spacing in Examples --- .../spark/sql/catalyst/expressions/CallMethodViaReflection.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala index c79f2769d38a..b7d291f9036f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala @@ -56,7 +56,6 @@ import org.apache.spark.util.Utils Examples: > SELECT _FUNC_('java.util.UUID', 'randomUUID'); c33fb387-8500-4bfa-81d2-6e0e3e930df2 - > SELECT _FUNC_('java.util.UUID', 'fromString', 'a5cf6c42-0c85-418f-af6c-3e4e5b1328f2'); a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 """) From 1979d920afcac5794131b7605b8a170f837b461e Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 18:20:30 +0900 Subject: [PATCH 30/47] Remove repeated _FUNC_, consolidate usages and simplify the arguments --- .../expressions/CallMethodViaReflection.scala | 24 +- .../spark/sql/catalyst/expressions/Cast.scala | 16 +- .../catalyst/expressions/InputFileName.scala | 9 +- .../MonotonicallyIncreasingID.scala | 7 - .../expressions/SparkPartitionID.scala | 9 +- .../aggregate/ApproximatePercentile.scala | 46 +- .../expressions/aggregate/Average.scala | 7 +- .../aggregate/CentralMomentAgg.scala | 42 +- .../catalyst/expressions/aggregate/Corr.scala | 10 +- .../expressions/aggregate/Count.scala | 11 +- .../expressions/aggregate/Covariance.scala | 20 +- .../expressions/aggregate/First.scala | 19 +- .../aggregate/HyperLogLogPlusPlus.scala | 20 +- .../catalyst/expressions/aggregate/Last.scala | 19 +- .../catalyst/expressions/aggregate/Max.scala | 6 +- .../catalyst/expressions/aggregate/Min.scala | 6 +- .../catalyst/expressions/aggregate/Sum.scala | 7 +- .../expressions/aggregate/collect.scala | 12 +- .../sql/catalyst/expressions/arithmetic.scala | 120 ++-- .../expressions/bitwiseExpressions.scala | 54 +- .../expressions/collectionOperations.scala | 67 +-- .../expressions/complexTypeCreator.scala | 70 +-- .../expressions/conditionalExpressions.scala | 6 +- .../expressions/datetimeExpressions.scala | 351 +++++------ .../sql/catalyst/expressions/generators.scala | 60 +- .../expressions/jsonExpressions.scala | 28 +- .../expressions/mathExpressions.scala | 559 +++++++----------- .../spark/sql/catalyst/expressions/misc.scala | 97 ++- .../expressions/nullExpressions.scala | 125 ++-- .../sql/catalyst/expressions/predicates.scala | 78 +-- .../expressions/randomExpressions.scala | 36 +- .../expressions/regexpExpressions.scala | 78 +-- .../expressions/stringExpressions.scala | 532 +++++++---------- .../expressions/windowExpressions.scala | 32 +- 34 files changed, 1000 insertions(+), 1583 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala index b7d291f9036f..d6f289ac90c2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala @@ -44,20 +44,18 @@ import org.apache.spark.util.Utils * and the remaining are input arguments to the Java method. */ @ExpressionDescription( - usage = "_FUNC_(class, method[, arg1[, arg2 ..]]) calls method with reflection", + usage = "_FUNC_(class, method[, arg1[, arg2 ..]]) - Calls method with reflection.", extended = """ - _FUNC_(class, method[, arg1[, arg2 ..]]) - - Arguments: - class - string type literal that represents full-qualified class name. - method - string type literal that represents method name. - arg - string type literal that represents arguments for the method. - - Examples: - > SELECT _FUNC_('java.util.UUID', 'randomUUID'); - c33fb387-8500-4bfa-81d2-6e0e3e930df2 - > SELECT _FUNC_('java.util.UUID', 'fromString', 'a5cf6c42-0c85-418f-af6c-3e4e5b1328f2'); - a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 + Arguments: + class - a string literal that represents full-qualified class name. + method - a string literal that represents method name. + arg - a string literal that represents arguments for the method. + + Examples: + > SELECT _FUNC_('java.util.UUID', 'randomUUID'); + c33fb387-8500-4bfa-81d2-6e0e3e930df2 + > SELECT _FUNC_('java.util.UUID', 'fromString', 'a5cf6c42-0c85-418f-af6c-3e4e5b1328f2'); + a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 """) case class CallMethodViaReflection(children: Seq[Expression]) extends Expression with CodegenFallback { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 85ad1e08d4cd..7e849afe4c5e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -114,17 +114,15 @@ object Cast { /** Cast the child expression to the target data type. */ @ExpressionDescription( - usage = "_FUNC_(expr AS type) - Cast value expr to the target data type.", + usage = "_FUNC_(expr AS type) - Casts value expr to the target data type.", extended = """ - _FUNC_(expr AS type) + Arguments: + expr - an expression of any type. + type - data type. - Arguments: - expr - any type expression. - type - data type. - - Examples: - > SELECT _FUNC_('10' as int); - 10 + Examples: + > SELECT _FUNC_('10' as int); + 10 """) case class Cast(child: Expression, dataType: DataType) extends UnaryExpression with NullIntolerant { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala index bb3fc98d3a63..258cb985e0c0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/InputFileName.scala @@ -27,14 +27,7 @@ import org.apache.spark.unsafe.types.UTF8String * Expression that returns the name of the current file being read. */ @ExpressionDescription( - usage = "_FUNC_() - Returns the name of the current file being read if available.", - extended = """ - _FUNC_(expr) - - Examples: - > SELECT _FUNC_(); - - """) + usage = "_FUNC_() - Returns the name of the current file being read if available.") case class InputFileName() extends LeafExpression with Nondeterministic { override def nullable: Boolean = true diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala index 99bf2d28cc66..bb14ebdd06b0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala @@ -39,13 +39,6 @@ import org.apache.spark.sql.types.{DataType, LongType} The current implementation puts the partition ID in the upper 31 bits, and the lower 33 bits represent the record number within each partition. The assumption is that the data frame has less than 1 billion partitions, and each partition has less than 8 billion records. - """, - extended = """ - _FUNC_() - - Examples: - > SELECT _FUNC_(); - 0 """) case class MonotonicallyIncreasingID() extends LeafExpression with Nondeterministic { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala index c1909ddd4d51..6d3f672fa070 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala @@ -26,14 +26,7 @@ import org.apache.spark.sql.types.{DataType, IntegerType} * Expression that returns the current partition id of the Spark task. */ @ExpressionDescription( - usage = "_FUNC_() - Returns the current partition id of the Spark task.", - extended = """ - _FUNC_() - - Examples: - > SELECT _FUNC_(); - 0 - """) + usage = "_FUNC_() - Returns the current partition id of the Spark task.") case class SparkPartitionID() extends LeafExpression with Nondeterministic { override def nullable: Boolean = false diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala index df4a2629428c..926e457136f5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala @@ -55,42 +55,20 @@ import org.apache.spark.sql.types._ and 1.0. The `accuracy` parameter (default: 10000) is a positive integer literal which controls approximation accuracy at the cost of memory. Higher value of `accuracy` yields better accuracy, `1.0/accuracy` is the relative error of the approximation. - - _FUNC_(col, array(percentage1 [, percentage2]...) [, accuracy]) - Returns the approximate - percentile array of column `col` at the given percentage array. Each value of the - percentage array must be between 0.0 and 1.0. The `accuracy` parameter (default: 10000) is - a positive integer literal which controls approximation accuracy at the cost of memory. - Higher value of `accuracy` yields better accuracy, `1.0/accuracy` is the relative error of - the approximation. + When percentage is an array, each value of the percentage array must be between 0.0 and 1.0. """, extended = """ - _FUNC_(col, percentage [, accuracy]) - - Arguments: - col - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - array(...) - an array that contains any numeric type literal that can be implicitly - converted to numeric type. - accuracy - any numeric type or any nonnumeric type literal that can be implicitly - converted to numeric type. - - Examples: - > SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100); - [10.0,10.0,10.0] - - _FUNC_(col, array(percentage1 [, percentage2]...) [, accuracy]) - - Arguments: - col - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - percentage - any numeric type or any nonnumeric type literal that can be - implicitly converted to numeric type. - accuracy - any numeric type or any nonnumeric type literal that can be implicitly - converted to numeric type. - - Examples: - > SELECT percentile_approx(10.0, 0.5, 100); - 10.0 + Arguments: + col - a numeric expression. + percentage - a numeric literal or an array literal of numeric type that defines the + percentile. For example, 0.5 means 50-percentile. + accuracy - a numeric literal. + + Examples: + > SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100); + [10.0,10.0,10.0] + > SELECT percentile_approx(10.0, 0.5, 100); + 10.0 """) case class ApproximatePercentile( child: Expression, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala index 57ba96fb4dd7..9701ed0fc9b1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Average.scala @@ -26,11 +26,8 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the mean calculated from values of a group.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. """) case class Average(child: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala index 6008616a71f9..f1f3c93bc1f3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala @@ -134,11 +134,8 @@ abstract class CentralMomentAgg(child: Expression) extends DeclarativeAggregate @ExpressionDescription( usage = "_FUNC_(expr) - Returns the population standard deviation calculated from values of a group.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. """) // scalastyle:on line.size.limit case class StddevPop(child: Expression) extends CentralMomentAgg(child) { @@ -158,11 +155,8 @@ case class StddevPop(child: Expression) extends CentralMomentAgg(child) { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sample standard deviation calculated from values of a group.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. """) // scalastyle:on line.size.limit case class StddevSamp(child: Expression) extends CentralMomentAgg(child) { @@ -182,11 +176,8 @@ case class StddevSamp(child: Expression) extends CentralMomentAgg(child) { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the population variance calculated from values of a group.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. """) case class VariancePop(child: Expression) extends CentralMomentAgg(child) { @@ -204,11 +195,8 @@ case class VariancePop(child: Expression) extends CentralMomentAgg(child) { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sample variance calculated from values of a group.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. """) case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) { @@ -226,11 +214,8 @@ case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the skewness value calculated from values of a group.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. """) case class Skewness(expr: Expression) extends CentralMomentAgg(expr) { @@ -248,11 +233,8 @@ case class Skewness(expr: Expression) extends CentralMomentAgg(expr) { @ExpressionDescription( usage = "_FUNC_(expr) - Returns the kurtosis value calculated from values of a group.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. """) case class Kurtosis(child: Expression) extends CentralMomentAgg(child) { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala index 854856c109d7..549ff871402e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Corr.scala @@ -32,13 +32,9 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns Pearson coefficient of correlation between a set of number pairs.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. """) // scalastyle:on line.size.limit case class Corr(x: Expression, y: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala index b607eb065b5a..2672b269af03 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala @@ -31,15 +31,8 @@ import org.apache.spark.sql.types._ _FUNC_(DISTINCT expr[, expr...]) - Returns the number of rows for which the supplied expression(s) are unique and non-NULL. """, extended = """ - _FUNC_(expr) - - Arguments: - expr - any type expression that represents data to count. - - _FUNC_(DISTINCT expr[, expr...]) - - Arguments: - expr - any type expression that represents data to count. + Arguments: + expr - an expression of any type that represents data to count. """) // scalastyle:on line.size.limit case class Count(children: Seq[Expression]) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala index 5e3d8a2dbd87..b54a185c3c54 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala @@ -79,13 +79,9 @@ abstract class Covariance(x: Expression, y: Expression) extends DeclarativeAggre @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the population covariance of a set of number pairs.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. """) case class CovPopulation(left: Expression, right: Expression) extends Covariance(left, right) { override val evaluateExpression: Expression = { @@ -98,13 +94,9 @@ case class CovPopulation(left: Expression, right: Expression) extends Covariance @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the sample covariance of a set of number pairs.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. """) case class CovSample(left: Expression, right: Expression) extends Covariance(left, right) { override val evaluateExpression: Expression = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala index 83e7f39adb43..44176d2497e5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala @@ -30,23 +30,14 @@ import org.apache.spark.sql.types._ */ @ExpressionDescription( usage = """ - _FUNC_(expr) - Returns the first value of expr for a group of rows. - - _FUNC_(expr, isIgnoreNull) - Returns the first value of expr for a group of rows. + _FUNC_(expr[, isIgnoreNull]) - Returns the first value of expr for a group of rows. If isIgnoreNull is true, returns only non-null values. """, extended = """ - _FUNC_(expr) - - Arguments: - expr - any type expression that represents data to collect the first. - - _FUNC_(expr, isIgnoreNull) - - Arguments: - expr - any type expression that represents data to collect the first. - isIgnoreNull - boolean type literal. If isIgnoreNull is true, returns only non-null - values. Default is false. + Arguments: + expr - an expression of any type that represents data to collect the first. + isIgnoreNull - a boolean literal. If isIgnoreNull is true, returns only non-null + values. Default is false. """) case class First(child: Expression, ignoreNullsExpr: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala index 7b0a06457f8e..2670b8e6bec1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala @@ -48,23 +48,13 @@ import org.apache.spark.sql.types._ // scalastyle:on @ExpressionDescription( usage = """ - _FUNC_(expr) - Returns the estimated cardinality by HyperLogLog++. - - _FUNC_(expr, relativeSD) - Returns the estimated cardinality by HyperLogLog++ - with relativeSD, the maximum estimation error allowed. + _FUNC_(expr[, relativeSD]) - Returns the estimated cardinality by HyperLogLog++. + relativeSD defines the maximum estimation error allowed. """, extended = """ - _FUNC_(expr) - - Arguments: - expr - any type expression that represents data to count. - - _FUNC_(expr, relativeSD) - - Arguments: - expr - any type expression that represents data to count. - relativeSD - any numeric type that can be implicitly converted to double type. - This represents maximum estimation error allowed (default = 0.05). + Arguments: + expr - an expression of any type that represents data to count. + relativeSD - a numeric literal that defines the maximum estimation error allowed. """) case class HyperLogLogPlusPlus( child: Expression, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala index 948bc73d4e94..010a7a1cb45a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala @@ -30,23 +30,14 @@ import org.apache.spark.sql.types._ */ @ExpressionDescription( usage = """ - _FUNC_(expr) - Returns the last value of expr for a group of rows. - - _FUNC_(expr, isIgnoreNull) - Returns the last value of expr for a group of rows. + _FUNC_(expr[, isIgnoreNull]) - Returns the last value of expr for a group of rows. If isIgnoreNull is true, returns only non-null values. """, extended = """ - _FUNC_ (expr) - - Arguments: - expr - any type expression that represents data to collect the last. - - _FUNC_ (expr, isIgnoreNull) - - Arguments: - expr - any type expression that represents data to collect the last. - isIgnoreNull - boolean type literal. If isIgnoreNull is true, returns only non-null - values. Default is false. + Arguments: + expr - an expression of any type that represents data to collect the last. + isIgnoreNull - a boolean literal. If isIgnoreNull is true, returns only non-null + values. Default is false. """) case class Last(child: Expression, ignoreNullsExpr: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala index 576d43097b24..dcaab5239b61 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala @@ -25,10 +25,8 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the maximum value of expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any type expression. + Arguments: + expr - an expression of any type. """) case class Max(child: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala index b13826c95682..ddaaf9a91d2a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala @@ -25,10 +25,8 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the minimum value of expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any type expression. + Arguments: + expr - an expression of any type. """) case class Min(child: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala index 528837b75574..bc1eed264c0c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Sum.scala @@ -25,11 +25,8 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sum calculated from values of a group.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. """) case class Sum(child: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala index 001d05ab3408..aa80bac662f3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala @@ -88,10 +88,8 @@ abstract class Collect extends ImperativeAggregate { @ExpressionDescription( usage = "_FUNC_(expr) - Collects and returns a list of non-unique elements.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any type expression that represents data to collect. + Arguments: + expr - an expression of any type that represents data to collect as a list. """) case class CollectList( child: Expression, @@ -117,10 +115,8 @@ case class CollectList( @ExpressionDescription( usage = "_FUNC_(expr) - Collects and returns a set of unique elements.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any type expression that represents data to collect. + Arguments: + expr - an expression of any type that represents data to collect as a set. """) case class CollectSet( child: Expression, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 350cf2449005..70f750b07461 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -27,10 +27,8 @@ import org.apache.spark.unsafe.types.CalendarInterval @ExpressionDescription( usage = "_FUNC_(expr) - Returns the negative value of expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or interval type expression. + Arguments: + expr - a numeric or interval expression. """) case class UnaryMinus(child: Expression) extends UnaryExpression with ExpectsInputTypes with NullIntolerant { @@ -70,10 +68,8 @@ case class UnaryMinus(child: Expression) extends UnaryExpression @ExpressionDescription( usage = "_FUNC_(expr) - Returns the positive value of expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or interval type expression. + Arguments: + expr - a numeric or interval expression. """) case class UnaryPositive(child: Expression) extends UnaryExpression with ExpectsInputTypes with NullIntolerant { @@ -97,14 +93,12 @@ case class UnaryPositive(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the absolute value of the numeric value.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type expression. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(-1); - 1 + Examples: + > SELECT _FUNC_(-1); + 1 """) case class Abs(child: Expression) extends UnaryExpression with ExpectsInputTypes with NullIntolerant { @@ -154,13 +148,9 @@ object BinaryArithmetic { @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns expr1+expr2.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any numeric type, interval type or any nonnumeric type expression that - can be implicitly converted to numeric type. - expr2 - any numeric type, interval type or any nonnumeric type expression that - can be implicitly converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. """) case class Add(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -194,13 +184,9 @@ case class Add(left: Expression, right: Expression) extends BinaryArithmetic wit @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns expr1-expr2.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any numeric type, interval type or any nonnumeric type expression that - can be implicitly converted to numeric type. - expr2 - any numeric type, interval type or any nonnumeric type expression that - can be implicitly converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. """) case class Subtract(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -235,13 +221,9 @@ case class Subtract(left: Expression, right: Expression) @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Multiplies expr1 by expr2.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. """) case class Multiply(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -259,17 +241,13 @@ case class Multiply(left: Expression, right: Expression) @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Divides expr1 by expr2.", extended = """ - expr1 _FUNC_ expr2 + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT 3 _FUNC_ 2; - 1.5 + Examples: + > SELECT 3 _FUNC_ 2; + 1.5 """) case class Divide(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -348,13 +326,9 @@ case class Divide(left: Expression, right: Expression) @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns the remainder when dividing expr1 by expr2.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. """) case class Remainder(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -438,17 +412,13 @@ case class Remainder(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the positive modulo.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Examples: - > SELECT _FUNC_(10, 3); - 1 + Examples: + > SELECT _FUNC_(10, 3); + 1 """) case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -554,14 +524,12 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic wi @ExpressionDescription( usage = "_FUNC_(expr, ...) - Returns the least value of all parameters, skipping null values.", extended = """ - _FUNC_(expr, ...) + Arguments: + expr - an expression of any type except struct, map and array. - Arguments: - expr - any type expression except struct/map/array type. - - Examples: - > SELECT _FUNC_(10, 9, 2, 4, 3); - 2 + Examples: + > SELECT _FUNC_(10, 9, 2, 4, 3); + 2 """) case class Least(children: Seq[Expression]) extends Expression { @@ -624,14 +592,12 @@ case class Least(children: Seq[Expression]) extends Expression { @ExpressionDescription( usage = "_FUNC_(expr, ...) - Returns the greatest value of all parameters, skipping null values.", extended = """ - _FUNC_(expr, ...) - - Arguments: - expr - any type expression except struct/map/array type. + Arguments: + expr - an expression of any type except struct, map and array. - Examples: - > SELECT _FUNC_(10, 9, 2, 4, 3); - 10 + Examples: + > SELECT _FUNC_(10, 9, 2, 4, 3); + 10 """) case class Greatest(children: Seq[Expression]) extends Expression { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index dbd6f761ca79..757d6e18c8f8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -29,15 +29,13 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Bitwise AND.", extended = """ - expr1 _FUNC_ expr2 + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Arguments: - expr1 - any intergal numeric type expression. - expr2 - any intergal numeric type expression. - - Examples: - > SELECT 3 _FUNC_ 5; - 1 + Examples: + > SELECT 3 _FUNC_ 5; + 1 """) case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithmetic { @@ -67,15 +65,13 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Bitwise OR.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any intergal numeric type expression. - expr2 - any intergal numeric type expression. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Examples: - > SELECT 3 _FUNC_ 5; - 7 + Examples: + > SELECT 3 _FUNC_ 5; + 7 """) case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmetic { @@ -105,15 +101,13 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Bitwise exclusive OR.", extended = """ - expr1 _FUNC_ expr2 + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Arguments: - expr1 - any intergal numeric type expression. - expr2 - any intergal numeric type expression. - - Examples: - > SELECT 3 _FUNC_ 5; - 2 + Examples: + > SELECT 3 _FUNC_ 5; + 2 """) case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithmetic { @@ -141,14 +135,12 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme @ExpressionDescription( usage = "_FUNC_ expr - Bitwise NOT.", extended = """ - _FUNC_ expr - - Arguments: - expr - any intergal numeric type expression. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_ 0; - -1 + Examples: + > SELECT _FUNC_ 0; + -1 """) case class BitwiseNot(child: Expression) extends UnaryExpression with ExpectsInputTypes { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index 8f2c5235cee7..7b547067bc86 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -30,14 +30,12 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr) - Returns the size of an array or a map. Returns -1 if null.", extended = """ - _FUNC_(expr) + Arguments: + expr - an array or map expression. - Arguments: - expr - array type or map type expression. - - Examples: - > SELECT _FUNC_(array('b', 'd', 'c', 'a')); - 4 + Examples: + > SELECT _FUNC_(array('b', 'd', 'c', 'a')); + 4 """) case class Size(child: Expression) extends UnaryExpression with ExpectsInputTypes { override def dataType: DataType = IntegerType @@ -70,14 +68,12 @@ case class Size(child: Expression) extends UnaryExpression with ExpectsInputType @ExpressionDescription( usage = "_FUNC_(map) - Returns an unordered array containing the keys of the map.", extended = """ - _FUNC_(map) - - Arguments: - map - map type expression. + Arguments: + map - a map expression. - Examples: - > SELECT _FUNC_(map(1, 'a', 2, 'b')); - [1,2] + Examples: + > SELECT _FUNC_(map(1, 'a', 2, 'b')); + [1,2] """) case class MapKeys(child: Expression) extends UnaryExpression with ExpectsInputTypes { @@ -103,14 +99,12 @@ case class MapKeys(child: Expression) @ExpressionDescription( usage = "_FUNC_(map) - Returns an unordered array containing the values of the map.", extended = """ - _FUNC_(map) - - Arguments: - map - map type expression. + Arguments: + map - a map expression. - Examples: - > SELECT _FUNC_(map(1, 'a', 2, 'b')); - ["a","b"] + Examples: + > SELECT _FUNC_(map(1, 'a', 2, 'b')); + ["a","b"] """) case class MapValues(child: Expression) extends UnaryExpression with ExpectsInputTypes { @@ -136,17 +130,15 @@ case class MapValues(child: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(array, ascendingOrder) - Sorts the input array in ascending order according to the natural ordering of the array elements.", + usage = "_FUNC_(array[, ascendingOrder]) - Sorts the input array in ascending order according to the natural ordering of the array elements.", extended = """ - _FUNC_(array, ascendingOrder) + Arguments: + array - an array expression. + ascendingOrder - a boolean literal that represents ascending/descending order. - Arguments: - array - array type expression. - ascendingOrder - boolean type literal. - - Examples: - > SELECT _FUNC_(array('b', 'd', 'c', 'a'), true); - ["a","b","c","d"] + Examples: + > SELECT _FUNC_(array('b', 'd', 'c', 'a'), true); + ["a","b","c","d"] """) // scalastyle:on line.size.limit case class SortArray(base: Expression, ascendingOrder: Expression) @@ -233,16 +225,13 @@ case class SortArray(base: Expression, ascendingOrder: Expression) @ExpressionDescription( usage = "_FUNC_(array, value) - Returns TRUE if the array contains the value.", extended = """ - _FUNC_(array, value) - - Arguments: - array - array type expression. - value - expression in the element type of expr1 or any type that can be implicitly - converted to the element type of expr1. + Arguments: + array - an array expression. + value - an expression of the element type of array. - Examples: - > SELECT _FUNC_(array(1, 2, 3), 2); - true + Examples: + > SELECT _FUNC_(array(1, 2, 3), 2); + true """) case class ArrayContains(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index a70690ceca2c..296f999617a5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -30,14 +30,12 @@ import org.apache.spark.unsafe.types.UTF8String @ExpressionDescription( usage = "_FUNC_(expr, ...) - Returns an array with the given elements.", extended = """ - _FUNC_(expr, ...) + Arguments: + expr - an expression of any type. - Arguments: - expr - any type expression. - - Examples: - > SELECT _FUNC_(1, 2, 3); - [1,2,3] + Examples: + > SELECT _FUNC_(1, 2, 3); + [1,2,3] """) case class CreateArray(children: Seq[Expression]) extends Expression { @@ -94,15 +92,13 @@ case class CreateArray(children: Seq[Expression]) extends Expression { @ExpressionDescription( usage = "_FUNC_(key0, value0, key1, value1...) - Creates a map with the given key/value pairs.", extended = """ - _FUNC_(key0, value0, key1, value1...) - - Arguments: - key - any type expression. - value - any type expression. + Arguments: + key - an expression of any type. + value - an expression of any type. - Examples: - > SELECT _FUNC_(1.0, '2', 3.0, '4'); - {1.0:"2",3.0:"4"} + Examples: + > SELECT _FUNC_(1.0, '2', 3.0, '4'); + {1.0:"2",3.0:"4"} """) case class CreateMap(children: Seq[Expression]) extends Expression { lazy val keys = children.indices.filter(_ % 2 == 0).map(children) @@ -198,14 +194,12 @@ case class CreateMap(children: Seq[Expression]) extends Expression { @ExpressionDescription( usage = "_FUNC_(expr1, expr2, expr2 ...) - Creates a struct with the given field values.", extended = """ - _FUNC_(expr1, expr2, expr2 ...) - - Arguments: - expr - any type expression. + Arguments: + expr - an expression of any type. - Examples: - > SELECT _FUNC_(1, 2, 3); - {"col1":1,"col2":2,"col3":3} + Examples: + > SELECT _FUNC_(1, 2, 3); + {"col1":1,"col2":2,"col3":3} """) case class CreateStruct(children: Seq[Expression]) extends Expression { @@ -267,15 +261,13 @@ case class CreateStruct(children: Seq[Expression]) extends Expression { @ExpressionDescription( usage = "_FUNC_(name1, val1, name2, val2, ...) - Creates a struct with the given field names and values.", extended = """ - _FUNC_(name1, val1, name2, val2, ...) + Arguments: + name - a string expression literal that represents the field name. + val - an expression of any type. - Arguments: - name - string type expression that allows constant folding and represents the field name. - val - any type expression. - - Examples: - > SELECT _FUNC_("a", 1, "b", 2, "c", 3); - {"a":1,"b":2,"c":3} + Examples: + > SELECT _FUNC_("a", 1, "b", 2, "c", 3); + {"a":1,"b":2,"c":3} """) // scalastyle:on line.size.limit case class CreateNamedStruct(children: Seq[Expression]) extends Expression { @@ -443,16 +435,14 @@ case class CreateNamedStructUnsafe(children: Seq[Expression]) extends Expression @ExpressionDescription( usage = "_FUNC_(text[, pairDelim, keyValueDelim]) - Creates a map after splitting the text into key/value pairs using delimiters. Default delimiters are ',' for pairDelim and ':' for keyValueDelim.", extended = """ - _FUNC_(text[, pairDelim, keyValueDelim]) - - Arguments: - text - string type expression that represents data to convert into map. - pairDelim - string type expression that allows constant folding. - keyValueDelim - string type expression that allows constant folding. - - Examples: - > SELECT _FUNC_('a:1,b:2,c:3',',',':'); - map("a":"1","b":"2","c":"3") + Arguments: + text - a string expression that represents data to convert into map. + pairDelim - a string literal that defines the delimiter to separate each pair.For example, ",". + keyValueDelim - a string literal that defines the delimiter to separate key and value in the pair. For example, ":". + + Examples: + > SELECT _FUNC_('a:1,b:2,c:3', ',', ':'); + map("a":"1","b":"2","c":"3") """) // scalastyle:on line.size.limit case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: Expression) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala index 22d8bccd58bb..87afe6ee2a90 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala @@ -29,9 +29,9 @@ import org.apache.spark.sql.types._ _FUNC_(expr1, expr2, expr3) Arguments: - expr1 - boolean type expression. - expr2 - any type expression that represents the return value when expr1 is TRUE. - expr3 - any type expression that represents the return value when expr1 is FALSE. + expr1 - a boolean expression. + expr2 - an expression of any type that represents the return value when expr1 is TRUE. + expr3 - an expression of any type that represents the return value when expr1 is FALSE. Examples: > SELECT _FUNC_(1 < 2, 'a', 'b'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 365cced8a493..39c44a4fbfd9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -77,17 +77,13 @@ case class CurrentTimestamp() extends LeafExpression with CodegenFallback { @ExpressionDescription( usage = "_FUNC_(start_date, num_days) - Returns the date that is num_days after start_date.", extended = """ - _FUNC_(start_date, num_days) + Arguments: + start_date - a date expression. + num_days - a numeric expression that represents the number of days to add. - Arguments: - start_date - date type or any type expression that can be implicitly converted to - date type. - num_days - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_('2016-07-30', 1); - 2016-07-31 + Examples: + > SELECT _FUNC_('2016-07-30', 1); + 2016-07-31 """) case class DateAdd(startDate: Expression, days: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -118,17 +114,13 @@ case class DateAdd(startDate: Expression, days: Expression) @ExpressionDescription( usage = "_FUNC_(start_date, num_days) - Returns the date that is num_days before start_date.", extended = """ - _FUNC_(start_date, num_days) - - Arguments: - start_date - date type or any type expression that can be implicitly converted to - date type. - num_days - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + start_date - a date expression. + num_days - a numeric expression that represents the number of days to subtract. - Examples: - > SELECT _FUNC_('2016-07-30', 1); - 2016-07-29 + Examples: + > SELECT _FUNC_('2016-07-30', 1); + 2016-07-29 """) case class DateSub(startDate: Expression, days: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -155,15 +147,12 @@ case class DateSub(startDate: Expression, days: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the hour component of the string/timestamp/interval.", extended = """ - _FUNC_(expr) + Arguments: + expr - a timestamp expression. - Arguments: - expr - timestamp type or any type expression that can be implicitly converted to - timestamp type. - - Examples: - > SELECT _FUNC_('2009-07-30 12:58:59'); - 12 + Examples: + > SELECT _FUNC_('2009-07-30 12:58:59'); + 12 """) case class Hour(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -184,15 +173,12 @@ case class Hour(child: Expression) extends UnaryExpression with ImplicitCastInpu @ExpressionDescription( usage = "_FUNC_(expr) - Returns the minute component of the string/timestamp/interval.", extended = """ - _FUNC_(expr) - - Arguments: - expr - timestamp type or any type expression that can be implicitly converted to - timestamp type. + Arguments: + expr - a timestamp expression. - Examples: - > SELECT _FUNC_('2009-07-30 12:58:59'); - 58 + Examples: + > SELECT _FUNC_('2009-07-30 12:58:59'); + 58 """) case class Minute(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -213,15 +199,12 @@ case class Minute(child: Expression) extends UnaryExpression with ImplicitCastIn @ExpressionDescription( usage = "_FUNC_(expr) - Returns the second component of the string/timestamp/interval.", extended = """ - _FUNC_(expr) + Arguments: + expr - a timestamp expression. - Arguments: - expr - timestamp type or any type expression that can be implicitly converted to - timestamp type. - - Examples: - > SELECT _FUNC_('2009-07-30 12:58:59'); - 59 + Examples: + > SELECT _FUNC_('2009-07-30 12:58:59'); + 59 """) case class Second(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -242,14 +225,12 @@ case class Second(child: Expression) extends UnaryExpression with ImplicitCastIn @ExpressionDescription( usage = "_FUNC_(expr) - Returns the day of year of date/timestamp.", extended = """ - _FUNC_(expr) - - Arguments: - expr - date type or any type expression that can be implicitly converted to date type. + Arguments: + expr - a date expression. - Examples: - > SELECT _FUNC_('2016-04-09'); - 100 + Examples: + > SELECT _FUNC_('2016-04-09'); + 100 """) case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -270,14 +251,12 @@ case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCas @ExpressionDescription( usage = "_FUNC_(expr) - Returns the year component of the date/timestamp/interval.", extended = """ - _FUNC_(expr) - - Arguments: - expr - date type or any type expression that can be implicitly converted to date type. + Arguments: + expr - a date expression. - Examples: - > SELECT _FUNC_('2016-07-30'); - 2016 + Examples: + > SELECT _FUNC_('2016-07-30'); + 2016 """) case class Year(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -298,14 +277,12 @@ case class Year(child: Expression) extends UnaryExpression with ImplicitCastInpu @ExpressionDescription( usage = "_FUNC_(expr) - Returns the quarter of the year for date, in the range 1 to 4.", extended = """ - _FUNC_(expr) + Arguments: + expr - a date expression. - Arguments: - expr - date type or any type expression that can be implicitly converted to date type. - - Examples: - > SELECT _FUNC_('2016-08-31'); - 3 + Examples: + > SELECT _FUNC_('2016-08-31'); + 3 """) case class Quarter(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -326,14 +303,12 @@ case class Quarter(child: Expression) extends UnaryExpression with ImplicitCastI @ExpressionDescription( usage = "_FUNC_(expr) - Returns the month component of the date/timestamp/interval.", extended = """ - _FUNC_(expr) - - Arguments: - expr - date type or any type expression that can be implicitly converted to date type. + Arguments: + expr - a date expression. - Examples: - > SELECT _FUNC_('2016-07-30'); - 7 + Examples: + > SELECT _FUNC_('2016-07-30'); + 7 """) case class Month(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -354,14 +329,12 @@ case class Month(child: Expression) extends UnaryExpression with ImplicitCastInp @ExpressionDescription( usage = "_FUNC_(expr) - Returns the day of month of date/timestamp, or the day of interval.", extended = """ - _FUNC_(expr) + Arguments: + expr - a date expression. - Arguments: - expr - date type or any type expression that can be implicitly converted to date type. - - Examples: - > SELECT _FUNC_('2009-07-30'); - 30 + Examples: + > SELECT _FUNC_('2009-07-30'); + 30 """) case class DayOfMonth(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -382,14 +355,12 @@ case class DayOfMonth(child: Expression) extends UnaryExpression with ImplicitCa @ExpressionDescription( usage = "_FUNC_(expr) - Returns the week of the year of the given date.", extended = """ - _FUNC_(expr) - - Arguments: - expr - date type or any type expression that can be implicitly converted to date type. + Arguments: + expr - a date expression. - Examples: - > SELECT _FUNC_('2008-02-20'); - 8 + Examples: + > SELECT _FUNC_('2008-02-20'); + 8 """) case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -431,15 +402,13 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa @ExpressionDescription( usage = "_FUNC_(expr, fmt) - Converts expr to a value of string in the format specified by the date format fmt.", extended = """ - _FUNC_(expr) + Arguments: + expr - a timestamp expression. + fmt - string type expression that represents date/timestamp format used in java.text.SimpleDateFormat. - Arguments: - expr - timestamp type or any type expression that can be implicitly converted to timestamp type. - fmt - string type expression that represents date/timestamp format used in java.text.SimpleDateFormat. - - Examples: - > SELECT _FUNC_('2016-04-08', 'y'); - 2016 + Examples: + > SELECT _FUNC_('2016-04-08', 'y'); + 2016 """) // scalastyle:on line.size.limit case class DateFormatClass(left: Expression, right: Expression) extends BinaryExpression @@ -472,16 +441,14 @@ case class DateFormatClass(left: Expression, right: Expression) extends BinaryEx @ExpressionDescription( usage = "_FUNC_(expr[, pattern]) - Returns the UNIX timestamp of the give time.", extended = """ - _FUNC_(expr[, pattern]) - - Arguments: - expr - date/string/timestamp type expression. - pattern - string type expression that represents date/timestamp format used in - java.text.SimpleDateFormat. - - Examples: - > SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd'); - 1460041200 + Arguments: + expr - a date, string or timestamp expression. + pattern - a string expression that represents date/timestamp format used in + java.text.SimpleDateFormat. + + Examples: + > SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd'); + 1460041200 """) case class ToUnixTimestamp(timeExp: Expression, format: Expression) extends UnixTime { override def left: Expression = timeExp @@ -507,18 +474,16 @@ case class ToUnixTimestamp(timeExp: Expression, format: Expression) extends Unix @ExpressionDescription( usage = "_FUNC_([expr[, pattern]]) - Returns the UNIX timestamp of current or specified time.", extended = """ - _FUNC_(expr[, pattern]) - - Arguments: - expr - date/string/timestamp type expression. - pattern - string type expression that represents date/timestamp format used in - java.text.SimpleDateFormat. - - Examples: - > SELECT _FUNC_(); - 1476884637 - > SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd'); - 1460041200 + Arguments: + expr - a date, string or timestamp expression. + pattern - a string expression that represents date/timestamp format used in + java.text.SimpleDateFormat. + + Examples: + > SELECT _FUNC_(); + 1476884637 + > SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd'); + 1460041200 """) case class UnixTimestamp(timeExp: Expression, format: Expression) extends UnixTime { override def left: Expression = timeExp @@ -646,17 +611,14 @@ abstract class UnixTime extends BinaryExpression with ExpectsInputTypes { @ExpressionDescription( usage = "_FUNC_(unix_time, format) - Returns unix_time in the specified format.", extended = """ - _FUNC_(unix_time, format) - - Arguments: - unix_time - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - format - string type expression that represents date/timestamp format used in - java.text.SimpleDateFormat. - - Examples: - > SELECT _FUNC_(0, 'yyyy-MM-dd HH:mm:ss'); - 1970-01-01 00:00:00 + Arguments: + unix_time - a numeric expression. + format - a string expression that represents date/timestamp format used in + java.text.SimpleDateFormat. + + Examples: + > SELECT _FUNC_(0, 'yyyy-MM-dd HH:mm:ss'); + 1970-01-01 00:00:00 """) case class FromUnixTime(sec: Expression, format: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -745,14 +707,12 @@ case class FromUnixTime(sec: Expression, format: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the last day of the month which the date belongs to.", extended = """ - _FUNC_(expr) + Arguments: + expr - a date expression. - Arguments: - expr - date type or any type expression that can be implicitly converted to date type. - - Examples: - > SELECT _FUNC_('2009-01-12'); - 2009-01-31 + Examples: + > SELECT _FUNC_('2009-01-12'); + 2009-01-31 """) case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def child: Expression = startDate @@ -784,17 +744,15 @@ case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitC @ExpressionDescription( usage = "_FUNC_(start_date, day_of_week) - Returns the first date which is later than start_date and named as indicated.", extended = """ - _FUNC_(expr) - - Arguments: - start_date - date type or any type expression that can be implicitly converted to date type. - day_of_week - string type expression or any type expression that can be implicitly converted to string type - that represents a day of week. "SU", "SUN", "SUNDAY", "MO", "MON", "MONDAY", "TU", "TUE", "TUESDAY", "WE", - "WED", "WEDNESDAY", "TH", "THU", "THURSDAY", "FR", "FRI", "FRIDAY", "SA", "SAT" or "SATURDAY" can be given. - - Examples: - > SELECT _FUNC_('2015-01-14', 'TU'); - 2015-01-20 + Arguments: + start_date - a date expression. + day_of_week - a string expression that represents a day of week. Available values are "SU", "SUN", "SUNDAY", "MO", + "MON", "MONDAY", "TU", "TUE", "TUESDAY", "WE", "WED", "WEDNESDAY", "TH", "THU", "THURSDAY", "FR", "FRI", "FRIDAY", + "SA", "SAT" and "SATURDAY". + + Examples: + > SELECT _FUNC_('2015-01-14', 'TU'); + 2015-01-20 """) // scalastyle:on line.size.limit case class NextDay(startDate: Expression, dayOfWeek: Expression) @@ -886,15 +844,13 @@ case class TimeAdd(start: Expression, interval: Expression) @ExpressionDescription( usage = "_FUNC_(expr, timezone) - Assumes given timestamp is UTC and converts to given timezone.", extended = """ - _FUNC_(expr, timezone) - - Arguments: - expr - timestamp type or any type expression that can be implicitly converted to timestamp type. - timezone - string type expression that represents time zone (e.g., 'Asia/Seoul'). + Arguments: + expr - a timestamp expression. + timezone - a string expression that represents time zone (e.g., 'Asia/Seoul'). - Examples: - > SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul'); - 2016-08-31 09:00:00 + Examples: + > SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul'); + 2016-08-31 09:00:00 """) // scalastyle:on line.size.limit case class FromUTCTimestamp(left: Expression, right: Expression) @@ -977,17 +933,13 @@ case class TimeSub(start: Expression, interval: Expression) @ExpressionDescription( usage = "_FUNC_(start_date, num_months) - Returns the date that is num_months after start_date.", extended = """ - _FUNC_(start_date, num_months) - - Arguments: - start_date - date type or any type expression that can be implicitly converted to - date type. - num_months - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + start_date - a date expression. + num_months - a numeric expression that defines the number of months to add. - Examples: - > SELECT _FUNC_('2016-08-31', 1); - 2016-09-30 + Examples: + > SELECT _FUNC_('2016-08-31', 1); + 2016-09-30 """) case class AddMonths(startDate: Expression, numMonths: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1019,15 +971,13 @@ case class AddMonths(startDate: Expression, numMonths: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns number of months between dates expr1 and expr2.", extended = """ - _FUNC_(expr1, expr2) + Arguments: + expr1 - a timestamp expression. + expr2 - a timestamp expression. - Arguments: - expr - timestamp type or any type expression that can be implicitly converted to - timestamp type. - - Examples: - > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); - 3.94959677 + Examples: + > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); + 3.94959677 """) case class MonthsBetween(date1: Expression, date2: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1060,15 +1010,13 @@ case class MonthsBetween(date1: Expression, date2: Expression) @ExpressionDescription( usage = "_FUNC_(expr, timezone) - Assumes given timestamp is in given timezone and converts to UTC.", extended = """ - _FUNC_(expr, timezone) - - Arguments: - timestamp - timestamp type or any type expression that can be implicitly converted to timestamp type. - timezone - string type expression that represents time zone (e.g., 'Asia/Seoul'). + Arguments: + expr - a timestamp expression. + timezone - a string expression that represents time zone (e.g., 'Asia/Seoul'). - Examples: - > SELECT _FUNC_('2016-08-31', 'Asia/Seoul'); - 2016-08-30 15:00:00 + Examples: + > SELECT _FUNC_('2016-08-31', 'Asia/Seoul'); + 2016-08-30 15:00:00 """) // scalastyle:on line.size.limit case class ToUTCTimestamp(left: Expression, right: Expression) @@ -1122,14 +1070,12 @@ case class ToUTCTimestamp(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Extracts the date part of the date or datetime expression expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - date type or any type expression that can be implicitly converted to date type. + Arguments: + expr - a date expression. - Examples: - > SELECT _FUNC_('2009-07-30 04:17:52'); - 2009-07-30 + Examples: + > SELECT _FUNC_('2009-07-30 04:17:52'); + 2009-07-30 """) case class ToDate(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -1155,17 +1101,15 @@ case class ToDate(child: Expression) extends UnaryExpression with ImplicitCastIn @ExpressionDescription( usage = "_FUNC_(expr, fmt) - Returns returns date with the time portion of the day truncated to the unit specified by the format model fmt.", extended = """ - _FUNC_(expr, fmt) - - Arguments: - expr - date type or any type expression that can be implicitly converted to date type. - fmt - string type expression that represents the truncate level. It could be "YEAR", "YYYY", "YY", "MON", "MONTH" or "MM". - - Examples: - > SELECT _FUNC_('2009-02-12', 'MM'); - 2009-02-01 - > SELECT _FUNC_('2015-10-27', 'YEAR'); - 2015-01-01 + Arguments: + expr - a date expression. + fmt - a string expression that represents the truncate level. Available values are "YEAR", "YYYY", "YY", "MON", "MONTH" and "MM". + + Examples: + > SELECT _FUNC_('2009-02-12', 'MM'); + 2009-02-01 + > SELECT _FUNC_('2015-10-27', 'YEAR'); + 2015-01-01 """) // scalastyle:on line.size.limit case class TruncDate(date: Expression, format: Expression) @@ -1240,14 +1184,13 @@ case class TruncDate(date: Expression, format: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the number of days between expr1 and expr2.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr - date type or any type expression that can be implicitly converted to date type. + Arguments: + expr1 - a date expression. + expr2 - a date expression. - Examples: - > SELECT _FUNC_('2009-07-30', '2009-07-31'); - 1 + Examples: + > SELECT _FUNC_('2009-07-30', '2009-07-31'); + 1 """) case class DateDiff(endDate: Expression, startDate: Expression) extends BinaryExpression with ImplicitCastInputTypes { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index f876aa0a7f67..023142200683 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -104,16 +104,14 @@ case class UserDefinedGenerator( @ExpressionDescription( usage = "_FUNC_(n, v1, ..., vk) - Separate v1, ..., vk into n rows.", extended = """ - _FUNC_(n, v1, ..., vk) - - Arguments: - n - integer type literal that represents the number of output rows. - v - any type expression. - - Examples: - > SELECT _FUNC_(2, 1, 2, 3); - 1 2 - 3 NULL + Arguments: + n - a integer literal that represents the number of output rows. + v - an expression of any type. + + Examples: + > SELECT _FUNC_(2, 1, 2, 3); + 1 2 + 3 NULL """) case class Stack(children: Seq[Expression]) extends Expression with Generator with CodegenFallback { @@ -239,15 +237,13 @@ abstract class ExplodeBase(child: Expression, position: Boolean) @ExpressionDescription( usage = "_FUNC_(expr) - Separates the elements of array a into multiple rows, or the elements of map a into multiple rows and columns.", extended = """ - _FUNC_(expr) - - Arguments: - expr - array type expression that contains map type or struct type expression as the element. + Arguments: + expr - an array expression that contains map or struct expression as the element. - Examples: - > SELECT _FUNC_(array(10, 20)); - 10 - 20 + Examples: + > SELECT _FUNC_(array(10, 20)); + 10 + 20 """) // scalastyle:on line.size.limit case class Explode(child: Expression) extends ExplodeBase(child, position = false) @@ -265,15 +261,13 @@ case class Explode(child: Expression) extends ExplodeBase(child, position = fals @ExpressionDescription( usage = "_FUNC_(expr) - Separates the elements of array a into multiple rows with positions, or the elements of a map into multiple rows and columns with positions.", extended = """ - _FUNC_(expr) + Arguments: + expr - an array expression that contains map or struct expression as the element. - Arguments: - expr - array type expression that contains map type or struct type expression as the element. - - Examples: - > SELECT _FUNC_(array(10,20)); - 0 10 - 1 20 + Examples: + > SELECT _FUNC_(array(10,20)); + 0 10 + 1 20 """) // scalastyle:on line.size.limit case class PosExplode(child: Expression) extends ExplodeBase(child, position = true) @@ -284,15 +278,13 @@ case class PosExplode(child: Expression) extends ExplodeBase(child, position = t @ExpressionDescription( usage = "_FUNC_(expr) - Explodes an array of structs into a table.", extended = """ - _FUNC_(expr) - - Arguments: - expr - array type expression that contains struct type expression as the element. + Arguments: + expr - an array expression that contains struct expression as the element. - Examples: - > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b'))); - 1 a - 2 b + Examples: + > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b'))); + 1 a + 2 b """) case class Inline(child: Expression) extends UnaryExpression with Generator with CodegenFallback { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index 931bf95840c5..0800c87846b3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -111,15 +111,13 @@ private[this] object SharedFactory { @ExpressionDescription( usage = "_FUNC_(json_txt, path) - Extracts a json object from path.", extended = """ - _FUNC_(json_txt, path) + Arguments: + json_txt - a string expression that represents JSON document. + path - a string expression that represents path for JSON document. - Arguments: - json_txt - string type expression that represents JSON document. - path - string type expression that represents path for JSON document. - - Examples: - > SELECT _FUNC_('{"a":"b"}', '$.a'); - b + Examples: + > SELECT _FUNC_('{"a":"b"}', '$.a'); + b """) case class GetJsonObject(json: Expression, path: Expression) extends BinaryExpression with ExpectsInputTypes with CodegenFallback { @@ -338,15 +336,13 @@ case class GetJsonObject(json: Expression, path: Expression) @ExpressionDescription( usage = "_FUNC_(jsonStr, p1, p2, ..., pn) - like get_json_object, but it takes multiple names and return a tuple. All the input parameters and output column types are string.", extended = """ - _FUNC_(jsonStr, p1, p2, ..., pn) - - Arguments: - json_txt - string type expression that represents JSON document. - p - string type expression that represents key name for JSON document. + Arguments: + json_txt - a string expression that represents JSON document. + p - a string expression that represents key name for JSON document. - Examples: - > SELECT _FUNC_('{"a":1, "b":2}', 'a', 'b'); - 1 2 + Examples: + > SELECT _FUNC_('{"a":1, "b":2}', 'a', 'b'); + 1 2 """) // scalastyle:on line.size.limit case class JsonTuple(children: Seq[Expression]) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 40a31ff6be2a..a6194c6cd0af 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -141,11 +141,9 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String) @ExpressionDescription( usage = "_FUNC_() - Returns Euler's number, E.", extended = """ - _FUNC_() - - Examples: - > SELECT _FUNC_(); - 2.718281828459045 + Examples: + > SELECT _FUNC_(); + 2.718281828459045 """) case class EulerNumber() extends LeafMathExpression(math.E, "E") @@ -156,11 +154,9 @@ case class EulerNumber() extends LeafMathExpression(math.E, "E") @ExpressionDescription( usage = "_FUNC_() - Returns PI.", extended = """ - _FUNC_() - - Examples: - > SELECT _FUNC_(); - 3.141592653589793 + Examples: + > SELECT _FUNC_(); + 3.141592653589793 """) case class Pi() extends LeafMathExpression(math.Pi, "PI") @@ -173,45 +169,36 @@ case class Pi() extends LeafMathExpression(math.Pi, "PI") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the arc cosine of expr if -1<=expr<=1 or NaN otherwise.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(1); - 0.0 - > SELECT _FUNC_(2); - NaN + Arguments: + expr - a numeric expression. + + Examples: + > SELECT _FUNC_(1); + 0.0 + > SELECT _FUNC_(2); + NaN """) case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the arc sin of expr if -1<=expr<=1 or NaN otherwise.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(0); - 0.0 - > SELECT _FUNC_(2); - NaN + Arguments: + expr - a numeric expression. + + Examples: + > SELECT _FUNC_(0); + 0.0 + > SELECT _FUNC_(2); + NaN """) case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the arc tangent.", extended = """ - _FUNC_(expr) - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + expr - a numeric expression. Examples: > SELECT _FUNC_(0); @@ -222,32 +209,26 @@ case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN" @ExpressionDescription( usage = "_FUNC_(expr) - Returns the cube root of a double value.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(27.0); - 3.0 + Examples: + > SELECT _FUNC_(27.0); + 3.0 """) case class Cbrt(child: Expression) extends UnaryMathExpression(math.cbrt, "CBRT") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the smallest integer not smaller than expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(-0.1); - 0 - > SELECT _FUNC_(5); - 5 + Arguments: + expr - a numeric expression. + + Examples: + > SELECT _FUNC_(-0.1); + 0 + > SELECT _FUNC_(5); + 5 """) case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil, "CEIL") { override def dataType: DataType = child.dataType match { @@ -278,30 +259,24 @@ case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil, "CEIL" @ExpressionDescription( usage = "_FUNC_(expr) - Returns the cosine of expr.", extended = """ - _FUNC_(expr) + Arguments: + expr - a numeric expression. - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(0); - 1.0 + Examples: + > SELECT _FUNC_(0); + 1.0 """) case class Cos(child: Expression) extends UnaryMathExpression(math.cos, "COS") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the hyperbolic cosine of expr.", extended = """ - _FUNC_(expr) + Arguments: + expr - a numeric expression. - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(0); - 1.0 + Examples: + > SELECT _FUNC_(0); + 1.0 """) case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH") @@ -315,21 +290,16 @@ case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH" @ExpressionDescription( usage = "_FUNC_(num, from_base, to_base) - Convert num from from_base to to_base.", extended = """ - _FUNC_(num, from_base, to_base) - - Arguments: - num - any string type or any type expression that can be implicitly - converted to numeric type. - from_base - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - to_base - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_('100', 2, 10); - 4 - > SELECT _FUNC_(-10, 16, -10); - 16 + Arguments: + num - a string expression. + from_base - a numeric expression. + to_base - a numeric expression. + + Examples: + > SELECT _FUNC_('100', 2, 10); + 4 + > SELECT _FUNC_(-10, 16, -10); + 16 """) case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -362,47 +332,38 @@ case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expre @ExpressionDescription( usage = "_FUNC_(expr) - Returns e to the power of expr.", extended = """ - _FUNC_(expr) + Arguments: + expr - a numeric expression. - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(0); - 1.0 + Examples: + > SELECT _FUNC_(0); + 1.0 """) case class Exp(child: Expression) extends UnaryMathExpression(math.exp, "EXP") @ExpressionDescription( usage = "_FUNC_(expr) - Returns exp(expr) - 1.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(0); - 0.0 + Examples: + > SELECT _FUNC_(0); + 0.0 """) case class Expm1(child: Expression) extends UnaryMathExpression(math.expm1, "EXPM1") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the largest integer not greater than expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(-0.1); - -1 - > SELECT _FUNC_(5); - 5 + Arguments: + expr - a numeric expression. + + Examples: + > SELECT _FUNC_(-0.1); + -1 + > SELECT _FUNC_(5); + 5 """) case class Floor(child: Expression) extends UnaryMathExpression(math.floor, "FLOOR") { override def dataType: DataType = child.dataType match { @@ -464,15 +425,12 @@ object Factorial { @ExpressionDescription( usage = "_FUNC_(expr) - Returns expr factorial for expr is [0..20]. Otherwise, NULL.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(5); - 120 + Examples: + > SELECT _FUNC_(5); + 120 """) case class Factorial(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -509,30 +467,24 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas @ExpressionDescription( usage = "_FUNC_(expr) - Returns the natural logarithm of x with base expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(1); - 0.0 + Examples: + > SELECT _FUNC_(1); + 0.0 """) case class Log(child: Expression) extends UnaryLogExpression(math.log, "LOG") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the logarithm of expr with base 2.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(2); - 1.0 + Examples: + > SELECT _FUNC_(2); + 1.0 """) case class Log2(child: Expression) extends UnaryLogExpression((x: Double) => math.log(x) / math.log(2), "LOG2") { @@ -552,30 +504,24 @@ case class Log2(child: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns the logarithm of expr with base 10.", extended = """ - _FUNC_(expr) + Arguments: + expr - a numeric expression. - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(10); - 1.0 + Examples: + > SELECT _FUNC_(10); + 1.0 """) case class Log10(child: Expression) extends UnaryLogExpression(math.log10, "LOG10") @ExpressionDescription( usage = "_FUNC_(expr) - Returns log(1 + expr).", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(0); - 0.0 + Examples: + > SELECT _FUNC_(0); + 0.0 """) case class Log1p(child: Expression) extends UnaryLogExpression(math.log1p, "LOG1P") { protected override val yAsymptote: Double = -1.0 @@ -585,15 +531,12 @@ case class Log1p(child: Expression) extends UnaryLogExpression(math.log1p, "LOG1 @ExpressionDescription( usage = "_FUNC_(expr) - Returns the double value that is closest in value to the argument and is equal to a mathematical integer.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(12.3456); - 12.0 + Examples: + > SELECT _FUNC_(12.3456); + 12.0 """) // scalastyle:on line.size.limit case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND") { @@ -603,105 +546,84 @@ case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sign of expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(40); - 1.0 + Examples: + > SELECT _FUNC_(40); + 1.0 """) case class Signum(child: Expression) extends UnaryMathExpression(math.signum, "SIGNUM") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the sine of expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(0); - 0.0 + Examples: + > SELECT _FUNC_(0); + 0.0 """) case class Sin(child: Expression) extends UnaryMathExpression(math.sin, "SIN") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the hyperbolic sine of expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(0); - 0.0 + Examples: + > SELECT _FUNC_(0); + 0.0 """) case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the square root of expr.", extended = """ - _FUNC_(expr) + Arguments: + expr - a numeric expression. - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(4); - 2.0 + Examples: + > SELECT _FUNC_(4); + 2.0 """) case class Sqrt(child: Expression) extends UnaryMathExpression(math.sqrt, "SQRT") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the tangent of expr.", extended = """ - _FUNC_(expr) + Arguments: + expr - a numeric expression. - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(0); - 0.0 + Examples: + > SELECT _FUNC_(0); + 0.0 """) case class Tan(child: Expression) extends UnaryMathExpression(math.tan, "TAN") @ExpressionDescription( usage = "_FUNC_(expr) - Returns the hyperbolic tangent of expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(0); - 0.0 + Examples: + > SELECT _FUNC_(0); + 0.0 """) case class Tanh(child: Expression) extends UnaryMathExpression(math.tanh, "TANH") @ExpressionDescription( usage = "_FUNC_(expr) - Converts radians to degrees.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(3.141592653589793); - 180.0 + Examples: + > SELECT _FUNC_(3.141592653589793); + 180.0 """) case class ToDegrees(child: Expression) extends UnaryMathExpression(math.toDegrees, "DEGREES") { override def funcName: String = "toDegrees" @@ -710,15 +632,12 @@ case class ToDegrees(child: Expression) extends UnaryMathExpression(math.toDegre @ExpressionDescription( usage = "_FUNC_(expr) - Converts degrees to radians.", extended = """ - _FUNC_(expr) + Arguments: + expr - a numeric expression. - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(180); - 3.141592653589793 + Examples: + > SELECT _FUNC_(180); + 3.141592653589793 """) case class ToRadians(child: Expression) extends UnaryMathExpression(math.toRadians, "RADIANS") { override def funcName: String = "toRadians" @@ -727,15 +646,12 @@ case class ToRadians(child: Expression) extends UnaryMathExpression(math.toRadia @ExpressionDescription( usage = "_FUNC_(expr) - Returns expr in binary.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(13); - 1101 + Examples: + > SELECT _FUNC_(13); + 1101 """) case class Bin(child: Expression) extends UnaryExpression with Serializable with ImplicitCastInputTypes { @@ -831,17 +747,14 @@ object Hex { @ExpressionDescription( usage = "_FUNC_(expr) - Converts the expr to hexadecimal.", extended = """ - _FUNC_(expr) - - Arguments: - expr - numeric type, binary type, string type or any type expression that can be - implicitly converted to one of these types. - - Examples: - > SELECT _FUNC_(17); - 11 - > SELECT _FUNC_('Spark SQL'); - 537061726B2053514C + Arguments: + expr - a numeric, binary or string expression. + + Examples: + > SELECT _FUNC_(17); + 11 + > SELECT _FUNC_('Spark SQL'); + 537061726B2053514C """) case class Hex(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -874,14 +787,12 @@ case class Hex(child: Expression) extends UnaryExpression with ImplicitCastInput @ExpressionDescription( usage = "_FUNC_(expr) - Converts hexadecimal expr to binary.", extended = """ - _FUNC_(expr) + Arguments: + expr - a string expression. - Arguments: - expr - string type or any type expression that can be implicitly converted to string type. - - Examples: - > SELECT decode(_FUNC_('537061726B2053514C'), 'UTF-8'); - Spark SQL + Examples: + > SELECT decode(_FUNC_('537061726B2053514C'), 'UTF-8'); + Spark SQL """) case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -914,17 +825,13 @@ case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInp @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the arc tangent2.", extended = """ - _FUNC_(expr1, expr2) + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(0, 0); - 0.0 + Examples: + > SELECT _FUNC_(0, 0); + 0.0 """) case class Atan2(left: Expression, right: Expression) extends BinaryMathExpression(math.atan2, "ATAN2") { @@ -942,17 +849,13 @@ case class Atan2(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Raise expr1 to the power of expr2.", extended = """ - _FUNC_(expr1, expr2) + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(2, 3); - 8.0 + Examples: + > SELECT _FUNC_(2, 3); + 8.0 """) case class Pow(left: Expression, right: Expression) extends BinaryMathExpression(math.pow, "POWER") { @@ -971,17 +874,13 @@ case class Pow(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Bitwise left shift.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Examples: - > SELECT _FUNC_(2, 1); - 4 + Examples: + > SELECT _FUNC_(2, 1); + 4 """) case class ShiftLeft(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1003,7 +902,6 @@ case class ShiftLeft(left: Expression, right: Expression) } } - /** * Bitwise right shift. * @@ -1013,17 +911,13 @@ case class ShiftLeft(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Bitwise right shift.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Examples: - > SELECT _FUNC_(4, 1); - 2 + Examples: + > SELECT _FUNC_(4, 1); + 2 """) case class ShiftRight(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1045,7 +939,6 @@ case class ShiftRight(left: Expression, right: Expression) } } - /** * Bitwise unsigned right shift, for integer and long data type. * @@ -1055,17 +948,13 @@ case class ShiftRight(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Bitwise unsigned right shift.", extended = """ - _FUNC_(expr1, expr2) + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(4, 1); - 2 + Examples: + > SELECT _FUNC_(4, 1); + 2 """) case class ShiftRightUnsigned(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1090,17 +979,13 @@ case class ShiftRightUnsigned(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns sqrt(expr1**2 + expr2**2).", extended = """ - _FUNC_(expr1, expr2) + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(3, 4); - 5.0 + Examples: + > SELECT _FUNC_(3, 4); + 5.0 """) case class Hypot(left: Expression, right: Expression) extends BinaryMathExpression(math.hypot, "HYPOT") @@ -1115,17 +1000,13 @@ case class Hypot(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the logarithm of expr1 with base expr2.", extended = """ - _FUNC_(expr1, expr2) + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_(10, 100); - 2.0 + Examples: + > SELECT _FUNC_(10, 100); + 2.0 """) case class Logarithm(left: Expression, right: Expression) extends BinaryMathExpression((c1, c2) => math.log(c2) / math.log(c1), "LOG") { @@ -1353,17 +1234,13 @@ abstract class RoundBase(child: Expression, scale: Expression, @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Round expr1 to expr2 decimal places using HALF_UP rounding mode.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Examples: - > SELECT _FUNC_(2.5, 0); - 3.0 + Examples: + > SELECT _FUNC_(2.5, 0); + 3.0 """) // scalastyle:on line.size.limit case class Round(child: Expression, scale: Expression) @@ -1381,17 +1258,13 @@ case class Round(child: Expression, scale: Expression) @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Round expr1 to expr2 decimal places using HALF_EVEN rounding mode.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Examples: - > SELECT _FUNC_(2.5, 0); - 2.0 + Examples: + > SELECT _FUNC_(2.5, 0); + 2.0 """) // scalastyle:on line.size.limit case class BRound(child: Expression, scale: Expression) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala index e0bcd7b393d4..a86581cd6e41 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala @@ -40,15 +40,12 @@ import org.apache.spark.unsafe.Platform @ExpressionDescription( usage = "_FUNC_(expr) - Returns an MD5 128-bit checksum as a hex string of expr.", extended = """ - _FUNC_(expr) + Arguments: + expr - a binary expression. - Arguments: - expr - binary type or any type expression that can be implicitly converted to - binary type. - - Examples: - > SELECT _FUNC_('Spark'); - 8cde774d6f7333752ed72cacddb05126 + Examples: + > SELECT _FUNC_('Spark'); + 8cde774d6f7333752ed72cacddb05126 """) case class Md5(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -80,17 +77,14 @@ case class Md5(child: Expression) extends UnaryExpression with ImplicitCastInput SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256. """, extended = """ - _FUNC_(expr, bitLength) - - Arguments: - expr - string type or any type expression that can be implicitly converted to - string type. - bitLength - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - - Examples: - > SELECT _FUNC_('Spark', 0); - 529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b + Arguments: + expr - a string expression. + bitLength - a numeric expression that defines the length of bit. Available values are 224, + 256, 384 and 512. + + Examples: + > SELECT _FUNC_('Spark', 256); + 529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b """) // scalastyle:on line.size.limit case class Sha2(left: Expression, right: Expression) @@ -161,15 +155,12 @@ case class Sha2(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns a sha1 hash value as a hex string of the expr.", extended = """ - _FUNC_(expr) + Arguments: + expr - a binary expression. - Arguments: - expr - binary type or any type expression that that can be implicitly converted to - binary type. - - Examples: - > SELECT _FUNC_('Spark'); - 85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c + Examples: + > SELECT _FUNC_('Spark'); + 85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c """) case class Sha1(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -194,15 +185,12 @@ case class Sha1(child: Expression) extends UnaryExpression with ImplicitCastInpu @ExpressionDescription( usage = "_FUNC_(expr) - Returns a cyclic redundancy check value as a bigint of the expr.", extended = """ - _FUNC_(expr) - - Arguments: - expr - binary type or any type expression that that can be implicitly converted to - binary type. + Arguments: + expr - a binary expression. - Examples: - > SELECT _FUNC_('Spark'); - 1557323817 + Examples: + > SELECT _FUNC_('Spark'); + 1557323817 """) case class Crc32(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -535,14 +523,12 @@ abstract class InterpretedHashFunction { @ExpressionDescription( usage = "_FUNC_(expr1, expr2, ...) - Returns a hash value of the arguments.", extended = """ - _FUNC_(expr) + Arguments: + expr - an expression of any type. - Arguments: - expr - any type expression. - - Examples: - > SELECT _FUNC_('Spark', array(123), 2); - -1321691492 + Examples: + > SELECT _FUNC_('Spark', array(123), 2); + -1321691492 """) case class Murmur3Hash(children: Seq[Expression], seed: Int) extends HashExpression[Int] { def this(arguments: Seq[Expression]) = this(arguments, 42) @@ -599,15 +585,12 @@ case class PrintToStderr(child: Expression) extends UnaryExpression { @ExpressionDescription( usage = "_FUNC_(expr) - Throw an exception if expr is not true.", extended = """ - _FUNC_(expr) - - Arguments: - expr - boolean type or any type expression that can be implicitly converted to - boolean type. + Arguments: + expr - a boolean expression - Examples: - > SELECT _FUNC_(0 < 1); - NULL + Examples: + > SELECT _FUNC_(0 < 1); + NULL """) case class AssertTrue(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -678,11 +661,9 @@ object XxHash64Function extends InterpretedHashFunction { @ExpressionDescription( usage = "_FUNC_() - Returns the current database.", extended = """ - _FUNC_() - - Examples: - > SELECT _FUNC_(); - default + Examples: + > SELECT _FUNC_(); + default """) case class CurrentDatabase() extends LeafExpression with Unevaluable { override def dataType: DataType = StringType @@ -703,10 +684,8 @@ case class CurrentDatabase() extends LeafExpression with Unevaluable { @ExpressionDescription( usage = "_FUNC_(expr1, expr2, ...) - Returns a hash value of the arguments.", extended = """ - _FUNC_(expr1, expr2, ...) - - Arguments: - expr - any type expression. + Arguments: + expr - an expression of any type. """) case class HiveHash(children: Seq[Expression]) extends HashExpression[Int] { override val seed = 0 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala index b74f25e8829a..84a4c2527058 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala @@ -38,14 +38,12 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr1, expr2, ...) - Returns the first non-null argument if exists. Otherwise, NULL.", extended = """ - _FUNC_(expr1, expr2, ...) + Arguments: + expr - an expression of any type. - Arguments: - expr - any type expression. - - Examples: - > SELECT _FUNC_(NULL, 1, NULL); - 1 + Examples: + > SELECT _FUNC_(NULL, 1, NULL); + 1 """) // scalastyle:on line.size.limit case class Coalesce(children: Seq[Expression]) extends Expression { @@ -101,15 +99,13 @@ case class Coalesce(children: Seq[Expression]) extends Expression { @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns expr2 if expr1 is null, or expr1 otherwise.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - any type expression. - expr2 - any type expression. + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. - Examples: - > SELECT _FUNC_(NULL, array('2')); - ["2"] + Examples: + > SELECT _FUNC_(NULL, array('2')); + ["2"] """) case class IfNull(left: Expression, right: Expression) extends RuntimeReplaceable { override def children: Seq[Expression] = Seq(left, right) @@ -130,15 +126,13 @@ case class IfNull(left: Expression, right: Expression) extends RuntimeReplaceabl @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns null if expr1 equals to expr2, or expr1 otherwise.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - any type expression. - expr2 - any type expression. + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. - Examples: - > SELECT _FUNC_(2, 2); - NULL + Examples: + > SELECT _FUNC_(2, 2); + NULL """) case class NullIf(left: Expression, right: Expression) extends RuntimeReplaceable { override def children: Seq[Expression] = Seq(left, right) @@ -161,15 +155,13 @@ case class NullIf(left: Expression, right: Expression) extends RuntimeReplaceabl @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns expr2 if expr1 is null, or expr1 otherwise.", extended = """ - _FUNC_(expr1, expr2) + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. - Arguments: - expr1 - any type expression. - expr2 - any type expression. - - Examples: - > SELECT _FUNC_(NULL, array('2')); - ["2"] + Examples: + > SELECT _FUNC_(NULL, array('2')); + ["2"] """) case class Nvl(left: Expression, right: Expression) extends RuntimeReplaceable { override def children: Seq[Expression] = Seq(left, right) @@ -191,16 +183,14 @@ case class Nvl(left: Expression, right: Expression) extends RuntimeReplaceable { @ExpressionDescription( usage = "_FUNC_(expr1, expr2, expr3) - Returns expr2 if expr1 is not null, or expr3 otherwise.", extended = """ - _FUNC_(expr1, expr2, expr3) - - Arguments: - expr1 - any type expression. - expr2 - any type expression. - expr3 - any type expression. - - Examples: - > SELECT _FUNC_(NULL, 2, 1); - 1 + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. + expr3 - an expression of any type. + + Examples: + > SELECT _FUNC_(NULL, 2, 1); + 1 """) // scalastyle:on line.size.limit case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression) @@ -227,15 +217,12 @@ case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if expr is NaN and false otherwise.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr - a numeric expression. - Examples: - > SELECT _FUNC_(cast('NaN' as double)); - true + Examples: + > SELECT _FUNC_(cast('NaN' as double)); + true """) case class IsNaN(child: Expression) extends UnaryExpression with Predicate with ImplicitCastInputTypes { @@ -276,17 +263,13 @@ case class IsNaN(child: Expression) extends UnaryExpression @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns expr1 if it's not NaN, or expr2 otherwise.", extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. - expr2 - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. + Arguments: + expr1 - a numeric expression. + expr2 - a numeric expression. - Examples: - > SELECT _FUNC_(cast('NaN' as double), 123); - 123.0 + Examples: + > SELECT _FUNC_(cast('NaN' as double), 123); + 123.0 """) case class NaNvl(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -344,14 +327,12 @@ case class NaNvl(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if expr is NULL and false otherwise.", extended = """ - _FUNC_(expr) + Arguments: + expr - an expression of any type. - Arguments: - expr - any type expression. - - Examples: - > SELECT _FUNC_(1); - false + Examples: + > SELECT _FUNC_(1); + false """) case class IsNull(child: Expression) extends UnaryExpression with Predicate { override def nullable: Boolean = false @@ -375,14 +356,12 @@ case class IsNull(child: Expression) extends UnaryExpression with Predicate { @ExpressionDescription( usage = "_FUNC_(expr) - Returns true if expr is not NULL and false otherwise.", extended = """ - _FUNC_(expr) - - Arguments: - expr - any type expression. + Arguments: + expr - an expression of any type. - Examples: - > SELECT _FUNC_(1); - true + Examples: + > SELECT _FUNC_(1); + true """) case class IsNotNull(child: Expression) extends UnaryExpression with Predicate { override def nullable: Boolean = false diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index 405256c00670..559cb3233911 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -94,10 +94,8 @@ trait PredicateHelper { @ExpressionDescription( usage = "_FUNC_ expr - Logical not.", extended = """ - _FUNC_ expr - - Arguments: - expr - any type expression. + Arguments: + expr - an expression of any type. """) case class Not(child: Expression) extends UnaryExpression with Predicate with ImplicitCastInputTypes with NullIntolerant { @@ -122,10 +120,8 @@ case class Not(child: Expression) @ExpressionDescription( usage = "expr1 _FUNC_(expr2, expr3, ...) - Returns true if expr equals to any valN.", extended = """ - expr1 _FUNC_(expr2, expr3, ...) - - Arguments: - expr - any type expression. + Arguments: + expr - an expression of any type. """) case class In(value: Expression, list: Seq[Expression]) extends Predicate with ImplicitCastInputTypes { @@ -265,11 +261,9 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Logical AND.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any type expression. - expr2 - any type expression. + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. """) case class And(left: Expression, right: Expression) extends BinaryOperator with Predicate { @@ -335,11 +329,9 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Logical OR.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any type expression. - expr2 - any type expression. + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. """) case class Or(left: Expression, right: Expression) extends BinaryOperator with Predicate { @@ -437,11 +429,9 @@ object Equality { @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 equals expr2 and false otherwise.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any type expression. - expr2 - any type expression. + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. """) case class EqualTo(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { @@ -473,11 +463,9 @@ case class EqualTo(left: Expression, right: Expression) but returns TRUE if both are NULL, FALSE if one of the them is NULL. """, extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any type expression. - expr2 - any type expression. + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. """) case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComparison { @@ -520,11 +508,9 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is less than expr2.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any type expression. - expr2 - any type expression. + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. """) case class LessThan(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { @@ -541,11 +527,9 @@ case class LessThan(left: Expression, right: Expression) @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is not greater than expr2.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any type expression. - expr2 - any type expression. + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. """) case class LessThanOrEqual(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { @@ -562,11 +546,9 @@ case class LessThanOrEqual(left: Expression, right: Expression) @ExpressionDescription( usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is greater than expr2.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any type expression. - expr2 - any type expression. + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. """) case class GreaterThan(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { @@ -581,13 +563,11 @@ case class GreaterThan(left: Expression, right: Expression) } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns TRUE if a is not smaller than b.", + usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is not smaller than expr2.", extended = """ - expr1 _FUNC_ expr2 - - Arguments: - expr1 - any type expression. - expr2 - any type expression. + Arguments: + expr1 - an expression of any type. + expr2 - an expression of any type. """) case class GreaterThanOrEqual(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala index ffa52b357121..04ef36a09751 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala @@ -59,16 +59,14 @@ abstract class RDG extends LeafExpression with Nondeterministic { @ExpressionDescription( usage = "_FUNC_([seed]) - Returns a random column with i.i.d. uniformly distributed values in [0, 1].", extended = """ - _FUNC_([seed]) - - Arguments: - seed - numeric type literal. - - Examples: - > SELECT _FUNC_(); - 0.9629742951434543 - > SELECT _FUNC_(0); - 0.8446490682263027 + Arguments: + seed - a numeric literal. + + Examples: + > SELECT _FUNC_(); + 0.9629742951434543 + > SELECT _FUNC_(0); + 0.8446490682263027 """) // scalastyle:on line.size.limit case class Rand(seed: Long) extends RDG { @@ -96,16 +94,14 @@ case class Rand(seed: Long) extends RDG { @ExpressionDescription( usage = "_FUNC_([seed]) - Returns a random column with i.i.d. values drawn from the standard normal distribution.", extended = """ - _FUNC_([seed]) - - Arguments: - seed - numeric type literal. - - Examples: - > SELECT _FUNC_(); - -0.3254147983080288 - > SELECT _FUNC_(0); - 1.1164209726833079 + Arguments: + seed - a numeric literal. + + Examples: + > SELECT _FUNC_(); + -0.3254147983080288 + > SELECT _FUNC_(0); + 1.1164209726833079 """) // scalastyle:on line.size.limit case class Randn(seed: Long) extends RDG { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index dc3bf19bc5c2..4e235b692e7d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -70,13 +70,9 @@ trait StringRegexExpression extends ImplicitCastInputTypes { @ExpressionDescription( usage = "str _FUNC_ pattern - Returns true if str matches pattern and false otherwise.", extended = """ - str _FUNC_ pattern - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - pattern - string type or any type expression that can be implicitly converted - to string type. + Arguments: + str - a string expression. + pattern - a string expression. """) case class Like(left: Expression, right: Expression) extends BinaryExpression with StringRegexExpression { @@ -132,13 +128,9 @@ case class Like(left: Expression, right: Expression) @ExpressionDescription( usage = "str _FUNC_ regexp - Returns true if str matches regexp and false otherwise.", extended = """ - str _FUNC_ regexp - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - regexp - string type or any type expression that can be implicitly converted - to string type. + Arguments: + str - a string expression. + regexp - a string expression. """) case class RLike(left: Expression, right: Expression) extends BinaryExpression with StringRegexExpression { @@ -195,17 +187,13 @@ case class RLike(left: Expression, right: Expression) @ExpressionDescription( usage = "_FUNC_(str, regex) - Splits str around occurrences that match regex.", extended = """ - _FUNC_(str, regex) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - regex - string type or any type expression that can be implicitly converted - to string type. + Arguments: + str - a string expression. + regex - a string expression. - Examples: - > SELECT _FUNC_('oneAtwoBthreeC', '[ABC]'); - ["one","two","three",""] + Examples: + > SELECT _FUNC_('oneAtwoBthreeC', '[ABC]'); + ["one","two","three",""] """) case class StringSplit(str: Expression, pattern: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -239,19 +227,14 @@ case class StringSplit(str: Expression, pattern: Expression) @ExpressionDescription( usage = "_FUNC_(str, regexp, rep) - Replaces all substrings of str that match regexp with rep.", extended = """ - _FUNC_(str, regex, rep) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - regexp - string type or any type expression that can be implicitly converted - to string type. - rep - string type or any type expression that can be implicitly converted - to string type. - - Examples: - > SELECT _FUNC_('100-200', '(\d+)', 'num'); - num-num + Arguments: + str - a string expression. + regexp - a string expression. + rep - a string expression. + + Examples: + > SELECT _FUNC_('100-200', '(\d+)', 'num'); + num-num """) case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -355,19 +338,14 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio @ExpressionDescription( usage = "_FUNC_(str, regexp[, idx]) - Extracts a group that matches regexp.", extended = """ - _FUNC_(str, regexp[, idx]) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - regexp - string type or any type expression that can be implicitly converted - to string type. - idx - any numeric type or any nonnumeric tpye expression that can be implicitly converted - to numeric type. - - Examples: - > SELECT _FUNC_('100-200', '(\d+)-(\d+)', 1); - 100 + Arguments: + str - a string expression. + regexp - a string expression. + idx - a numeric expression that defines the index of a capturing group in regexp. + + Examples: + > SELECT _FUNC_('100-200', '(\d+)-(\d+)', 1); + 100 """) case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expression) extends TernaryExpression with ImplicitCastInputTypes { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 64fa2e86059e..37a697c0f7dc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -43,15 +43,12 @@ import org.apache.spark.unsafe.types.{ByteArray, UTF8String} @ExpressionDescription( usage = "_FUNC_(str1, str2, ..., strN) - Returns the concatenation of str1, str2, ..., strN.", extended = """ - _FUNC_(str1, str2, ..., strN) + Arguments: + str - a string expression. - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - - Examples: - > SELECT _FUNC_('Spark','SQL'); - SparkSQL + Examples: + > SELECT _FUNC_('Spark','SQL'); + SparkSQL """) case class Concat(children: Seq[Expression]) extends Expression with ImplicitCastInputTypes { @@ -92,17 +89,13 @@ case class Concat(children: Seq[Expression]) extends Expression with ImplicitCas @ExpressionDescription( usage = "_FUNC_(sep, [str | array(str)]+) - Returns the concatenation of the strings separated by sep.", extended = """ - _FUNC_(sep, [str | array(str)]+) - - Arguments: - sep - string type or any type expression that can be implicitly converted - to string type. - str - string type or any type expression that can be implicitly converted - to string type. + Arguments: + sep - a string expression that represents the separator. + str - a string expression. - Examples: - > SELECT _FUNC_(' ', Spark', 'SQL'); - Spark SQL + Examples: + > SELECT _FUNC_(' ', Spark', 'SQL'); + Spark SQL """) // scalastyle:on line.size.limit case class ConcatWs(children: Seq[Expression]) @@ -193,17 +186,13 @@ case class ConcatWs(children: Seq[Expression]) @ExpressionDescription( usage = "_FUNC_(n, str1, str2, ...) - Returns the n-th string, e.g. returns str2 when n is 2.", extended = """ - _FUNC_(n, str1, str2, ...) - - Arguments: - n - any numeric type or any nonnumeric expression that can be implicitly converted - to numeric type. - str - string type or any type expression that can be implicitly converted - to string type. + Arguments: + n - a numeric expression that defines the index of str for a return value. + str - a string expression. - Examples: - > SELECT _FUNC_(1, 'scala', 'java') FROM src LIMIT 1; - scala + Examples: + > SELECT _FUNC_(1, 'scala', 'java') FROM src LIMIT 1; + scala """) case class Elt(children: Seq[Expression]) extends Expression with ImplicitCastInputTypes { @@ -283,15 +272,12 @@ trait String2StringExpression extends ImplicitCastInputTypes { @ExpressionDescription( usage = "_FUNC_(str) - Returns str with all characters changed to uppercase.", extended = """ - _FUNC_(str) + Arguments: + str - a string expression. - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - - Examples: - > SELECT _FUNC_('SparkSql'); - SPARKSQL + Examples: + > SELECT _FUNC_('SparkSql'); + SPARKSQL """) case class Upper(child: Expression) extends UnaryExpression with String2StringExpression { @@ -309,15 +295,12 @@ case class Upper(child: Expression) @ExpressionDescription( usage = "_FUNC_(str) - Returns str with all characters changed to lowercase.", extended = """ - _FUNC_(str) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. + Arguments: + str - a string expression. - Examples: - > SELECT _FUNC_('SparkSql'); - sparksql + Examples: + > SELECT _FUNC_('SparkSql'); + sparksql """) case class Lower(child: Expression) extends UnaryExpression with String2StringExpression { @@ -404,16 +387,14 @@ object StringTranslate { @ExpressionDescription( usage = "_FUNC_(input, from, to) - Translates the input string by replacing the characters present in the from string with the corresponding characters in the to string.", extended = """ - _FUNC_(input, from, to) - - Arguments: - input - string type or any type expression that can be implicitly converted to string type. - from - string type or any type expression that can be implicitly converted to string type. - to - string type or any type expression that can be implicitly converted to string type. - - Examples: - > SELECT _FUNC_('AaBbCc', 'abc', '123'); - A1B2C3 + Arguments: + input - a string expression. + from - a string expression. + to - a string expression. + + Examples: + > SELECT _FUNC_('AaBbCc', 'abc', '123'); + A1B2C3 """) // scalastyle:on line.size.limit case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replaceExpr: Expression) @@ -478,16 +459,13 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac Returns 0, if the string wasn't found or if the given string (left) contains a comma. """, extended = """ - _FUNC_(str, str_array) + Arguments: + input - a string expression that represents a string to find. + str_array - a string expression that represents a comma-delimited string. For example, "a,bcd,e,f". - Arguments: - input - string type or any type expression that can be implicitly converted to string type. - from - string type or any type expression that can be implicitly converted to string type. - to - string type or any type expression that can be implicitly converted to string type. - - Examples: - > SELECT _FUNC_('ab','abc,b,ab,c,def'); - 3 + Examples: + > SELECT _FUNC_('ab','abc,b,ab,c,def'); + 3 """) // scalastyle:on case class FindInSet(left: Expression, right: Expression) extends BinaryExpression @@ -515,15 +493,12 @@ case class FindInSet(left: Expression, right: Expression) extends BinaryExpressi @ExpressionDescription( usage = "_FUNC_(str) - Removes the leading and trailing space characters from str.", extended = """ - _FUNC_(str) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. + Arguments: + str - a string expression. - Examples: - > SELECT _FUNC_(' SparkSQL '); - SparkSQL + Examples: + > SELECT _FUNC_(' SparkSQL '); + SparkSQL """) case class StringTrim(child: Expression) extends UnaryExpression with String2StringExpression { @@ -543,15 +518,12 @@ case class StringTrim(child: Expression) @ExpressionDescription( usage = "_FUNC_(str) - Removes the leading and trailing space characters from str.", extended = """ - _FUNC_(str) + Arguments: + str - a string expression. - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - - Examples: - > SELECT _FUNC_(' SparkSQL'); - SparkSQL + Examples: + > SELECT _FUNC_(' SparkSQL'); + SparkSQL """) case class StringTrimLeft(child: Expression) extends UnaryExpression with String2StringExpression { @@ -571,15 +543,12 @@ case class StringTrimLeft(child: Expression) @ExpressionDescription( usage = "_FUNC_(str) - Removes the trailing space characters from str.", extended = """ - _FUNC_(str) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. + Arguments: + str - a string expression. - Examples: - > SELECT _FUNC_(' SparkSQL '); - SparkSQL + Examples: + > SELECT _FUNC_(' SparkSQL '); + SparkSQL """) case class StringTrimRight(child: Expression) extends UnaryExpression with String2StringExpression { @@ -603,17 +572,13 @@ case class StringTrimRight(child: Expression) @ExpressionDescription( usage = "_FUNC_(str, substr) - Returns the (1-based) index of the first occurrence of substr in str.", extended = """ - _FUNC_(str, substr) + Arguments: + str - a string expression. + substr - a string expression that represents a string to find. - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - substr - string type or any type expression that can be implicitly converted - to string type. - - Examples: - > SELECT _FUNC_('SparkSQL', 'SQL'); - 6 + Examples: + > SELECT _FUNC_('SparkSQL', 'SQL'); + 6 """) case class StringInstr(str: Expression, substr: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -651,16 +616,14 @@ case class StringInstr(str: Expression, substr: Expression) when searching for delim. """, extended = """ - _FUNC_(str, delim, count) - - Arguments: - str - string type or any type expression that can be implicitly converted to string type. - delim - string type or any type expression that can be implicitly converted to string type. - count - any numeric type or any nonnumeric expression that can be implicitly converted to numeric type. - - Examples: - > SELECT _FUNC_('www.apache.org', '.', 2); - www.apache + Arguments: + str - a string expression. + delim - a string expression that represents the delimiter. + count - a numeric expression that defines the number of occurrences of the delimiter delim . + + Examples: + > SELECT _FUNC_('www.apache.org', '.', 2); + www.apache """) // scalastyle:on line.size.limit case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: Expression) @@ -690,18 +653,17 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: @ExpressionDescription( usage = """ _FUNC_(substr, str[, pos]) - Returns the position of the first occurrence of substr in str after position pos. - The given pos and return value are 1-based.""", + The given pos and return value are 1-based. + """, extended = """ - _FUNC_(substr, str[, pos]) - - Arguments: - substr - string type or any type expression that can be implicitly converted to string type. - str - string type or any type expression that can be implicitly converted to string type. - pos - any numeric type or any nonnumeric expression that can be implicitly converted to numeric type. - - Examples: - > SELECT _FUNC_('bar', 'foobarbar', 5); - 7 + Arguments: + substr - a string expression that represents a string to find. + str - a string expression. + pos - a numeric expression that defines the position to start to locate. + + Examples: + > SELECT _FUNC_('bar', 'foobarbar', 5); + 7 """) // scalastyle:on line.size.limit case class StringLocate(substr: Expression, str: Expression, start: Expression) @@ -782,21 +744,16 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression) If str is longer than len, the return value is shortened to len characters. """, extended = """ - _FUNC_(str, len, pad) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - len - any numeric type or any nonnumeric expression that can be implicitly converted - to numeric type. - pad - string type or any type expression that can be implicitly converted - to string type. - - Examples: - > SELECT _FUNC_('hi', 5, '??'); - ???hi - > SELECT _FUNC_('hi', 1, '??'); - h + Arguments: + str - a string expression. + len - a numeric expression that defines the number of pad. + pad - a string expression that represents the string for padding. + + Examples: + > SELECT _FUNC_('hi', 5, '??'); + ???hi + > SELECT _FUNC_('hi', 1, '??'); + h """) case class StringLPad(str: Expression, len: Expression, pad: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -825,21 +782,16 @@ case class StringLPad(str: Expression, len: Expression, pad: Expression) If str is longer than len, the return value is shortened to len characters. """, extended = """ - _FUNC_(str, len, pad) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - len - any numeric type or any nonnumeric expression that can be implicitly converted - to numeric type. - pad - string type or any type expression that can be implicitly converted - to string type. - - Examples: - > SELECT _FUNC_('hi', 5, '??'); - hi??? - > SELECT _FUNC_('hi', 1, '??'); - h + Arguments: + str - a string expression. + len - a numeric expression that defines the number of pad. + pad - a string expression that represents the string for padding. + + Examples: + > SELECT _FUNC_('hi', 5, '??'); + hi??? + > SELECT _FUNC_('hi', 1, '??'); + h """) case class StringRPad(str: Expression, len: Expression, pad: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -876,26 +828,21 @@ object ParseUrl { * Extracts a part from a URL */ @ExpressionDescription( - usage = "_FUNC_(url, partToExtract[, key]) - extracts a part from a URL", + usage = "_FUNC_(url, partToExtract[, key]) - Extracts a part from a URL.", extended = """ - _FUNC_(url, partToExtract[, key]) - - Arguments: - url - string type or any type expression that can be implicitly converted - to string type. - partToExtract - string type or any type expression that can be implicitly converted - to string type. It can be one of HOST, PATH, QUERY, REF, PROTOCOL, AUTHORITY, FILE, - USERINFO. - key - string type or any type expression that can be implicitly converted - to string type. It specifies which query to extract. - - Examples: - > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST') - spark.apache.org - > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY') - query=1 - > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY', 'query') - 1 + Arguments: + url - a string expression. + partToExtract - a string expression. Available values are "HOST", "PATH", "QUERY", "REF", + "PROTOCOL", "AUTHORITY", "FILE", "USERINFO". + key - a string expression that represents which key to extract. + + Examples: + > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST') + spark.apache.org + > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY') + query=1 + > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY', 'query') + 1 """) case class ParseUrl(children: Seq[Expression]) extends Expression with ExpectsInputTypes with CodegenFallback { @@ -1044,15 +991,13 @@ case class ParseUrl(children: Seq[Expression]) @ExpressionDescription( usage = "_FUNC_(strfmt, obj, ...) - Returns a formatted string from printf-style format strings.", extended = """ - _FUNC_(strfmt, obj, ...) - - Arguments: - strfmt - string type or any type expression that can be implicitly converted to string type. - obj - any type expression. + Arguments: + strfmt - a string expression. + obj - an expression of any type. - Examples: - > SELECT _FUNC_("Hello World %d %s", 100, "days"); - Hello World 100 days + Examples: + > SELECT _FUNC_("Hello World %d %s", 100, "days"); + Hello World 100 days """) // scalastyle:on line.size.limit case class FormatString(children: Expression*) extends Expression with ImplicitCastInputTypes { @@ -1129,15 +1074,12 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC All other letters are in lowercase. Words are delimited by white space. """, extended = """ - _FUNC_(str) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. + Arguments: + str - a string expression. - Examples: - > SELECT initcap('sPark sql'); - Spark Sql + Examples: + > SELECT initcap('sPark sql'); + Spark Sql """) case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -1158,17 +1100,13 @@ case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastI @ExpressionDescription( usage = "_FUNC_(str, n) - Returns the string which repeat the given string value n times.", extended = """ - _FUNC_(str, n) + Arguments: + str - a string expression. + n - a numeric expression that defines the number of repetition. - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - n - any numeric type or any nonnumeric expression that can be implicitly converted - to numeric type. - - Examples: - > SELECT _FUNC_('123', 2); - 123123 + Examples: + > SELECT _FUNC_('123', 2); + 123123 """) case class StringRepeat(str: Expression, times: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1195,15 +1133,12 @@ case class StringRepeat(str: Expression, times: Expression) @ExpressionDescription( usage = "_FUNC_(str) - Returns the reversed given string.", extended = """ - _FUNC_(str) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. + Arguments: + str - a string expression. - Examples: - > SELECT _FUNC_('Spark SQL'); - LQS krapS + Examples: + > SELECT _FUNC_('Spark SQL'); + LQS krapS """) case class StringReverse(child: Expression) extends UnaryExpression with String2StringExpression { override def convert(v: UTF8String): UTF8String = v.reverse() @@ -1221,15 +1156,12 @@ case class StringReverse(child: Expression) extends UnaryExpression with String2 @ExpressionDescription( usage = "_FUNC_(n) - Returns a n spaces string.", extended = """ - _FUNC_(n) + Arguments: + n - a numeric expression that defines the number of spaces. - Arguments: - n - any numeric type or any nonnumeric expression that can be implicitly converted - to numeric type. - - Examples: - > SELECT concat(_FUNC_(2), '1'); - 1 + Examples: + > SELECT concat(_FUNC_(2), '1'); + 1 """) case class StringSpace(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -1260,20 +1192,18 @@ case class StringSpace(child: Expression) @ExpressionDescription( usage = "_FUNC_(str, pos[, len]) - Returns the substring of str that starts at pos and is of length len or the slice of byte array that starts at pos and is of length len.", extended = """ - _FUNC_(str, pos[, len]) - - Arguments: - str - string type or any type expression that can be implicitly converted to string type. - pos - any numeric type or any nonnumeric expression that can be implicitly converted to numeric type. - len - any numeric type or any nonnumeric expression that can be implicitly converted to numeric type. - - Examples: - > SELECT _FUNC_('Spark SQL', 5); - k SQL - > SELECT _FUNC_('Spark SQL', -3); - SQL - > SELECT _FUNC_('Spark SQL', 5, 1); - k + Arguments: + str - a string expression. + pos - a numeric expression that represents the position to start slicing. + len - a numeric expression that represents the length to slice. + + Examples: + > SELECT _FUNC_('Spark SQL', 5); + k SQL + > SELECT _FUNC_('Spark SQL', -3); + SQL + > SELECT _FUNC_('Spark SQL', 5, 1); + k """) // scalastyle:on line.size.limit case class Substring(str: Expression, pos: Expression, len: Expression) @@ -1314,19 +1244,14 @@ case class Substring(str: Expression, pos: Expression, len: Expression) * A function that return the length of the given string or binary expression. */ @ExpressionDescription( - usage = "_FUNC_(str | binary) - Returns the length of str or number of bytes in binary data.", + usage = "_FUNC_(expr) - Returns the length of str or number of bytes in binary data.", extended = """ - _FUNC_(str | binary) + Arguments: + expr - a string or binary expression. - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - binary - binary type or any type expression that can be implicitly converted - to binary type. - - Examples: - > SELECT _FUNC_('Spark SQL'); - 9 + Examples: + > SELECT _FUNC_('Spark SQL'); + 9 """) case class Length(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { override def dataType: DataType = IntegerType @@ -1351,17 +1276,13 @@ case class Length(child: Expression) extends UnaryExpression with ImplicitCastIn @ExpressionDescription( usage = "_FUNC_(str1, str2) - Returns the Levenshtein distance between the two given strings.", extended = """ - _FUNC_(str1, str2) - - Arguments: - str1 - string type or any type expression that can be implicitly converted - to string type. - str2 - string type or any type expression that can be implicitly converted - to string type. + Arguments: + str1 - a string expression. + str2 - a string expression. - Examples: - > SELECT _FUNC_('kitten', 'sitting'); - 3 + Examples: + > SELECT _FUNC_('kitten', 'sitting'); + 3 """) case class Levenshtein(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1384,15 +1305,12 @@ case class Levenshtein(left: Expression, right: Expression) extends BinaryExpres @ExpressionDescription( usage = "_FUNC_(str) - Returns soundex code of the string.", extended = """ - _FUNC_(str) + Arguments: + str - a string expression. - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - - Examples: - > SELECT _FUNC_('Miller'); - M460 + Examples: + > SELECT _FUNC_('Miller'); + M460 """) case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputTypes { @@ -1413,17 +1331,14 @@ case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputT @ExpressionDescription( usage = "_FUNC_(str) - Returns the numeric value of the first character of str.", extended = """ - _FUNC_(str) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - - Examples: - > SELECT _FUNC_('222'); - 50 - > SELECT _FUNC_(2); - 50 + Arguments: + str - a string expression. + + Examples: + > SELECT _FUNC_('222'); + 50 + > SELECT _FUNC_(2); + 50 """) case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -1459,15 +1374,12 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp @ExpressionDescription( usage = "_FUNC_(bin) - Converts the argument from binary to a base 64 string.", extended = """ - _FUNC_(bin) - - Arguments: - bin - binary type or any type expression that can be implicitly converted - to binary type. + Arguments: + bin - a binary expression. - Examples: - > SELECT _FUNC_('Spark SQL'); - U3BhcmsgU1FM + Examples: + > SELECT _FUNC_('Spark SQL'); + U3BhcmsgU1FM """) case class Base64(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -1494,15 +1406,12 @@ case class Base64(child: Expression) extends UnaryExpression with ImplicitCastIn @ExpressionDescription( usage = "_FUNC_(str) - Convert the argument from a base 64 string to binary.", extended = """ - _FUNC_(str) + Arguments: + str - a string expression. - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - - Examples: - > SELECT _FUNC_('U3BhcmsgU1FM'); - Spark SQL + Examples: + > SELECT _FUNC_('U3BhcmsgU1FM'); + Spark SQL """) case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCastInputTypes { @@ -1528,14 +1437,10 @@ case class UnBase64(child: Expression) extends UnaryExpression with ImplicitCast @ExpressionDescription( usage = "_FUNC_(bin, str) - Decodes the first argument using the second argument character set.", extended = """ - _FUNC_(bin, str) - - Arguments: - bin - binary type or any type expression that can be implicitly converted - to binary type. - str - string type or any type expression that can be implicitly converted - to string type. It represents the character set which can be one of 'US-ASCII', - 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'. + Arguments: + bin - a binary expression. + str - a string expression that represents the character set which can be one of "US-ASCII", + "ISO-8859-1", "UTF-8", "UTF-16BE", "UTF-16LE", "UTF-16". """) case class Decode(bin: Expression, charset: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1567,18 +1472,16 @@ case class Decode(bin: Expression, charset: Expression) * (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'). * If either argument is null, the result will also be null. */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(str, str) - Encodes the first argument using the second argument character set.", + usage = "_FUNC_(str1, str2) - Encodes the first argument using the second argument character set.", extended = """ - _FUNC_(str, str) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - str - string type or any type expression that can be implicitly converted - to string type. It represents the character set which can be one of 'US-ASCII', - 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16'. + Arguments: + str1 - a string expression. + str2 - a string expression that represents the character set which can be one of "US-ASCII", + "ISO-8859-1", "UTF-8", "UTF-16BE", "UTF-16LE", "UTF-16". """) +// scalastyle:on line.size.limit case class Encode(value: Expression, charset: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -1615,15 +1518,13 @@ case class Encode(value: Expression, charset: Expression) This is supposed to function like MySQL's FORMAT. """, extended = """ - _FUNC_(expr1, expr2) - - Arguments: - expr1 - numeric type expression. - expr2 - integer type expression. + Arguments: + expr1 - numeric type expression. + expr2 - numeric type expression that defines the decimal places to round. - Examples: - > SELECT _FUNC_(12332.123456, 4); - 12,332.1235 + Examples: + > SELECT _FUNC_(12332.123456, 4); + 12,332.1235 """) case class FormatNumber(x: Expression, d: Expression) extends BinaryExpression with ExpectsInputTypes { @@ -1748,19 +1649,16 @@ case class FormatNumber(x: Expression, d: Expression) @ExpressionDescription( usage = "_FUNC_(str[, lang, country]) - Splits str into an array of array of words.", extended = """ - _FUNC_(str[, lang, country]) - - Arguments: - str - string type or any type expression that can be implicitly converted - to string type. - lang - string type or any type expression that can be implicitly converted - to string type. - country - string type or any type expression that can be implicitly converted - to string type. - - Examples: - > SELECT _FUNC_('Hi there! Good morning.'); - [["Hi","there"],["Good","morning"]] + Arguments: + str - a string expression. + lang - a string expression that represents an ISO 639 alpha-2 or alpha-3 language code, + or a language subtag up to 8 characters in length. + country - a string expression that represents an ISO 3166 alpha-2 country code or + a UN M.49 numeric-3 area code. + + Examples: + > SELECT _FUNC_('Hi there! Good morning.'); + [["Hi","there"],["Good","morning"]] """) case class Sentences( str: Expression, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala index e350c2f9773d..b89c1e16a4ff 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala @@ -391,13 +391,10 @@ abstract class OffsetWindowFunction does not have any subsequent row), 'default' is returned. """, extended = """ - _FUNC_(input[, offset[, default]]) - - Arguments: - input - any type expression. - offset - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. Default is 1. - default - any type expression. Defualt is NULL. + Arguments: + input - an expression of any type. + offset - a numeric expression. Default is 1. + default - an expression of any type. Defualt is NULL. """) case class Lead(input: Expression, offset: Expression, default: Expression) extends OffsetWindowFunction { @@ -431,13 +428,10 @@ case class Lead(input: Expression, offset: Expression, default: Expression) does not have any previous row), 'default' is returned. """, extended = """ - _FUNC_(input[, offset[, default]]) - - Arguments: - input - any type expression. - offset - any numeric type or any nonnumeric type expression that can be implicitly - converted to numeric type. Default is 1. - default - any type expression. Defualt is NULL. + Arguments: + input - an expression of any type. + offset - a numeric expression. Default is 1. + default - an expression of any type. Defualt is NULL. """) case class Lag(input: Expression, offset: Expression, default: Expression) extends OffsetWindowFunction { @@ -549,14 +543,12 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction { */ @ExpressionDescription( usage = """ - _FUNC_(expr) - The NTILE(n) function divides the rows for each window partition + _FUNC_(n) - The NTILE(n) function divides the rows for each window partition into 'n' buckets ranging from 1 to at most 'n'. """, extended = """ - _FUNC_(expr) - - Arguments: - expr - integer type expression that allows constant folding. + Arguments: + n - an integer literal. Default is 1. """) case class NTile(buckets: Expression) extends RowNumberLike with SizeBasedWindowFunction { def this() = this(Literal(1)) @@ -680,7 +672,7 @@ abstract class RankLike extends AggregateWindowFunction { */ @ExpressionDescription( usage = """ - _FUNC_() - RANK() computes the rank of a value in a group of values. The result + _FUNC_() - RANK() computes the rank of a value in a group of values. The result is one plus the number of rows preceding or equal to the current row in the ordering of the partition. Tie values will produce gaps in the sequence. """) From d56ac66c6549aa15d628498348dde358a7396ff3 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 18:41:39 +0900 Subject: [PATCH 31/47] Fix tests DDLSuite, SQLQuerySuite and hive/SQLQuerySuite with consistent names --- .../expressions/datetimeExpressions.scala | 72 +++++++++---------- .../expressions/mathExpressions.scala | 36 +++++----- .../org/apache/spark/sql/SQLQuerySuite.scala | 4 +- .../sql/execution/command/DDLSuite.scala | 16 ++--- .../sql/hive/execution/SQLQuerySuite.scala | 4 +- 5 files changed, 63 insertions(+), 69 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 39c44a4fbfd9..51b047844e3b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -145,10 +145,10 @@ case class DateSub(startDate: Expression, days: Expression) } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the hour component of the string/timestamp/interval.", + usage = "_FUNC_(timestamp) - Returns the hour component of the string/timestamp/interval.", extended = """ Arguments: - expr - a timestamp expression. + timestamp - a timestamp expression. Examples: > SELECT _FUNC_('2009-07-30 12:58:59'); @@ -171,10 +171,10 @@ case class Hour(child: Expression) extends UnaryExpression with ImplicitCastInpu } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the minute component of the string/timestamp/interval.", + usage = "_FUNC_(timestamp) - Returns the minute component of the string/timestamp/interval.", extended = """ Arguments: - expr - a timestamp expression. + timestamp - a timestamp expression. Examples: > SELECT _FUNC_('2009-07-30 12:58:59'); @@ -197,10 +197,10 @@ case class Minute(child: Expression) extends UnaryExpression with ImplicitCastIn } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the second component of the string/timestamp/interval.", + usage = "_FUNC_(timestamp) - Returns the second component of the string/timestamp/interval.", extended = """ Arguments: - expr - a timestamp expression. + timestamp - a timestamp expression. Examples: > SELECT _FUNC_('2009-07-30 12:58:59'); @@ -223,10 +223,10 @@ case class Second(child: Expression) extends UnaryExpression with ImplicitCastIn } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the day of year of date/timestamp.", + usage = "_FUNC_(date) - Returns the day of year of date/timestamp.", extended = """ Arguments: - expr - a date expression. + date - a date expression. Examples: > SELECT _FUNC_('2016-04-09'); @@ -249,10 +249,10 @@ case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCas } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the year component of the date/timestamp/interval.", + usage = "_FUNC_(date) - Returns the year component of the date/timestamp/interval.", extended = """ Arguments: - expr - a date expression. + date - a date expression. Examples: > SELECT _FUNC_('2016-07-30'); @@ -275,10 +275,10 @@ case class Year(child: Expression) extends UnaryExpression with ImplicitCastInpu } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the quarter of the year for date, in the range 1 to 4.", + usage = "_FUNC_(date) - Returns the quarter of the year for date, in the range 1 to 4.", extended = """ Arguments: - expr - a date expression. + date - a date expression. Examples: > SELECT _FUNC_('2016-08-31'); @@ -301,10 +301,10 @@ case class Quarter(child: Expression) extends UnaryExpression with ImplicitCastI } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the month component of the date/timestamp/interval.", + usage = "_FUNC_(date) - Returns the month component of the date/timestamp/interval.", extended = """ Arguments: - expr - a date expression. + date - a date expression. Examples: > SELECT _FUNC_('2016-07-30'); @@ -327,10 +327,10 @@ case class Month(child: Expression) extends UnaryExpression with ImplicitCastInp } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the day of month of date/timestamp, or the day of interval.", + usage = "_FUNC_(date) - Returns the day of month of date/timestamp, or the day of interval.", extended = """ Arguments: - expr - a date expression. + date - a date expression. Examples: > SELECT _FUNC_('2009-07-30'); @@ -353,10 +353,10 @@ case class DayOfMonth(child: Expression) extends UnaryExpression with ImplicitCa } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the week of the year of the given date.", + usage = "_FUNC_(date) - Returns the week of the year of the given date.", extended = """ Arguments: - expr - a date expression. + date - a date expression. Examples: > SELECT _FUNC_('2008-02-20'); @@ -400,10 +400,10 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr, fmt) - Converts expr to a value of string in the format specified by the date format fmt.", + usage = "_FUNC_(timestamp, fmt) - Converts expr to a value of string in the format specified by the date format fmt.", extended = """ Arguments: - expr - a timestamp expression. + timestamp - a timestamp expression. fmt - string type expression that represents date/timestamp format used in java.text.SimpleDateFormat. Examples: @@ -705,10 +705,10 @@ case class FromUnixTime(sec: Expression, format: Expression) * Returns the last day of the month which the date belongs to. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the last day of the month which the date belongs to.", + usage = "_FUNC_(date) - Returns the last day of the month which the date belongs to.", extended = """ Arguments: - expr - a date expression. + date - a date expression. Examples: > SELECT _FUNC_('2009-01-12'); @@ -842,10 +842,10 @@ case class TimeAdd(start: Expression, interval: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr, timezone) - Assumes given timestamp is UTC and converts to given timezone.", + usage = "_FUNC_(timestamp, timezone) - Assumes given timestamp is UTC and converts to given timezone.", extended = """ Arguments: - expr - a timestamp expression. + timestamp - a timestamp expression. timezone - a string expression that represents time zone (e.g., 'Asia/Seoul'). Examples: @@ -969,11 +969,11 @@ case class AddMonths(startDate: Expression, numMonths: Expression) * Returns number of months between dates date1 and date2. */ @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns number of months between dates expr1 and expr2.", + usage = "_FUNC_(timestamp1, timestamp2) - Returns number of months between dates expr1 and expr2.", extended = """ Arguments: - expr1 - a timestamp expression. - expr2 - a timestamp expression. + timestamp1 - a timestamp expression. + timestamp2 - a timestamp expression. Examples: > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); @@ -1008,10 +1008,10 @@ case class MonthsBetween(date1: Expression, date2: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr, timezone) - Assumes given timestamp is in given timezone and converts to UTC.", + usage = "_FUNC_(timestamp, timezone) - Assumes given timestamp is in given timezone and converts to UTC.", extended = """ Arguments: - expr - a timestamp expression. + timestamp - a timestamp expression. timezone - a string expression that represents time zone (e.g., 'Asia/Seoul'). Examples: @@ -1068,10 +1068,10 @@ case class ToUTCTimestamp(left: Expression, right: Expression) * Returns the date part of a timestamp or string. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Extracts the date part of the date or datetime expression expr.", + usage = "_FUNC_(date) - Extracts the date part of the date or datetime expression expr.", extended = """ Arguments: - expr - a date expression. + date - a date expression. Examples: > SELECT _FUNC_('2009-07-30 04:17:52'); @@ -1099,10 +1099,10 @@ case class ToDate(child: Expression) extends UnaryExpression with ImplicitCastIn */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr, fmt) - Returns returns date with the time portion of the day truncated to the unit specified by the format model fmt.", + usage = "_FUNC_(date, fmt) - Returns returns date with the time portion of the day truncated to the unit specified by the format model fmt.", extended = """ Arguments: - expr - a date expression. + date - a date expression. fmt - a string expression that represents the truncate level. Available values are "YEAR", "YYYY", "YY", "MON", "MONTH" and "MM". Examples: @@ -1182,11 +1182,11 @@ case class TruncDate(date: Expression, format: Expression) * Returns the number of days from startDate to endDate. */ @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns the number of days between expr1 and expr2.", + usage = "_FUNC_(date1, date2) - Returns the number of days between expr1 and expr2.", extended = """ Arguments: - expr1 - a date expression. - expr2 - a date expression. + date1 - a date expression. + date2 - a date expression. Examples: > SELECT _FUNC_('2009-07-30', '2009-07-31'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index a6194c6cd0af..f629ef3acb00 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -872,11 +872,11 @@ case class Pow(left: Expression, right: Expression) * @param right number of bits to left shift. */ @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Bitwise left shift.", + usage = "_FUNC_(base, expr) - Bitwise left shift.", extended = """ Arguments: - expr1 - a numeric expression. - expr2 - a numeric expression. + base - a numeric expression that defines the base number to shift. + expr - a numeric expression. Examples: > SELECT _FUNC_(2, 1); @@ -909,11 +909,11 @@ case class ShiftLeft(left: Expression, right: Expression) * @param right number of bits to right shift. */ @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Bitwise right shift.", + usage = "_FUNC_(base, expr) - Bitwise right shift.", extended = """ Arguments: - expr1 - a numeric expression. - expr2 - a numeric expression. + base - a numeric expression that defines the base number to shift. + expr - a numeric expression. Examples: > SELECT _FUNC_(4, 1); @@ -946,11 +946,11 @@ case class ShiftRight(left: Expression, right: Expression) * @param right the number of bits to right shift. */ @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Bitwise unsigned right shift.", + usage = "_FUNC_(base, expr) - Bitwise unsigned right shift.", extended = """ Arguments: - expr1 - a numeric expression. - expr2 - a numeric expression. + base - a numeric expression that defines the base number to shift. + expr - a numeric expression. Examples: > SELECT _FUNC_(4, 1); @@ -998,11 +998,11 @@ case class Hypot(left: Expression, right: Expression) * @param right the number to compute the logarithm of. */ @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns the logarithm of expr1 with base expr2.", + usage = "_FUNC_(base, expr) - Returns the logarithm of expr with base.", extended = """ Arguments: - expr1 - a numeric expression. - expr2 - a numeric expression. + base - a numeric expression that represents the logarithm base. + expr - a numeric expression. Examples: > SELECT _FUNC_(10, 100); @@ -1232,11 +1232,11 @@ abstract class RoundBase(child: Expression, scale: Expression, */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Round expr1 to expr2 decimal places using HALF_UP rounding mode.", + usage = "_FUNC_(expr, d) - Round expr to d decimal places using HALF_UP rounding mode.", extended = """ Arguments: - expr1 - a numeric expression. - expr2 - a numeric expression. + expr - a numeric expression. + d - a numeric expression that defines tge decimal places using HALF_UP rounding mode. Examples: > SELECT _FUNC_(2.5, 0); @@ -1256,11 +1256,11 @@ case class Round(child: Expression, scale: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Round expr1 to expr2 decimal places using HALF_EVEN rounding mode.", + usage = "_FUNC_(expr, d) - Round expr to d decimal places using HALF_EVEN rounding mode.", extended = """ Arguments: - expr1 - a numeric expression. - expr2 - a numeric expression. + expr - a numeric expression. + d - a numeric expression that defines decimal places using HALF_EVEN rounding mode. Examples: > SELECT _FUNC_(2.5, 0); diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index a6c1417e2707..c9fa60b4540d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -90,10 +90,8 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { "Class: org.apache.spark.sql.catalyst.expressions.Upper", "Usage: upper(str) - Returns str with all characters changed to uppercase", "Extended Usage:", - "upper(str)", "Arguments:", - "str - string type or any type expression that can be implicitly converted", - "to string type.", + "str - a string expression.", "Examples:", "> SELECT upper('SparkSql');", "SPARKSQL") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 775cf33699aa..7bc6ba6f2c13 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1455,7 +1455,7 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { sql("DESCRIBE FUNCTION log"), Row("Class: org.apache.spark.sql.catalyst.expressions.Logarithm") :: Row("Function: log") :: - Row("Usage: log(expr1, expr2) - Returns the logarithm of expr1 with base expr2.") :: Nil + Row("Usage: log(base, expr) - Returns the logarithm of expr with base.") :: Nil ) // predicate operator checkAnswer( @@ -1498,15 +1498,13 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { Row("Class: org.apache.spark.sql.catalyst.expressions.BitwiseXor") :: Row( """Extended Usage: - | expr1 ^ expr2 + | Arguments: + | expr1 - a numeric expression. + | expr2 - a numeric expression. | - | Arguments: - | expr1 - any intergal numeric type expression. - | expr2 - any intergal numeric type expression. - | - | Examples: - | > SELECT 3 ^ 5; - | 2 + | Examples: + | > SELECT 3 ^ 5; + | 2 | """.stripMargin) :: Row("Function: ^") :: Row("Usage: expr1 ^ expr2 - Bitwise exclusive OR.") :: Nil diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index adac0f564a00..beb3bee5674f 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -254,10 +254,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { "Class: org.apache.spark.sql.catalyst.expressions.Upper", "Usage: upper(str) - Returns str with all characters changed to uppercase", "Extended Usage:", - "upper(str)", "Arguments:", - "str - string type or any type expression that can be implicitly converted", - "to string type.", + "str - a string expression.", "Examples:", "> SELECT upper('SparkSql');", "SPARKSQL") From 8a773d4f8ed0f434c06f628b41f161c2fd842bee Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 18:46:11 +0900 Subject: [PATCH 32/47] Fix scala style. --- .../spark/sql/catalyst/expressions/datetimeExpressions.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 51b047844e3b..54cbc07ef4f4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -968,6 +968,7 @@ case class AddMonths(startDate: Expression, numMonths: Expression) /** * Returns number of months between dates date1 and date2. */ +// scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(timestamp1, timestamp2) - Returns number of months between dates expr1 and expr2.", extended = """ @@ -979,6 +980,7 @@ case class AddMonths(startDate: Expression, numMonths: Expression) > SELECT _FUNC_('1997-02-28 10:30:00', '1996-10-30'); 3.94959677 """) +// scalastyle:on line.size.limit case class MonthsBetween(date1: Expression, date2: Expression) extends BinaryExpression with ImplicitCastInputTypes { From f71e39eb6145cc9f1d3265573bfe245e202d70dc Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Fri, 21 Oct 2016 19:04:12 +0900 Subject: [PATCH 33/47] double-check --- .../catalyst/expressions/aggregate/CentralMomentAgg.scala | 8 ++++---- .../sql/catalyst/expressions/complexTypeCreator.scala | 2 +- .../spark/sql/catalyst/expressions/generators.scala | 4 ++-- .../spark/sql/catalyst/expressions/mathExpressions.scala | 2 ++ .../spark/sql/catalyst/expressions/nullExpressions.scala | 6 ++++++ 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala index f1f3c93bc1f3..d1e543ebda81 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala @@ -177,7 +177,7 @@ case class StddevSamp(child: Expression) extends CentralMomentAgg(child) { usage = "_FUNC_(expr) - Returns the population variance calculated from values of a group.", extended = """ Arguments: - expr - a numeric expression. + expr - a numeric expression. """) case class VariancePop(child: Expression) extends CentralMomentAgg(child) { @@ -196,7 +196,7 @@ case class VariancePop(child: Expression) extends CentralMomentAgg(child) { usage = "_FUNC_(expr) - Returns the sample variance calculated from values of a group.", extended = """ Arguments: - expr - a numeric expression. + expr - a numeric expression. """) case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) { @@ -215,7 +215,7 @@ case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) { usage = "_FUNC_(expr) - Returns the skewness value calculated from values of a group.", extended = """ Arguments: - expr - a numeric expression. + expr - a numeric expression. """) case class Skewness(expr: Expression) extends CentralMomentAgg(expr) { @@ -234,7 +234,7 @@ case class Skewness(expr: Expression) extends CentralMomentAgg(expr) { usage = "_FUNC_(expr) - Returns the kurtosis value calculated from values of a group.", extended = """ Arguments: - expr - a numeric expression. + expr - a numeric expression. """) case class Kurtosis(child: Expression) extends CentralMomentAgg(child) { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index 296f999617a5..0ff41940b255 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -437,7 +437,7 @@ case class CreateNamedStructUnsafe(children: Seq[Expression]) extends Expression extended = """ Arguments: text - a string expression that represents data to convert into map. - pairDelim - a string literal that defines the delimiter to separate each pair.For example, ",". + pairDelim - a string literal that defines the delimiter to separate each pair. For example, ",". keyValueDelim - a string literal that defines the delimiter to separate key and value in the pair. For example, ":". Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index 023142200683..f8ab824f8998 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -102,11 +102,11 @@ case class UserDefinedGenerator( * }}} */ @ExpressionDescription( - usage = "_FUNC_(n, v1, ..., vk) - Separate v1, ..., vk into n rows.", + usage = "_FUNC_(n, expr1, ..., exprk) - Separate expr1, ..., exprk into n rows.", extended = """ Arguments: n - a integer literal that represents the number of output rows. - v - an expression of any type. + expr - an expression of any type. Examples: > SELECT _FUNC_(2, 1, 2, 3); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index f629ef3acb00..74df0e107254 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -902,6 +902,7 @@ case class ShiftLeft(left: Expression, right: Expression) } } + /** * Bitwise right shift. * @@ -939,6 +940,7 @@ case class ShiftRight(left: Expression, right: Expression) } } + /** * Bitwise unsigned right shift, for integer and long data type. * diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala index 84a4c2527058..37f39cd5cdfd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala @@ -96,6 +96,7 @@ case class Coalesce(children: Seq[Expression]) extends Expression { } } + @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns expr2 if expr1 is null, or expr1 otherwise.", extended = """ @@ -123,6 +124,7 @@ case class IfNull(left: Expression, right: Expression) extends RuntimeReplaceabl } } + @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns null if expr1 equals to expr2, or expr1 otherwise.", extended = """ @@ -152,6 +154,7 @@ case class NullIf(left: Expression, right: Expression) extends RuntimeReplaceabl } } + @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns expr2 if expr1 is null, or expr1 otherwise.", extended = """ @@ -179,6 +182,7 @@ case class Nvl(left: Expression, right: Expression) extends RuntimeReplaceable { } } + // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(expr1, expr2, expr3) - Returns expr2 if expr1 is not null, or expr3 otherwise.", @@ -211,6 +215,7 @@ case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression) } } + /** * Evaluates to `true` iff it's NaN. */ @@ -256,6 +261,7 @@ case class IsNaN(child: Expression) extends UnaryExpression } } + /** * An Expression evaluates to `left` iff it's not NaN, or evaluates to `right` otherwise. * This Expression is useful for mapping NaN values to null. From caa71c89d67d7168a527c953c74be7bc5c4da27f Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sat, 22 Oct 2016 19:09:21 +0900 Subject: [PATCH 34/47] double-check again --- .../expressions/CallMethodViaReflection.scala | 4 +- .../spark/sql/catalyst/expressions/Cast.scala | 4 +- .../aggregate/ApproximatePercentile.scala | 6 +- .../expressions/aggregate/First.scala | 2 +- .../catalyst/expressions/aggregate/Last.scala | 2 +- .../expressions/complexTypeCreator.scala | 2 +- .../expressions/conditionalExpressions.scala | 18 ++- .../expressions/datetimeExpressions.scala | 20 ++-- .../expressions/regexpExpressions.scala | 12 +- .../sql/catalyst/expressions/xml/xpath.scala | 112 ++++++++---------- 10 files changed, 83 insertions(+), 99 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala index d6f289ac90c2..c2d49d4fb058 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala @@ -47,8 +47,8 @@ import org.apache.spark.util.Utils usage = "_FUNC_(class, method[, arg1[, arg2 ..]]) - Calls method with reflection.", extended = """ Arguments: - class - a string literal that represents full-qualified class name. - method - a string literal that represents method name. + class - a string literal that represents a fully-qualified class name. + method - a string literal that represents a method name. arg - a string literal that represents arguments for the method. Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 7e849afe4c5e..77f858e6f85c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -114,11 +114,11 @@ object Cast { /** Cast the child expression to the target data type. */ @ExpressionDescription( - usage = "_FUNC_(expr AS type) - Casts value expr to the target data type.", + usage = "_FUNC_(expr AS type) - Casts the value expr to the target data type.", extended = """ Arguments: expr - an expression of any type. - type - data type. + type - data type to cast expr into. Examples: > SELECT _FUNC_('10' as int); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala index 926e457136f5..dba488f44b54 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala @@ -56,13 +56,15 @@ import org.apache.spark.sql.types._ controls approximation accuracy at the cost of memory. Higher value of `accuracy` yields better accuracy, `1.0/accuracy` is the relative error of the approximation. When percentage is an array, each value of the percentage array must be between 0.0 and 1.0. + In this case, returns the approximate percentile array of column `col` at the given + percentage array. """, extended = """ Arguments: col - a numeric expression. percentage - a numeric literal or an array literal of numeric type that defines the - percentile. For example, 0.5 means 50-percentile. - accuracy - a numeric literal. + percentile between 0.0 and 1.0. For example, 0.5 means 50-percentile. + accuracy - a numeric literal of approximation accuracy. Examples: > SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala index 44176d2497e5..779e8f5c7281 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.types._ */ @ExpressionDescription( usage = """ - _FUNC_(expr[, isIgnoreNull]) - Returns the first value of expr for a group of rows. + _FUNC_(expr[, isIgnoreNull]) - Returns the first value of `expr` for a group of rows. If isIgnoreNull is true, returns only non-null values. """, extended = """ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala index 010a7a1cb45a..ae34d06145a5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.types._ */ @ExpressionDescription( usage = """ - _FUNC_(expr[, isIgnoreNull]) - Returns the last value of expr for a group of rows. + _FUNC_(expr[, isIgnoreNull]) - Returns the last value of `expr` for a group of rows. If isIgnoreNull is true, returns only non-null values. """, extended = """ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index 0ff41940b255..18cccb51f6e5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -438,7 +438,7 @@ case class CreateNamedStructUnsafe(children: Seq[Expression]) extends Expression Arguments: text - a string expression that represents data to convert into map. pairDelim - a string literal that defines the delimiter to separate each pair. For example, ",". - keyValueDelim - a string literal that defines the delimiter to separate key and value in the pair. For example, ":". + keyValueDelim - a string literal that defines the delimiter to separate the key and value in the pair. For example, ":". Examples: > SELECT _FUNC_('a:1,b:2,c:3', ',', ':'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala index 87afe6ee2a90..25dab123ef51 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala @@ -26,16 +26,14 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = "_FUNC_(expr1, expr2, expr3) - If expr1 is TRUE then IF() returns expr2; otherwise it returns expr3.", extended = """ - _FUNC_(expr1, expr2, expr3) - - Arguments: - expr1 - a boolean expression. - expr2 - an expression of any type that represents the return value when expr1 is TRUE. - expr3 - an expression of any type that represents the return value when expr1 is FALSE. - - Examples: - > SELECT _FUNC_(1 < 2, 'a', 'b'); - a + Arguments: + expr1 - a boolean expression. + expr2 - an expression of any type that represents the return value when expr1 is TRUE. + expr3 - an expression of any type that represents the return value when expr1 is FALSE. + + Examples: + > SELECT _FUNC_(1 < 2, 'a', 'b'); + a """) // scalastyle:on line.size.limit case class If(predicate: Expression, trueValue: Expression, falseValue: Expression) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 54cbc07ef4f4..e285e81e98df 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -400,11 +400,11 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp, fmt) - Converts expr to a value of string in the format specified by the date format fmt.", + usage = "_FUNC_(timestamp, fmt) - Converts timestamp to a value of string in the format specified by the date format fmt.", extended = """ Arguments: timestamp - a timestamp expression. - fmt - string type expression that represents date/timestamp format used in java.text.SimpleDateFormat. + fmt - string type expression that represents the date/timestamp format available in java.text.SimpleDateFormat. Examples: > SELECT _FUNC_('2016-04-08', 'y'); @@ -443,7 +443,7 @@ case class DateFormatClass(left: Expression, right: Expression) extends BinaryEx extended = """ Arguments: expr - a date, string or timestamp expression. - pattern - a string expression that represents date/timestamp format used in + pattern - a string expression that represents the date/timestamp format available in java.text.SimpleDateFormat. Examples: @@ -476,7 +476,7 @@ case class ToUnixTimestamp(timeExp: Expression, format: Expression) extends Unix extended = """ Arguments: expr - a date, string or timestamp expression. - pattern - a string expression that represents date/timestamp format used in + pattern - a string expression that represents the date/timestamp format available in java.text.SimpleDateFormat. Examples: @@ -613,7 +613,7 @@ abstract class UnixTime extends BinaryExpression with ExpectsInputTypes { extended = """ Arguments: unix_time - a numeric expression. - format - a string expression that represents date/timestamp format used in + format - a string expression that represents the date/timestamp format available in java.text.SimpleDateFormat. Examples: @@ -846,7 +846,7 @@ case class TimeAdd(start: Expression, interval: Expression) extended = """ Arguments: timestamp - a timestamp expression. - timezone - a string expression that represents time zone (e.g., 'Asia/Seoul'). + timezone - a string expression that represents time zone. For example, 'Asia/Seoul'. Examples: > SELECT from_utc_timestamp('2016-08-31', 'Asia/Seoul'); @@ -970,7 +970,7 @@ case class AddMonths(startDate: Expression, numMonths: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp1, timestamp2) - Returns number of months between dates expr1 and expr2.", + usage = "_FUNC_(timestamp1, timestamp2) - Returns number of months between dates timestamp1 and timestamp2.", extended = """ Arguments: timestamp1 - a timestamp expression. @@ -1014,7 +1014,7 @@ case class MonthsBetween(date1: Expression, date2: Expression) extended = """ Arguments: timestamp - a timestamp expression. - timezone - a string expression that represents time zone (e.g., 'Asia/Seoul'). + timezone - a string expression that represents time zone. For example, 'Asia/Seoul'. Examples: > SELECT _FUNC_('2016-08-31', 'Asia/Seoul'); @@ -1070,7 +1070,7 @@ case class ToUTCTimestamp(left: Expression, right: Expression) * Returns the date part of a timestamp or string. */ @ExpressionDescription( - usage = "_FUNC_(date) - Extracts the date part of the date or datetime expression expr.", + usage = "_FUNC_(date) - Extracts the date part of the date or timestamp expression.", extended = """ Arguments: date - a date expression. @@ -1184,7 +1184,7 @@ case class TruncDate(date: Expression, format: Expression) * Returns the number of days from startDate to endDate. */ @ExpressionDescription( - usage = "_FUNC_(date1, date2) - Returns the number of days between expr1 and expr2.", + usage = "_FUNC_(date1, date2) - Returns the number of days between date1 and date2.", extended = """ Arguments: date1 - a date expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 4e235b692e7d..05962f9a0beb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -72,7 +72,7 @@ trait StringRegexExpression extends ImplicitCastInputTypes { extended = """ Arguments: str - a string expression. - pattern - a string expression. + pattern - a string expression that defines a regular expression pattern. """) case class Like(left: Expression, right: Expression) extends BinaryExpression with StringRegexExpression { @@ -130,7 +130,7 @@ case class Like(left: Expression, right: Expression) extended = """ Arguments: str - a string expression. - regexp - a string expression. + regexp - a string expression that defines a regular expression pattern. """) case class RLike(left: Expression, right: Expression) extends BinaryExpression with StringRegexExpression { @@ -189,7 +189,7 @@ case class RLike(left: Expression, right: Expression) extended = """ Arguments: str - a string expression. - regex - a string expression. + regex - a string expression that defines a regular expression pattern. Examples: > SELECT _FUNC_('oneAtwoBthreeC', '[ABC]'); @@ -229,8 +229,8 @@ case class StringSplit(str: Expression, pattern: Expression) extended = """ Arguments: str - a string expression. - regexp - a string expression. - rep - a string expression. + regexp - a string expression that defines a regular expression pattern. + rep - a string expression that defines the replacement. Examples: > SELECT _FUNC_('100-200', '(\d+)', 'num'); @@ -340,7 +340,7 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio extended = """ Arguments: str - a string expression. - regexp - a string expression. + regexp - a string expression that defines a regular expression pattern. idx - a numeric expression that defines the index of a capturing group in regexp. Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index 950f4dbc9465..551fefa23cdb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -58,15 +58,13 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Evaluates a boolean XPath expression.", extended = """ - _FUNC_(xml, xpath) + Arguments: + xml - a string expression that represents XML document. + xpath - a string literal that represents XPath expression. - Arguments: - xml - string type expression that represents XML document. - xpath - string type literal that represents XPath expression. - - Examples: - > SELECT _FUNC_('1','a/b'); - true + Examples: + > SELECT _FUNC_('1','a/b'); + true """) case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract { @@ -81,15 +79,13 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a short value that matches the XPath expression.", extended = """ - _FUNC_(xml, xpath) - - Arguments: - xml - string type expression that represents XML document. - xpath - string type literal that represents XPath expression. + Arguments: + xml - a string expression that represents XML document. + xpath - a string literal that represents XPath expression. - Examples: - > SELECT _FUNC_('12', 'sum(a/b)'); - 3 + Examples: + > SELECT _FUNC_('12', 'sum(a/b)'); + 3 """) case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_int" @@ -104,15 +100,13 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns an integer value that matches the XPath expression.", extended = """ - _FUNC_(xml, xpath) + Arguments: + xml - a string expression that represents XML document. + xpath - a string literal that represents XPath expression. - Arguments: - xml - string type expression that represents XML document. - xpath - string type literal that represents XPath expression. - - Examples: - > SELECT _FUNC_('12', 'sum(a/b)'); - 3 + Examples: + > SELECT _FUNC_('12', 'sum(a/b)'); + 3 """) case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_int" @@ -127,15 +121,13 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a long value that matches the XPath expression.", extended = """ - _FUNC_(xml, xpath) - - Arguments: - xml - string type expression that represents XML document. - xpath - string type literal that represents XPath expression. + Arguments: + xml - a string expression that represents XML document. + xpath - a string literal that represents XPath expression. - Examples: - > SELECT _FUNC_('12', 'sum(a/b)'); - 3 + Examples: + > SELECT _FUNC_('12', 'sum(a/b)'); + 3 """) case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_long" @@ -150,15 +142,13 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a float value that matches the XPath expression.", extended = """ - _FUNC_(xml, xpath) + Arguments: + xml - a string expression that represents XML document. + xpath - a string literal that represents XPath expression. - Arguments: - xml - string type expression that represents XML document. - xpath - string type literal that represents XPath expression. - - Examples: - > SELECT _FUNC_('12', 'sum(a/b)'); - 3.0 + Examples: + > SELECT _FUNC_('12', 'sum(a/b)'); + 3.0 """) case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_float" @@ -173,15 +163,13 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a double value that matches the XPath expression.", extended = """ - _FUNC_(xml, xpath) - - Arguments: - xml - string type expression that represents XML document. - xpath - string type literal that represents XPath expression. + Arguments: + xml - a string expression that represents XML document. + xpath - a string literal that represents XPath expression. - Examples: - > SELECT _FUNC_('12', 'sum(a/b)'); - 3.0 + Examples: + > SELECT _FUNC_('12', 'sum(a/b)'); + 3.0 """) case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_float" @@ -197,15 +185,13 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the XPath expression.", extended = """ - _FUNC_(xml, xpath) + Arguments: + xml - a string expression that represents XML document. + path - a string literal that represents XPath expression. - Arguments: - xml - string type expression that represents XML document. - path - string type literal that represents XPath expression. - - Examples: - > SELECT _FUNC_('bcc','a/c'); - cc + Examples: + > SELECT _FUNC_('bcc','a/c'); + cc """) // scalastyle:on line.size.limit case class XPathString(xml: Expression, path: Expression) extends XPathExtract { @@ -222,15 +208,13 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract { @ExpressionDescription( usage = "_FUNC_(xml, xpath) - Returns a string array of values within xml nodes that match the XPath expression", extended = """ - _FUNC_(xml, xpath) - - Arguments: - xml - string type expression that represents XML document. - path - string type literal that represents XPath expression. + Arguments: + xml - a string expression that represents XML document. + path - a string literal that represents XPath expression. - Examples: - > SELECT _FUNC_('b1b2b3c1c2','a/b/text()'); - ['b1','b2','b3'] + Examples: + > SELECT _FUNC_('b1b2b3c1c2','a/b/text()'); + ['b1','b2','b3'] """) // scalastyle:on line.size.limit case class XPathList(xml: Expression, path: Expression) extends XPathExtract { From 5163a871929268865f321de8b861911ac7b0249c Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sat, 22 Oct 2016 21:24:29 +0900 Subject: [PATCH 35/47] Address Sean's comments --- .../sql/catalyst/expressions/arithmetic.scala | 19 +++++-- .../expressions/bitwiseExpressions.scala | 14 +++--- .../expressions/datetimeExpressions.scala | 10 ++-- .../expressions/mathExpressions.scala | 50 ++++++++++++------- .../spark/sql/catalyst/expressions/misc.scala | 2 +- .../sql/catalyst/expressions/predicates.scala | 6 +-- .../expressions/stringExpressions.scala | 8 +-- .../sql/catalyst/expressions/xml/xpath.scala | 2 +- 8 files changed, 68 insertions(+), 43 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 70f750b07461..6741df7baee1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.CalendarInterval @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the negative value of expr.", + usage = "_FUNC_(expr) - Returns the negated value of expr.", extended = """ Arguments: expr - a numeric or interval expression. @@ -66,7 +66,7 @@ case class UnaryMinus(child: Expression) extends UnaryExpression } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the positive value of expr.", + usage = "_FUNC_(expr) - Returns the value of expr.", extended = """ Arguments: expr - a numeric or interval expression. @@ -219,7 +219,7 @@ case class Subtract(left: Expression, right: Expression) } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Multiplies expr1 by expr2.", + usage = "expr1 _FUNC_ expr2 - Returns expr1*expr2.", extended = """ Arguments: expr1 - a numeric expression. @@ -239,7 +239,7 @@ case class Multiply(left: Expression, right: Expression) } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Divides expr1 by expr2.", + usage = "expr1 _FUNC_ expr2 - Returns expr1/expr2. It always performs floating point division.", extended = """ Arguments: expr1 - a numeric expression. @@ -248,6 +248,9 @@ case class Multiply(left: Expression, right: Expression) Examples: > SELECT 3 _FUNC_ 2; 1.5 + Examples: + > SELECT 2L _FUNC_ 2L; + 1.0 """) case class Divide(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -324,11 +327,15 @@ case class Divide(left: Expression, right: Expression) } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns the remainder when dividing expr1 by expr2.", + usage = "expr1 _FUNC_ expr2 - Returns the remainder after expr1/expr2.", extended = """ Arguments: expr1 - a numeric expression. expr2 - a numeric expression. + + Examples: + > SELECT 2 _FUNC_ 1.8; + 0.2 """) case class Remainder(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -419,6 +426,8 @@ case class Remainder(left: Expression, right: Expression) Examples: > SELECT _FUNC_(10, 3); 1 + > SELECT _FUNC_(-10, 3); + 2 """) case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index 757d6e18c8f8..4e5e51b7e651 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -30,8 +30,8 @@ import org.apache.spark.sql.types._ usage = "expr1 _FUNC_ expr2 - Bitwise AND.", extended = """ Arguments: - expr1 - a numeric expression. - expr2 - a numeric expression. + expr1 - a integral numeric expression. + expr2 - a integral numeric expression. Examples: > SELECT 3 _FUNC_ 5; @@ -66,8 +66,8 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme usage = "expr1 _FUNC_ expr2 - Bitwise OR.", extended = """ Arguments: - expr1 - a numeric expression. - expr2 - a numeric expression. + expr1 - a integral numeric expression. + expr2 - a integral numeric expression. Examples: > SELECT 3 _FUNC_ 5; @@ -102,8 +102,8 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet usage = "expr1 _FUNC_ expr2 - Bitwise exclusive OR.", extended = """ Arguments: - expr1 - a numeric expression. - expr2 - a numeric expression. + expr1 - a integral numeric expression. + expr2 - a integral numeric expression. Examples: > SELECT 3 _FUNC_ 5; @@ -136,7 +136,7 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme usage = "_FUNC_ expr - Bitwise NOT.", extended = """ Arguments: - expr - a numeric expression. + expr - a integral numeric expression. Examples: > SELECT _FUNC_ 0; diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index e285e81e98df..4aa0ed94bcfb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -838,11 +838,12 @@ case class TimeAdd(start: Expression, interval: Expression) } /** - * Assumes given timestamp is UTC and converts to given timezone. + * Given a timestamp, which corresponds to a certain time of day in UTC, returns another timestamp + * that corresponds to the same time of day in the given timezone. */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp, timezone) - Assumes given timestamp is UTC and converts to given timezone.", + usage = "_FUNC_(timestamp, timezone) - Given a timestamp, which corresponds to a certain time of day in UTC, returns another timestamp that corresponds to the same time of day in the given timezone.", extended = """ Arguments: timestamp - a timestamp expression. @@ -1006,11 +1007,12 @@ case class MonthsBetween(date1: Expression, date2: Expression) } /** - * Assumes given timestamp is in given timezone and converts to UTC. + * Given a timestamp, which corresponds to a certain time of day in the given timezone, returns + * another timestamp that corresponds to the same time of day in UTC. */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp, timezone) - Assumes given timestamp is in given timezone and converts to UTC.", + usage = "_FUNC_(timestamp, timezone) - Given a timestamp, which corresponds to a certain time of day in the given timezone, returns another timestamp that corresponds to the same time of day in UTC.", extended = """ Arguments: timestamp - a timestamp expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 74df0e107254..451df73c5617 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -139,7 +139,7 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String) * evaluated by the optimizer during constant folding. */ @ExpressionDescription( - usage = "_FUNC_() - Returns Euler's number, E.", + usage = "_FUNC_() - Returns Euler's number, e.", extended = """ Examples: > SELECT _FUNC_(); @@ -152,7 +152,7 @@ case class EulerNumber() extends LeafMathExpression(math.E, "E") * evaluated by the optimizer during constant folding. */ @ExpressionDescription( - usage = "_FUNC_() - Returns PI.", + usage = "_FUNC_() - Returns pi.", extended = """ Examples: > SELECT _FUNC_(); @@ -166,8 +166,9 @@ case class Pi() extends LeafMathExpression(math.Pi, "PI") //////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////// +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the arc cosine of expr if -1<=expr<=1 or NaN otherwise.", + usage = "_FUNC_(expr) - Returns the inverse cosine (a.k.a. arccosine) of expr if -1<=expr<=1 or NaN otherwise.", extended = """ Arguments: expr - a numeric expression. @@ -178,10 +179,12 @@ case class Pi() extends LeafMathExpression(math.Pi, "PI") > SELECT _FUNC_(2); NaN """) +// scalastyle:on line.size.limit case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS") +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the arc sin of expr if -1<=expr<=1 or NaN otherwise.", + usage = "_FUNC_(expr) - Returns the inverse sine (a.k.a. arcsine) the arc sin of expr if -1<=expr<=1 or NaN otherwise.", extended = """ Arguments: expr - a numeric expression. @@ -192,10 +195,12 @@ case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS" > SELECT _FUNC_(2); NaN """) +// scalastyle:on line.size.limit case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN") +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the arc tangent.", + usage = "_FUNC_(expr) - Returns the inverse tangent (a.k.a. arctangent).", extended = """ Arguments: expr - a numeric expression. @@ -204,10 +209,11 @@ case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN" > SELECT _FUNC_(0); 0.0 """) +// scalastyle:on line.size.limit case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the cube root of a double value.", + usage = "_FUNC_(expr) - Returns the cube root of expr.", extended = """ Arguments: expr - a numeric expression. @@ -465,7 +471,7 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the natural logarithm of x with base expr.", + usage = "_FUNC_(expr) - Returns the natural logarithm (base e) of expr.", extended = """ Arguments: expr - a numeric expression. @@ -544,7 +550,7 @@ case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the sign of expr.", + usage = "_FUNC_(expr) - Returns -1.0, 0.0 or 1.0 as expr is negative, 0 or positive.", extended = """ Arguments: expr - a numeric expression. @@ -643,8 +649,9 @@ case class ToRadians(child: Expression) extends UnaryMathExpression(math.toRadia override def funcName: String = "toRadians" } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr) - Returns expr in binary.", + usage = "_FUNC_(expr) - Returns the string representation of the long value `expr` represented in binary.", extended = """ Arguments: expr - a numeric expression. @@ -652,7 +659,12 @@ case class ToRadians(child: Expression) extends UnaryMathExpression(math.toRadia Examples: > SELECT _FUNC_(13); 1101 + > SELECT _FUNC_(-13); + 1111111111111111111111111111111111111111111111111111111111110011 + > SELECT _FUNC_(13.3); + 1101 """) +// scalastyle:on line.size.limit case class Bin(child: Expression) extends UnaryExpression with Serializable with ImplicitCastInputTypes { @@ -822,8 +834,9 @@ case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInp //////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////// +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns the arc tangent2.", + usage = "_FUNC_(expr1, expr2) - Returns the angle in radians between the positive x-axis of a plane and the point given by the coordinates (expr1, expr2).", extended = """ Arguments: expr1 - a numeric expression. @@ -833,6 +846,7 @@ case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInp > SELECT _FUNC_(0, 0); 0.0 """) +// scalastyle:on line.size.limit case class Atan2(left: Expression, right: Expression) extends BinaryMathExpression(math.atan2, "ATAN2") { @@ -875,8 +889,8 @@ case class Pow(left: Expression, right: Expression) usage = "_FUNC_(base, expr) - Bitwise left shift.", extended = """ Arguments: - base - a numeric expression that defines the base number to shift. - expr - a numeric expression. + base - a integral numeric expression that defines the base number to shift. + expr - a integral numeric expression. Examples: > SELECT _FUNC_(2, 1); @@ -904,17 +918,17 @@ case class ShiftLeft(left: Expression, right: Expression) /** - * Bitwise right shift. + * Bitwise (signed) right shift. * * @param left the base number to shift. * @param right number of bits to right shift. */ @ExpressionDescription( - usage = "_FUNC_(base, expr) - Bitwise right shift.", + usage = "_FUNC_(base, expr) - Bitwise (signed) right shift.", extended = """ Arguments: - base - a numeric expression that defines the base number to shift. - expr - a numeric expression. + base - a integral numeric expression that defines the base number to shift. + expr - a integral numeric expression. Examples: > SELECT _FUNC_(4, 1); @@ -951,8 +965,8 @@ case class ShiftRight(left: Expression, right: Expression) usage = "_FUNC_(base, expr) - Bitwise unsigned right shift.", extended = """ Arguments: - base - a numeric expression that defines the base number to shift. - expr - a numeric expression. + base - a integral numeric expression that defines the base number to shift. + expr - a integral numeric expression. Examples: > SELECT _FUNC_(4, 1); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala index a86581cd6e41..0cb96f0059de 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala @@ -183,7 +183,7 @@ case class Sha1(child: Expression) extends UnaryExpression with ImplicitCastInpu * For input of type [[BinaryType]] */ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns a cyclic redundancy check value as a bigint of the expr.", + usage = "_FUNC_(expr) - Returns a cyclic redundancy check value of the expr as a bigint.", extended = """ Arguments: expr - a binary expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index 559cb3233911..1ec2bbff411d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -459,7 +459,7 @@ case class EqualTo(left: Expression, right: Expression) @ExpressionDescription( usage = """ - expr1 _FUNC_ expr2 - Returns same result with EQUAL(=) operator for non-null operands. + expr1 _FUNC_ expr2 - Returns same result as the EQUAL(=) operator for non-null operands. but returns TRUE if both are NULL, FALSE if one of the them is NULL. """, extended = """ @@ -525,7 +525,7 @@ case class LessThan(left: Expression, right: Expression) } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is not greater than expr2.", + usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is less than or equal to expr2.", extended = """ Arguments: expr1 - an expression of any type. @@ -563,7 +563,7 @@ case class GreaterThan(left: Expression, right: Expression) } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is not smaller than expr2.", + usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is greater than or equal to expr2.", extended = """ Arguments: expr1 - an expression of any type. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 37a697c0f7dc..06b366cb1324 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -1151,10 +1151,10 @@ case class StringReverse(child: Expression) extends UnaryExpression with String2 } /** - * Returns a n spaces string. + * Returns a string consisting of n spaces. */ @ExpressionDescription( - usage = "_FUNC_(n) - Returns a n spaces string.", + usage = "_FUNC_(n) - Returns a string consisting of n spaces.", extended = """ Arguments: n - a numeric expression that defines the number of spaces. @@ -1300,10 +1300,10 @@ case class Levenshtein(left: Expression, right: Expression) extends BinaryExpres } /** - * A function that return soundex code of the given string expression. + * A function that return Soundex code of the given string expression. */ @ExpressionDescription( - usage = "_FUNC_(str) - Returns soundex code of the string.", + usage = "_FUNC_(str) - Returns Soundex code of the string.", extended = """ Arguments: str - a string expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index 551fefa23cdb..d0117da0770d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -206,7 +206,7 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a string array of values within xml nodes that match the XPath expression", + usage = "_FUNC_(xml, xpath) - Returns a string array of values within the nodes of xml that match the XPath expression.", extended = """ Arguments: xml - a string expression that represents XML document. From f8d11aa6f3c261e384500a36d1b6faefa1b6d985 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sat, 22 Oct 2016 21:43:00 +0900 Subject: [PATCH 36/47] another double-check --- .../catalyst/expressions/aggregate/Covariance.scala | 2 ++ .../expressions/aggregate/HyperLogLogPlusPlus.scala | 1 + .../spark/sql/catalyst/expressions/arithmetic.scala | 12 ++++++++++++ 3 files changed, 15 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala index b54a185c3c54..cea86fa9c8ae 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala @@ -76,6 +76,7 @@ abstract class Covariance(x: Expression, y: Expression) extends DeclarativeAggre } } + @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the population covariance of a set of number pairs.", extended = """ @@ -91,6 +92,7 @@ case class CovPopulation(left: Expression, right: Expression) extends Covariance override def prettyName: String = "covar_pop" } + @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the sample covariance of a set of number pairs.", extended = """ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala index 2670b8e6bec1..c17255343565 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala @@ -55,6 +55,7 @@ import org.apache.spark.sql.types._ Arguments: expr - an expression of any type that represents data to count. relativeSD - a numeric literal that defines the maximum estimation error allowed. + Default is 0.05. """) case class HyperLogLogPlusPlus( child: Expression, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 6741df7baee1..d42c80a988dc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -151,6 +151,10 @@ object BinaryArithmetic { Arguments: expr1 - a numeric expression. expr2 - a numeric expression. + + Examples: + > SELECT 1 _FUNC_ 2; + 3 """) case class Add(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -187,6 +191,10 @@ case class Add(left: Expression, right: Expression) extends BinaryArithmetic wit Arguments: expr1 - a numeric expression. expr2 - a numeric expression. + + Examples: + > SELECT 2 _FUNC_ 1; + 1 """) case class Subtract(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -224,6 +232,10 @@ case class Subtract(left: Expression, right: Expression) Arguments: expr1 - a numeric expression. expr2 - a numeric expression. + + Examples: + > SELECT 2 _FUNC_ 3; + 6 """) case class Multiply(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { From da40f850151b16044ad068e8e03dce7b11909cf5 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sat, 22 Oct 2016 21:54:55 +0900 Subject: [PATCH 37/47] Fix the tests in DDLSuite --- .../org/apache/spark/sql/execution/command/DDLSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 7bc6ba6f2c13..883537325765 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1499,8 +1499,8 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { Row( """Extended Usage: | Arguments: - | expr1 - a numeric expression. - | expr2 - a numeric expression. + | expr1 - a integral numeric expression. + | expr2 - a integral numeric expression. | | Examples: | > SELECT 3 ^ 5; From feafdc21317b4d05921b73f6231d8b0d498bad43 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 23 Oct 2016 18:28:56 +0900 Subject: [PATCH 38/47] another double-check --- .../expressions/CallMethodViaReflection.scala | 2 +- .../catalyst/expressions/bitwiseExpressions.scala | 14 +++++++------- .../sql/catalyst/expressions/mathExpressions.scala | 12 ++++++------ .../spark/sql/execution/command/DDLSuite.scala | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala index c2d49d4fb058..8320ffaaa91e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala @@ -49,7 +49,7 @@ import org.apache.spark.util.Utils Arguments: class - a string literal that represents a fully-qualified class name. method - a string literal that represents a method name. - arg - a string literal that represents arguments for the method. + arg - an expression of any type that represents arguments for the method. Examples: > SELECT _FUNC_('java.util.UUID', 'randomUUID'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index 4e5e51b7e651..278449906c29 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -30,8 +30,8 @@ import org.apache.spark.sql.types._ usage = "expr1 _FUNC_ expr2 - Bitwise AND.", extended = """ Arguments: - expr1 - a integral numeric expression. - expr2 - a integral numeric expression. + expr1 - an integral numeric expression. + expr2 - an integral numeric expression. Examples: > SELECT 3 _FUNC_ 5; @@ -66,8 +66,8 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme usage = "expr1 _FUNC_ expr2 - Bitwise OR.", extended = """ Arguments: - expr1 - a integral numeric expression. - expr2 - a integral numeric expression. + expr1 - an integral numeric expression. + expr2 - an integral numeric expression. Examples: > SELECT 3 _FUNC_ 5; @@ -102,8 +102,8 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet usage = "expr1 _FUNC_ expr2 - Bitwise exclusive OR.", extended = """ Arguments: - expr1 - a integral numeric expression. - expr2 - a integral numeric expression. + expr1 - an integral numeric expression. + expr2 - an integral numeric expression. Examples: > SELECT 3 _FUNC_ 5; @@ -136,7 +136,7 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme usage = "_FUNC_ expr - Bitwise NOT.", extended = """ Arguments: - expr - a integral numeric expression. + expr - an integral numeric expression. Examples: > SELECT _FUNC_ 0; diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 451df73c5617..3701be5b45c9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -889,8 +889,8 @@ case class Pow(left: Expression, right: Expression) usage = "_FUNC_(base, expr) - Bitwise left shift.", extended = """ Arguments: - base - a integral numeric expression that defines the base number to shift. - expr - a integral numeric expression. + base - an integral numeric expression that defines the base number to shift. + expr - an integral numeric expression. Examples: > SELECT _FUNC_(2, 1); @@ -927,8 +927,8 @@ case class ShiftLeft(left: Expression, right: Expression) usage = "_FUNC_(base, expr) - Bitwise (signed) right shift.", extended = """ Arguments: - base - a integral numeric expression that defines the base number to shift. - expr - a integral numeric expression. + base - an integral numeric expression that defines the base number to shift. + expr - an integral numeric expression. Examples: > SELECT _FUNC_(4, 1); @@ -965,8 +965,8 @@ case class ShiftRight(left: Expression, right: Expression) usage = "_FUNC_(base, expr) - Bitwise unsigned right shift.", extended = """ Arguments: - base - a integral numeric expression that defines the base number to shift. - expr - a integral numeric expression. + base - an integral numeric expression that defines the base number to shift. + expr - an integral numeric expression. Examples: > SELECT _FUNC_(4, 1); diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 883537325765..5fd3458214d2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1499,8 +1499,8 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { Row( """Extended Usage: | Arguments: - | expr1 - a integral numeric expression. - | expr2 - a integral numeric expression. + | expr1 - an integral numeric expression. + | expr2 - an integral numeric expression. | | Examples: | > SELECT 3 ^ 5; From 2656c6265ef5dc02a0fd06b4f8154fd12939bdad Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sun, 23 Oct 2016 22:33:10 +0900 Subject: [PATCH 39/47] another double-check --- .../catalyst/expressions/CallMethodViaReflection.scala | 2 +- .../apache/spark/sql/catalyst/expressions/Cast.scala | 2 +- .../expressions/aggregate/ApproximatePercentile.scala | 2 +- .../sql/catalyst/expressions/aggregate/collect.scala | 4 ++-- .../spark/sql/catalyst/expressions/arithmetic.scala | 1 - .../catalyst/expressions/collectionOperations.scala | 2 +- .../spark/sql/catalyst/expressions/generators.scala | 10 +++++----- 7 files changed, 11 insertions(+), 12 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala index 8320ffaaa91e..fe3746cb98f9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala @@ -49,7 +49,7 @@ import org.apache.spark.util.Utils Arguments: class - a string literal that represents a fully-qualified class name. method - a string literal that represents a method name. - arg - an expression of any type that represents arguments for the method. + arg - a boolean, numeric or string expression that represents arguments for the method. Examples: > SELECT _FUNC_('java.util.UUID', 'randomUUID'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 77f858e6f85c..ddd63806be41 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -118,7 +118,7 @@ object Cast { extended = """ Arguments: expr - an expression of any type. - type - data type to cast expr into. + type - any data type to cast expr into. Examples: > SELECT _FUNC_('10' as int); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala index dba488f44b54..5e8cb50df68d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala @@ -64,7 +64,7 @@ import org.apache.spark.sql.types._ col - a numeric expression. percentage - a numeric literal or an array literal of numeric type that defines the percentile between 0.0 and 1.0. For example, 0.5 means 50-percentile. - accuracy - a numeric literal of approximation accuracy. + accuracy - a numeric literal that defines approximation accuracy. Examples: > SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala index aa80bac662f3..f20495c2b070 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala @@ -89,7 +89,7 @@ abstract class Collect extends ImperativeAggregate { usage = "_FUNC_(expr) - Collects and returns a list of non-unique elements.", extended = """ Arguments: - expr - an expression of any type that represents data to collect as a list. + expr - an expression of any type that represents data to collect as a list. """) case class CollectList( child: Expression, @@ -116,7 +116,7 @@ case class CollectList( usage = "_FUNC_(expr) - Collects and returns a set of unique elements.", extended = """ Arguments: - expr - an expression of any type that represents data to collect as a set. + expr - an expression of any type that represents data to collect as a set. """) case class CollectSet( child: Expression, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index d42c80a988dc..c80233fb5282 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -260,7 +260,6 @@ case class Multiply(left: Expression, right: Expression) Examples: > SELECT 3 _FUNC_ 2; 1.5 - Examples: > SELECT 2L _FUNC_ 2L; 1.0 """) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index 7b547067bc86..ef10d8ec3d21 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -134,7 +134,7 @@ case class MapValues(child: Expression) extended = """ Arguments: array - an array expression. - ascendingOrder - a boolean literal that represents ascending/descending order. + ascendingOrder - a boolean literal that represents ascending / descending order. Default is true. Examples: > SELECT _FUNC_(array('b', 'd', 'c', 'a'), true); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index f8ab824f8998..d54139de82d3 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -105,7 +105,7 @@ case class UserDefinedGenerator( usage = "_FUNC_(n, expr1, ..., exprk) - Separate expr1, ..., exprk into n rows.", extended = """ Arguments: - n - a integer literal that represents the number of output rows. + n - an integer literal that represents the number of output rows. expr - an expression of any type. Examples: @@ -235,10 +235,10 @@ abstract class ExplodeBase(child: Expression, position: Boolean) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr) - Separates the elements of array a into multiple rows, or the elements of map a into multiple rows and columns.", + usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple rows, or the elements of map `expr` into multiple rows and columns.", extended = """ Arguments: - expr - an array expression that contains map or struct expression as the element. + expr - an array or map expression. Examples: > SELECT _FUNC_(array(10, 20)); @@ -259,10 +259,10 @@ case class Explode(child: Expression) extends ExplodeBase(child, position = fals */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr) - Separates the elements of array a into multiple rows with positions, or the elements of a map into multiple rows and columns with positions.", + usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple rows with positions, or the elements of `expr` map into multiple rows and columns with positions.", extended = """ Arguments: - expr - an array expression that contains map or struct expression as the element. + expr - an array or map expression. Examples: > SELECT _FUNC_(array(10,20)); From 15c02cac00d08e81e6959997ab7e954eede12a39 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 24 Oct 2016 21:41:41 +0900 Subject: [PATCH 40/47] Address comments and another double-check --- .../expressions/CallMethodViaReflection.scala | 3 ++- .../aggregate/ApproximatePercentile.scala | 2 +- .../sql/catalyst/expressions/arithmetic.scala | 12 +++++----- .../expressions/bitwiseExpressions.scala | 14 +++++------ .../expressions/complexTypeCreator.scala | 6 ++--- .../expressions/mathExpressions.scala | 20 ++++++++-------- .../spark/sql/catalyst/expressions/misc.scala | 2 +- .../expressions/stringExpressions.scala | 2 +- .../expressions/windowExpressions.scala | 23 ++++++++----------- .../sql/catalyst/expressions/xml/xpath.scala | 16 ++++++------- .../sql/execution/command/DDLSuite.scala | 4 ++-- 11 files changed, 51 insertions(+), 53 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala index fe3746cb98f9..51b24ba6dbc2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala @@ -49,7 +49,8 @@ import org.apache.spark.util.Utils Arguments: class - a string literal that represents a fully-qualified class name. method - a string literal that represents a method name. - arg - a boolean, numeric or string expression that represents arguments for the method. + arg - a boolean, string or numeric expression except decimal that represents arguments for + the method. Examples: > SELECT _FUNC_('java.util.UUID', 'randomUUID'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala index 5e8cb50df68d..b016e34401ae 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala @@ -64,7 +64,7 @@ import org.apache.spark.sql.types._ col - a numeric expression. percentage - a numeric literal or an array literal of numeric type that defines the percentile between 0.0 and 1.0. For example, 0.5 means 50-percentile. - accuracy - a numeric literal that defines approximation accuracy. + accuracy - a numeric literal that defines approximation accuracy. Default is 10000. Examples: > SELECT percentile_approx(10.0, array(0.5, 0.4, 0.1), 100); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index c80233fb5282..9ef0cf8a0280 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -149,8 +149,8 @@ object BinaryArithmetic { usage = "expr1 _FUNC_ expr2 - Returns expr1+expr2.", extended = """ Arguments: - expr1 - a numeric expression. - expr2 - a numeric expression. + expr1 - a numeric or interval expression. + expr2 - a numeric or interval expression. Examples: > SELECT 1 _FUNC_ 2; @@ -189,8 +189,8 @@ case class Add(left: Expression, right: Expression) extends BinaryArithmetic wit usage = "expr1 _FUNC_ expr2 - Returns expr1-expr2.", extended = """ Arguments: - expr1 - a numeric expression. - expr2 - a numeric expression. + expr1 - a numeric or interval expression. + expr2 - a numeric or interval expression. Examples: > SELECT 2 _FUNC_ 1; @@ -545,7 +545,7 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic wi usage = "_FUNC_(expr, ...) - Returns the least value of all parameters, skipping null values.", extended = """ Arguments: - expr - an expression of any type except struct, map and array. + expr - an expression of any type. Examples: > SELECT _FUNC_(10, 9, 2, 4, 3); @@ -613,7 +613,7 @@ case class Least(children: Seq[Expression]) extends Expression { usage = "_FUNC_(expr, ...) - Returns the greatest value of all parameters, skipping null values.", extended = """ Arguments: - expr - an expression of any type except struct, map and array. + expr - an expression of any type. Examples: > SELECT _FUNC_(10, 9, 2, 4, 3); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index 278449906c29..72a600706827 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -30,8 +30,8 @@ import org.apache.spark.sql.types._ usage = "expr1 _FUNC_ expr2 - Bitwise AND.", extended = """ Arguments: - expr1 - an integral numeric expression. - expr2 - an integral numeric expression. + expr1 - an integral expression. + expr2 - an integral expression. Examples: > SELECT 3 _FUNC_ 5; @@ -66,8 +66,8 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme usage = "expr1 _FUNC_ expr2 - Bitwise OR.", extended = """ Arguments: - expr1 - an integral numeric expression. - expr2 - an integral numeric expression. + expr1 - an integral expression. + expr2 - an integral expression. Examples: > SELECT 3 _FUNC_ 5; @@ -102,8 +102,8 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet usage = "expr1 _FUNC_ expr2 - Bitwise exclusive OR.", extended = """ Arguments: - expr1 - an integral numeric expression. - expr2 - an integral numeric expression. + expr1 - an integral expression. + expr2 - an integral expression. Examples: > SELECT 3 _FUNC_ 5; @@ -136,7 +136,7 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme usage = "_FUNC_ expr - Bitwise NOT.", extended = """ Arguments: - expr - an integral numeric expression. + expr - an integral expression. Examples: > SELECT _FUNC_ 0; diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index 18cccb51f6e5..aebec87095b7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -433,12 +433,12 @@ case class CreateNamedStructUnsafe(children: Seq[Expression]) extends Expression */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(text[, pairDelim, keyValueDelim]) - Creates a map after splitting the text into key/value pairs using delimiters. Default delimiters are ',' for pairDelim and ':' for keyValueDelim.", + usage = "_FUNC_(text[, pairDelim[, keyValueDelim]]) - Creates a map after splitting the text into key/value pairs using delimiters. Default delimiters are ',' for pairDelim and ':' for keyValueDelim.", extended = """ Arguments: text - a string expression that represents data to convert into map. - pairDelim - a string literal that defines the delimiter to separate each pair. For example, ",". - keyValueDelim - a string literal that defines the delimiter to separate the key and value in the pair. For example, ":". + pairDelim - a string literal that defines the delimiter to separate each pair. For example, ",". Default is ",". + keyValueDelim - a string literal that defines the delimiter to separate the key and value in the pair. For example, ":". Default is ":". Examples: > SELECT _FUNC_('a:1,b:2,c:3', ',', ':'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index 3701be5b45c9..ac8ec7a33521 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -889,8 +889,8 @@ case class Pow(left: Expression, right: Expression) usage = "_FUNC_(base, expr) - Bitwise left shift.", extended = """ Arguments: - base - an integral numeric expression that defines the base number to shift. - expr - an integral numeric expression. + base - an integral expression that defines the base number to shift. + expr - an integral expression. Examples: > SELECT _FUNC_(2, 1); @@ -927,8 +927,8 @@ case class ShiftLeft(left: Expression, right: Expression) usage = "_FUNC_(base, expr) - Bitwise (signed) right shift.", extended = """ Arguments: - base - an integral numeric expression that defines the base number to shift. - expr - an integral numeric expression. + base - an integral expression that defines the base number to shift. + expr - an integral expression. Examples: > SELECT _FUNC_(4, 1); @@ -965,8 +965,8 @@ case class ShiftRight(left: Expression, right: Expression) usage = "_FUNC_(base, expr) - Bitwise unsigned right shift.", extended = """ Arguments: - base - an integral numeric expression that defines the base number to shift. - expr - an integral numeric expression. + base - an integral expression that defines the base number to shift. + expr - an integral expression. Examples: > SELECT _FUNC_(4, 1); @@ -1248,11 +1248,11 @@ abstract class RoundBase(child: Expression, scale: Expression, */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr, d) - Round expr to d decimal places using HALF_UP rounding mode.", + usage = "_FUNC_(expr, d) - Returns expr rounded to d decimal places using HALF_UP rounding mode.", extended = """ Arguments: expr - a numeric expression. - d - a numeric expression that defines tge decimal places using HALF_UP rounding mode. + d - a numeric literal that defines tge decimal places using HALF_UP rounding mode. Examples: > SELECT _FUNC_(2.5, 0); @@ -1272,11 +1272,11 @@ case class Round(child: Expression, scale: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr, d) - Round expr to d decimal places using HALF_EVEN rounding mode.", + usage = "_FUNC_(expr, d) - Returns expr rounded to d decimal places using HALF_EVEN rounding mode.", extended = """ Arguments: expr - a numeric expression. - d - a numeric expression that defines decimal places using HALF_EVEN rounding mode. + d - a numeric literal that defines decimal places using HALF_EVEN rounding mode. Examples: > SELECT _FUNC_(2.5, 0); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala index 0cb96f0059de..529ebacd42ef 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala @@ -78,7 +78,7 @@ case class Md5(child: Expression) extends UnaryExpression with ImplicitCastInput """, extended = """ Arguments: - expr - a string expression. + expr - a binary expression. bitLength - a numeric expression that defines the length of bit. Available values are 224, 256, 384 and 512. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 06b366cb1324..94c4c48e0c1d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -191,7 +191,7 @@ case class ConcatWs(children: Seq[Expression]) str - a string expression. Examples: - > SELECT _FUNC_(1, 'scala', 'java') FROM src LIMIT 1; + > SELECT _FUNC_(1, 'scala', 'java'); scala """) case class Elt(children: Seq[Expression]) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala index b89c1e16a4ff..76eb60579ce8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala @@ -383,7 +383,7 @@ abstract class OffsetWindowFunction */ @ExpressionDescription( usage = """ - _FUNC_(input[, offset[, default]]) - LEAD returns the value of 'x' at the 'offset'th row + _FUNC_(input[, offset[, default]]) - Returns the value of 'x' at the 'offset'th row after the current row in the window. The default value of 'offset' is 1 and the default value of 'default' is null. If the value of 'x' at the 'offset'th row is null, null is returned. @@ -420,7 +420,7 @@ case class Lead(input: Expression, offset: Expression, default: Expression) */ @ExpressionDescription( usage = """ - _FUNC_(input[, offset[, default]]) - LAG returns the value of 'x' at the 'offset'th row + _FUNC_(input[, offset[, default]]) - Returns the value of 'x' at the 'offset'th row before the current row in the window. The default value of 'offset' is 1 and the default value of 'default' is null. If the value of 'x' at the 'offset'th row is null, null is returned. @@ -489,9 +489,8 @@ object SizeBasedWindowFunction { */ @ExpressionDescription( usage = """ - _FUNC_() - The ROW_NUMBER() function assigns a unique, sequential number to - each row, starting with one, according to the ordering of rows within - the window partition. + _FUNC_() - Assigns a unique, sequential number to each row, starting with one, + according to the ordering of rows within the window partition. """) case class RowNumber() extends RowNumberLike { override val evaluateExpression = rowNumber @@ -508,8 +507,7 @@ case class RowNumber() extends RowNumberLike { */ @ExpressionDescription( usage = """ - _FUNC_() - The CUME_DIST() function computes the position of a value relative to - a all values in the partition. + _FUNC_() - Computes the position of a value relative to a all values in the partition. """) case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction { override def dataType: DataType = DoubleType @@ -543,8 +541,8 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction { */ @ExpressionDescription( usage = """ - _FUNC_(n) - The NTILE(n) function divides the rows for each window partition - into 'n' buckets ranging from 1 to at most 'n'. + _FUNC_(n) - Divides the rows for each window partition into 'n' buckets ranging + from 1 to at most 'n'. """, extended = """ Arguments: @@ -672,7 +670,7 @@ abstract class RankLike extends AggregateWindowFunction { */ @ExpressionDescription( usage = """ - _FUNC_() - RANK() computes the rank of a value in a group of values. The result + _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the number of rows preceding or equal to the current row in the ordering of the partition. Tie values will produce gaps in the sequence. """) @@ -694,7 +692,7 @@ case class Rank(children: Seq[Expression]) extends RankLike { */ @ExpressionDescription( usage = """ - _FUNC_() - The DENSE_RANK() function computes the rank of a value in a group of + _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the previously assigned rank value. Unlike Rank, DenseRank will not produce gaps in the ranking sequence. """) @@ -724,8 +722,7 @@ case class DenseRank(children: Seq[Expression]) extends RankLike { */ @ExpressionDescription( usage = """ - _FUNC_() - PERCENT_RANK() The PercentRank function computes the percentage - ranking of a value in a group of values. + _FUNC_() - Computes the percentage ranking of a value in a group of values. """) case class PercentRank(children: Seq[Expression]) extends RankLike with SizeBasedWindowFunction { def this() = this(Nil) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index d0117da0770d..37b36efb85d8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -60,7 +60,7 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xpath - a string literal that represents XPath literal. Examples: > SELECT _FUNC_('1','a/b'); @@ -81,7 +81,7 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xpath - a string literal that represents XPath literal. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -102,7 +102,7 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xpath - a string literal that represents XPath literal. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -123,7 +123,7 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xpath - a string literal that represents XPath literal. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -144,7 +144,7 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xpath - a string literal that represents XPath literal. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -165,7 +165,7 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xpath - a string literal that represents XPath literal. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -187,7 +187,7 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - path - a string literal that represents XPath expression. + path - a string literal that represents XPath literal. Examples: > SELECT _FUNC_('bcc','a/c'); @@ -210,7 +210,7 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - path - a string literal that represents XPath expression. + path - a string literal that represents XPath literal. Examples: > SELECT _FUNC_('b1b2b3c1c2','a/b/text()'); diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 5fd3458214d2..81cb194187ca 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1499,8 +1499,8 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { Row( """Extended Usage: | Arguments: - | expr1 - an integral numeric expression. - | expr2 - an integral numeric expression. + | expr1 - an integral expression. + | expr2 - an integral expression. | | Examples: | > SELECT 3 ^ 5; From 035baefbed67ac723c56ba178f2b122b0a5ee742 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Mon, 24 Oct 2016 22:23:01 +0900 Subject: [PATCH 41/47] revert mistakes --- .../sql/catalyst/expressions/xml/xpath.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index 37b36efb85d8..d0117da0770d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -60,7 +60,7 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath literal. + xpath - a string literal that represents XPath expression. Examples: > SELECT _FUNC_('1','a/b'); @@ -81,7 +81,7 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath literal. + xpath - a string literal that represents XPath expression. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -102,7 +102,7 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath literal. + xpath - a string literal that represents XPath expression. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -123,7 +123,7 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath literal. + xpath - a string literal that represents XPath expression. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -144,7 +144,7 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath literal. + xpath - a string literal that represents XPath expression. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -165,7 +165,7 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - xpath - a string literal that represents XPath literal. + xpath - a string literal that represents XPath expression. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); @@ -187,7 +187,7 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - path - a string literal that represents XPath literal. + path - a string literal that represents XPath expression. Examples: > SELECT _FUNC_('bcc','a/c'); @@ -210,7 +210,7 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract { extended = """ Arguments: xml - a string expression that represents XML document. - path - a string literal that represents XPath literal. + path - a string literal that represents XPath expression. Examples: > SELECT _FUNC_('b1b2b3c1c2','a/b/text()'); From c55ecb60c79694c79a7c60c0c4a759bff6c71fc6 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Tue, 25 Oct 2016 11:03:26 +0900 Subject: [PATCH 42/47] another double-check and remove extra change for newlines --- .../spark/sql/catalyst/expressions/aggregate/Covariance.scala | 1 - .../spark/sql/catalyst/expressions/complexTypeCreator.scala | 4 ++-- .../spark/sql/catalyst/expressions/nullExpressions.scala | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala index cea86fa9c8ae..07b951592671 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Covariance.scala @@ -76,7 +76,6 @@ abstract class Covariance(x: Expression, y: Expression) extends DeclarativeAggre } } - @ExpressionDescription( usage = "_FUNC_(expr1, expr2) - Returns the population covariance of a set of number pairs.", extended = """ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index aebec87095b7..39769d8e23cf 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -441,8 +441,8 @@ case class CreateNamedStructUnsafe(children: Seq[Expression]) extends Expression keyValueDelim - a string literal that defines the delimiter to separate the key and value in the pair. For example, ":". Default is ":". Examples: - > SELECT _FUNC_('a:1,b:2,c:3', ',', ':'); - map("a":"1","b":"2","c":"3") + > SELECT _FUNC_('a:1,b:2,c:3', ',', ':'); + map("a":"1","b":"2","c":"3") """) // scalastyle:on line.size.limit case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: Expression) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala index 37f39cd5cdfd..bed3ce9b46cd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala @@ -261,7 +261,6 @@ case class IsNaN(child: Expression) extends UnaryExpression } } - /** * An Expression evaluates to `left` iff it's not NaN, or evaluates to `right` otherwise. * This Expression is useful for mapping NaN values to null. From ad7b71dcb1f42272ead43a9671a49a12e38ba776 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Wed, 26 Oct 2016 19:38:09 +0900 Subject: [PATCH 43/47] Address comments --- .../sql/catalyst/expressions/arithmetic.scala | 5 +- .../expressions/bitwiseExpressions.scala | 8 +-- .../expressions/conditionalExpressions.scala | 16 +++-- .../expressions/windowExpressions.scala | 68 +++++++++---------- .../sql/catalyst/expressions/xml/xpath.scala | 62 ++++++++++------- .../sql/execution/command/DDLSuite.scala | 3 +- .../sql/hive/execution/SQLQuerySuite.scala | 2 +- 7 files changed, 93 insertions(+), 71 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index 9ef0cf8a0280..e70ecf58dc02 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -29,6 +29,9 @@ import org.apache.spark.unsafe.types.CalendarInterval extended = """ Arguments: expr - a numeric or interval expression. + Examples: + > SELECT _FUNC_(1); + -1 """) case class UnaryMinus(child: Expression) extends UnaryExpression with ExpectsInputTypes with NullIntolerant { @@ -428,7 +431,7 @@ case class Remainder(left: Expression, right: Expression) } @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns the positive modulo.", + usage = "_FUNC_(expr1, expr2) - Returns the positive value of expr1 mod expr2.", extended = """ Arguments: expr1 - a numeric expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index 72a600706827..b097021e93eb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.types._ * Code generation inherited from BinaryArithmetic. */ @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Bitwise AND.", + usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise AND of expr1 and expr2.", extended = """ Arguments: expr1 - an integral expression. @@ -63,7 +63,7 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme * Code generation inherited from BinaryArithmetic. */ @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Bitwise OR.", + usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise OR of expr1 and expr2.", extended = """ Arguments: expr1 - an integral expression. @@ -99,7 +99,7 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet * Code generation inherited from BinaryArithmetic. */ @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Bitwise exclusive OR.", + usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise exclusive OR of expr1 and expr2.", extended = """ Arguments: expr1 - an integral expression. @@ -133,7 +133,7 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme * A function that calculates bitwise not(~) of a number. */ @ExpressionDescription( - usage = "_FUNC_ expr - Bitwise NOT.", + usage = "_FUNC_ expr - Returns the result of bitwise NOT of expr.", extended = """ Arguments: expr - an integral expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala index 25dab123ef51..9d764d8f966d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala @@ -24,12 +24,12 @@ import org.apache.spark.sql.types._ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr1, expr2, expr3) - If expr1 is TRUE then IF() returns expr2; otherwise it returns expr3.", + usage = "_FUNC_(expr1, expr2, expr3) - If expr1 evaluates to true, then IF() returns expr2; otherwise it returns expr3.", extended = """ Arguments: expr1 - a boolean expression. - expr2 - an expression of any type that represents the return value when expr1 is TRUE. - expr3 - an expression of any type that represents the return value when expr1 is FALSE. + expr2 - an expression of any type that represents the return value when expr1 evaluates to true. + expr3 - an expression of any type that represents the return value when expr1 evaluates to false. Examples: > SELECT _FUNC_(1 < 2, 'a', 'b'); @@ -172,7 +172,15 @@ abstract class CaseWhenBase( */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "CASE WHEN expr1 THEN expr2 [WHEN expr3 THEN expr4]* [ELSE expr5] END - When expr1 = true, returns expr2; when expr3 = true, return expr4; else return expr5.") + usage = "CASE WHEN expr1 THEN expr2 [WHEN expr3 THEN expr4]* [ELSE expr5] END - When expr1 = true, returns expr2; when expr3 = true, return expr4; else return expr5.", + extended = """ + Arguments: + expr1 - a boolean expression. + expr2 - an expression of any type. + expr3 - a boolean expression. + expr4 - an expression of any type. + expr5 - an expression of any type. + """) // scalastyle:on line.size.limit case class CaseWhen( val branches: Seq[(Expression, Expression)], diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala index 76eb60579ce8..ea98774f39fb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala @@ -321,7 +321,7 @@ abstract class OffsetWindowFunction val input: Expression /** - * Default result value for the function when the 'offset'th row does not exist. + * Default result value for the function when the `offset`th row does not exist. */ val default: Expression @@ -372,29 +372,29 @@ abstract class OffsetWindowFunction } /** - * The Lead function returns the value of 'x' at the 'offset'th row after the current row in + * The Lead function returns the value of `input` at the `offset`th row after the current row in * the window. Offsets start at 0, which is the current row. The offset must be constant - * integer value. The default offset is 1. When the value of 'x' is null at the 'offset'th row, - * null is returned. If there is no such offset row, the default expression is evaluated. + * integer value. The default offset is 1. When the value of `input` is null at the `offset`th row, + * null is returned. If there is no such offset row, the `default` expression is evaluated. * - * @param input expression to evaluate 'offset' rows after the current row. + * @param input expression to evaluate `offset` rows after the current row. * @param offset rows to jump ahead in the partition. * @param default to use when the offset is larger than the window. The default value is null. */ @ExpressionDescription( usage = """ - _FUNC_(input[, offset[, default]]) - Returns the value of 'x' at the 'offset'th row + _FUNC_(input[, offset[, default]]) - Returns the value of `input` at the `offset`th row after the current row in the window. - The default value of 'offset' is 1 and the default value of 'default' is null. - If the value of 'x' at the 'offset'th row is null, null is returned. - If there is no such offset row (e.g. when the offset is 1, the last row of the window - does not have any subsequent row), 'default' is returned. + The default value of `offset` is 1 and the default value of `default` is null. + If the value of `input` at the `offset`th row is null, null is returned. + If there is no such an offset row (e.g. when the offset is 1, the last row of the window + does not have any subsequent row), `default` is returned. """, extended = """ Arguments: input - an expression of any type. offset - a numeric expression. Default is 1. - default - an expression of any type. Defualt is NULL. + default - an expression of any type. Default is null. """) case class Lead(input: Expression, offset: Expression, default: Expression) extends OffsetWindowFunction { @@ -409,29 +409,29 @@ case class Lead(input: Expression, offset: Expression, default: Expression) } /** - * The Lag function returns the value of 'x' at the 'offset'th row before the current row in + * The Lag function returns the value of `input` at the `offset`th row before the current row in * the window. Offsets start at 0, which is the current row. The offset must be constant - * integer value. The default offset is 1. When the value of 'x' is null at the 'offset'th row, - * null is returned. If there is no such offset row, the default expression is evaluated. + * integer value. The default offset is 1. When the value of `input` is null at the `offset`th row, + * null is returned. If there is no such offset row, the `default` expression is evaluated. * - * @param input expression to evaluate 'offset' rows before the current row. + * @param input expression to evaluate `offset` rows before the current row. * @param offset rows to jump back in the partition. * @param default to use when the offset row does not exist. */ @ExpressionDescription( usage = """ - _FUNC_(input[, offset[, default]]) - Returns the value of 'x' at the 'offset'th row + _FUNC_(input[, offset[, default]]) - Returns the value of `input` at the `offset`th row before the current row in the window. - The default value of 'offset' is 1 and the default value of 'default' is null. - If the value of 'x' at the 'offset'th row is null, null is returned. + The default value of `offset` is 1 and the default value of `default` is null. + If the value of `input` at the `offset`th row is null, null is returned. If there is no such offset row (e.g. when the offset is 1, the first row of the window - does not have any previous row), 'default' is returned. + does not have any previous row), `default` is returned. """, extended = """ Arguments: input - an expression of any type. offset - a numeric expression. Default is 1. - default - an expression of any type. Defualt is NULL. + default - an expression of any type. Default is null. """) case class Lag(input: Expression, offset: Expression, default: Expression) extends OffsetWindowFunction { @@ -498,7 +498,7 @@ case class RowNumber() extends RowNumberLike { } /** - * The CumeDist function computes the position of a value relative to a all values in the partition. + * The CumeDist function computes the position of a value relative to all values in the partition. * The result is the number of rows preceding or equal to the current row in the ordering of the * partition divided by the total number of rows in the window partition. Any tie values in the * ordering will evaluate to the same position. @@ -507,7 +507,7 @@ case class RowNumber() extends RowNumberLike { */ @ExpressionDescription( usage = """ - _FUNC_() - Computes the position of a value relative to a all values in the partition. + _FUNC_() - Computes the position of a value relative to all values in the partition. """) case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction { override def dataType: DataType = DoubleType @@ -611,9 +611,9 @@ case class NTile(buckets: Expression) extends RowNumberLike with SizeBasedWindow /** * A RankLike function is a WindowFunction that changes its value based on a change in the value of - * the order of the window in which is processed. For instance, when the value of 'x' changes in a - * window ordered by 'x' the rank function also changes. The size of the change of the rank function - * is (typically) not dependent on the size of the change in 'x'. + * the order of the window in which is processed. For instance, when the value of `input` changes + * in a window ordered by `input` the rank function also changes. The size of the change of the + * rank function is (typically) not dependent on the size of the change in `input`. * * This documentation has been based upon similar documentation for the Hive and Presto projects. */ @@ -659,7 +659,7 @@ abstract class RankLike extends AggregateWindowFunction { /** * The Rank function computes the rank of a value in a group of values. The result is one plus the - * number of rows preceding or equal to the current row in the ordering of the partition. Tie values + * number of rows preceding or equal to the current row in the ordering of the partition. The values * will produce gaps in the sequence. * * This documentation has been based upon similar documentation for the Hive and Presto projects. @@ -670,9 +670,9 @@ abstract class RankLike extends AggregateWindowFunction { */ @ExpressionDescription( usage = """ - _FUNC_() - Computes the rank of a value in a group of values. The result - is one plus the number of rows preceding or equal to the current row in the - ordering of the partition. Tie values will produce gaps in the sequence. + _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the number + of rows preceding or equal to the current row in the ordering of the partition. The values + will produce gaps in the sequence. """) case class Rank(children: Seq[Expression]) extends RankLike { def this() = this(Nil) @@ -681,8 +681,8 @@ case class Rank(children: Seq[Expression]) extends RankLike { /** * The DenseRank function computes the rank of a value in a group of values. The result is one plus - * the previously assigned rank value. Unlike Rank, DenseRank will not produce gaps in the ranking - * sequence. + * the previously assigned rank value. Unlike [[Rank]], [[DenseRank]] will not produce gaps in the + * ranking sequence. * * This documentation has been based upon similar documentation for the Hive and Presto projects. * @@ -692,9 +692,9 @@ case class Rank(children: Seq[Expression]) extends RankLike { */ @ExpressionDescription( usage = """ - _FUNC_() - Computes the rank of a value in a group of - values. The result is one plus the previously assigned rank value. Unlike Rank, - DenseRank will not produce gaps in the ranking sequence. + _FUNC_() - Computes the rank of a value in a group of values. The result is one plus the + previously assigned rank value. Unlike the function rank, dense_rank will not produce gaps + in the ranking sequence. """) case class DenseRank(children: Seq[Expression]) extends RankLike { def this() = this(Nil) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index d0117da0770d..9bf9e746d402 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -34,7 +34,7 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with override def left: Expression = xml override def right: Expression = path - /** XPath expressions are always nullable, e.g. if the xml string is empty. */ + /** an XPath expressions are always nullable, e.g. if the xml string is empty. */ override def nullable: Boolean = true override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType) @@ -56,11 +56,11 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with } @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Evaluates a boolean XPath expression.", + usage = "_FUNC_(xml, xpath) - Evaluates a boolean an XPath expression.", extended = """ Arguments: - xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xml - a string expression that represents an XML document. + xpath - a string literal that represents an XPath expression. Examples: > SELECT _FUNC_('1','a/b'); @@ -76,19 +76,21 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract } } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a short value that matches the XPath expression.", + usage = "_FUNC_(xml, xpath) - Returns a short value, or the value zero if no match is found, or a match is found but the value is non-numeric.", extended = """ Arguments: - xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xml - a string expression that represents an XML document. + xpath - a string literal that represents an XPath expression. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); 3 """) +// scalastyle:on line.size.limit case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { - override def prettyName: String = "xpath_int" + override def prettyName: String = "xpath_short" override def dataType: DataType = ShortType override def nullSafeEval(xml: Any, path: Any): Any = { @@ -97,17 +99,19 @@ case class XPathShort(xml: Expression, path: Expression) extends XPathExtract { } } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns an integer value that matches the XPath expression.", + usage = "_FUNC_(xml, xpath) - Returns an integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", extended = """ Arguments: - xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xml - a string expression that represents an XML document. + xpath - a string literal that represents an XPath expression. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); 3 """) +// scalastyle:on line.size.limit case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_int" override def dataType: DataType = IntegerType @@ -118,17 +122,19 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { } } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a long value that matches the XPath expression.", + usage = "_FUNC_(xml, xpath) - Returns a long value, or the value zero if no match is found, or a match is found but the value is non-numeric.", extended = """ Arguments: - xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xml - a string expression that represents an XML document. + xpath - a string literal that represents an XPath expression. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); 3 """) +// scalastyle:on line.size.limit case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_long" override def dataType: DataType = LongType @@ -139,17 +145,19 @@ case class XPathLong(xml: Expression, path: Expression) extends XPathExtract { } } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a float value that matches the XPath expression.", + usage = "_FUNC_(xml, xpath) - Returns a float value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric.", extended = """ Arguments: - xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xml - a string expression that represents an XML document. + xpath - a string literal that represents an XPath expression. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); 3.0 """) +// scalastyle:on line.size.limit case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_float" override def dataType: DataType = FloatType @@ -160,17 +168,19 @@ case class XPathFloat(xml: Expression, path: Expression) extends XPathExtract { } } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a double value that matches the XPath expression.", + usage = "_FUNC_(xml, xpath) - Returns a double value, the value zero if no match is found, or NaN if a match is found but the value is non-numeric.", extended = """ Arguments: - xml - a string expression that represents XML document. - xpath - a string literal that represents XPath expression. + xml - a string expression that represents an XML document. + xpath - a string literal that represents an XPath expression. Examples: > SELECT _FUNC_('12', 'sum(a/b)'); 3.0 """) +// scalastyle:on line.size.limit case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_float" override def dataType: DataType = DoubleType @@ -183,11 +193,11 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the XPath expression.", + usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the an XPath expression.", extended = """ Arguments: - xml - a string expression that represents XML document. - path - a string literal that represents XPath expression. + xml - a string expression that represents an XML document. + xpath - a string literal that represents an XPath expression. Examples: > SELECT _FUNC_('bcc','a/c'); @@ -206,11 +216,11 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a string array of values within the nodes of xml that match the XPath expression.", + usage = "_FUNC_(xml, xpath) - Returns a string array of values within the nodes of xml that match the an XPath expression.", extended = """ Arguments: - xml - a string expression that represents XML document. - path - a string literal that represents XPath expression. + xml - a string expression that represents an XML document. + xpath - a string literal that represents an XPath expression. Examples: > SELECT _FUNC_('b1b2b3c1c2','a/b/text()'); diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 81cb194187ca..423f3119f58b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1507,7 +1507,8 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { | 2 | """.stripMargin) :: Row("Function: ^") :: - Row("Usage: expr1 ^ expr2 - Bitwise exclusive OR.") :: Nil + Row("Usage: expr1 ^ expr2 - Returns the result of " + + "bitwise exclusive OR of expr1 and expr2.") :: Nil ) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index beb3bee5674f..c139f2bd806d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -274,7 +274,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { checkKeywordsExist(sql("describe functioN `~`"), "Function: ~", "Class: org.apache.spark.sql.catalyst.expressions.BitwiseNot", - "Usage: ~ expr - Bitwise NOT.") + "Usage: ~ expr - Returns the result of bitwise NOT of expr.") // Hard coded describe functions checkKeywordsExist(sql("describe function `<>`"), From 3965d00bdb8477be191dbd2e68447a5be6f45ebb Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 27 Oct 2016 10:46:22 +0900 Subject: [PATCH 44/47] Address comments --- .../sql/catalyst/expressions/predicates.scala | 14 +++++++------- .../catalyst/expressions/randomExpressions.scala | 4 ++-- .../catalyst/expressions/stringExpressions.scala | 4 ++-- .../spark/sql/catalyst/expressions/xml/xpath.scala | 10 ++++++---- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index 1ec2bbff411d..7beb5042aca6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -95,7 +95,7 @@ trait PredicateHelper { usage = "_FUNC_ expr - Logical not.", extended = """ Arguments: - expr - an expression of any type. + expr - a boolean expression. """) case class Not(child: Expression) extends UnaryExpression with Predicate with ImplicitCastInputTypes with NullIntolerant { @@ -121,7 +121,7 @@ case class Not(child: Expression) usage = "expr1 _FUNC_(expr2, expr3, ...) - Returns true if expr equals to any valN.", extended = """ Arguments: - expr - an expression of any type. + expr - a boolean expression. """) case class In(value: Expression, list: Seq[Expression]) extends Predicate with ImplicitCastInputTypes { @@ -262,8 +262,8 @@ case class InSet(child: Expression, hset: Set[Any]) extends UnaryExpression with usage = "expr1 _FUNC_ expr2 - Logical AND.", extended = """ Arguments: - expr1 - an expression of any type. - expr2 - an expression of any type. + expr1 - a boolean expression. + expr2 - a boolean expression. """) case class And(left: Expression, right: Expression) extends BinaryOperator with Predicate { @@ -330,8 +330,8 @@ case class And(left: Expression, right: Expression) extends BinaryOperator with usage = "expr1 _FUNC_ expr2 - Logical OR.", extended = """ Arguments: - expr1 - an expression of any type. - expr2 - an expression of any type. + expr1 - a boolean expression. + expr2 - a boolean expression. """) case class Or(left: Expression, right: Expression) extends BinaryOperator with Predicate { @@ -459,7 +459,7 @@ case class EqualTo(left: Expression, right: Expression) @ExpressionDescription( usage = """ - expr1 _FUNC_ expr2 - Returns same result as the EQUAL(=) operator for non-null operands. + expr1 _FUNC_ expr2 - Returns same result as the EQUAL(=) operator for non-null operands, but returns TRUE if both are NULL, FALSE if one of the them is NULL. """, extended = """ diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala index 04ef36a09751..1574fce8fced 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala @@ -57,7 +57,7 @@ abstract class RDG extends LeafExpression with Nondeterministic { /** Generate a random column with i.i.d. uniformly distributed values in [0, 1). */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_([seed]) - Returns a random column with i.i.d. uniformly distributed values in [0, 1].", + usage = "_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.). uniformly distributed values in [0, 1).", extended = """ Arguments: seed - a numeric literal. @@ -92,7 +92,7 @@ case class Rand(seed: Long) extends RDG { /** Generate a random column with i.i.d. values drawn from the standard normal distribution. */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_([seed]) - Returns a random column with i.i.d. values drawn from the standard normal distribution.", + usage = "_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.) values drawn from the standard normal distribution.", extended = """ Arguments: seed - a numeric literal. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 94c4c48e0c1d..b4ffc8ebe238 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -612,7 +612,7 @@ case class StringInstr(str: Expression, substr: Expression) _FUNC_(str, delim, count) - Returns the substring from str before count occurrences of the delimiter delim. If count is positive, everything to the left of the final delimiter (counting from the left) is returned. If count is negative, everything to the right of the final delimiter - (counting from the right) is returned. Substring_index performs a case-sensitive match + (counting from the right) is returned. The function substring_index performs a case-sensitive match when searching for delim. """, extended = """ @@ -746,7 +746,7 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression) extended = """ Arguments: str - a string expression. - len - a numeric expression that defines the number of pad. + len - a numeric expression that defines the number of pads. pad - a string expression that represents the string for padding. Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index 9bf9e746d402..97e7244f1880 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -55,8 +55,9 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with def path: Expression } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Evaluates a boolean an XPath expression.", + usage = "_FUNC_(xml, xpath) - Returns true if the XPath expression evaluates to true, or if a matching node is found.", extended = """ Arguments: xml - a string expression that represents an XML document. @@ -66,6 +67,7 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with > SELECT _FUNC_('1','a/b'); true """) +// scalastyle:on line.size.limit case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract { override def prettyName: String = "xpath_boolean" @@ -124,7 +126,7 @@ case class XPathInt(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a long value, or the value zero if no match is found, or a match is found but the value is non-numeric.", + usage = "_FUNC_(xml, xpath) - Returns a long integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", extended = """ Arguments: xml - a string expression that represents an XML document. @@ -193,7 +195,7 @@ case class XPathDouble(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the an XPath expression.", + usage = "_FUNC_(xml, xpath) - Returns the text contents of the first xml node that matches the XPath expression.", extended = """ Arguments: xml - a string expression that represents an XML document. @@ -216,7 +218,7 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract { // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a string array of values within the nodes of xml that match the an XPath expression.", + usage = "_FUNC_(xml, xpath) - Returns a string array of values within the nodes of xml that match the XPath expression.", extended = """ Arguments: xml - a string expression that represents an XML document. From 498d69c0bc8db84041e97bcae0587455ae30ad93 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 27 Oct 2016 10:49:56 +0900 Subject: [PATCH 45/47] Missed ones --- .../spark/sql/catalyst/expressions/stringExpressions.scala | 2 +- .../org/apache/spark/sql/catalyst/expressions/xml/xpath.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index b4ffc8ebe238..c1668b3f2e2f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -784,7 +784,7 @@ case class StringLPad(str: Expression, len: Expression, pad: Expression) extended = """ Arguments: str - a string expression. - len - a numeric expression that defines the number of pad. + len - a numeric expression that defines the number of pads. pad - a string expression that represents the string for padding. Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index 97e7244f1880..5717c3ffd850 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -34,7 +34,7 @@ abstract class XPathExtract extends BinaryExpression with ExpectsInputTypes with override def left: Expression = xml override def right: Expression = path - /** an XPath expressions are always nullable, e.g. if the xml string is empty. */ + /** XPath expressions are always nullable, e.g. if the xml string is empty. */ override def nullable: Boolean = true override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType) @@ -80,7 +80,7 @@ case class XPathBoolean(xml: Expression, path: Expression) extends XPathExtract // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(xml, xpath) - Returns a short value, or the value zero if no match is found, or a match is found but the value is non-numeric.", + usage = "_FUNC_(xml, xpath) - Returns a short integer value, or the value zero if no match is found, or a match is found but the value is non-numeric.", extended = """ Arguments: xml - a string expression that represents an XML document. From 400cee5222922fc13ea8d6dc32b8408a8db756d6 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Thu, 27 Oct 2016 23:49:13 +0900 Subject: [PATCH 46/47] add back-quotes, fix typos and specify the types more specifically --- .../expressions/CallMethodViaReflection.scala | 4 +- .../spark/sql/catalyst/expressions/Cast.scala | 4 +- .../MonotonicallyIncreasingID.scala | 10 +-- .../aggregate/ApproximatePercentile.scala | 4 +- .../expressions/aggregate/Count.scala | 6 +- .../expressions/aggregate/First.scala | 4 +- .../aggregate/HyperLogLogPlusPlus.scala | 4 +- .../catalyst/expressions/aggregate/Last.scala | 4 +- .../catalyst/expressions/aggregate/Max.scala | 2 +- .../catalyst/expressions/aggregate/Min.scala | 2 +- .../sql/catalyst/expressions/arithmetic.scala | 18 ++--- .../expressions/bitwiseExpressions.scala | 8 +-- .../expressions/collectionOperations.scala | 4 +- .../expressions/complexTypeCreator.scala | 2 +- .../expressions/conditionalExpressions.scala | 8 +-- .../expressions/datetimeExpressions.scala | 34 ++++----- .../sql/catalyst/expressions/generators.scala | 2 +- .../expressions/jsonExpressions.scala | 12 ++-- .../expressions/mathExpressions.scala | 70 +++++++++---------- .../spark/sql/catalyst/expressions/misc.scala | 10 +-- .../expressions/nullExpressions.scala | 18 ++--- .../sql/catalyst/expressions/predicates.scala | 16 ++--- .../expressions/randomExpressions.scala | 6 +- .../expressions/regexpExpressions.scala | 14 ++-- .../expressions/stringExpressions.scala | 70 ++++++++++--------- .../expressions/windowExpressions.scala | 26 ++++--- .../sql/execution/command/functions.scala | 10 +-- .../org/apache/spark/sql/SQLQuerySuite.scala | 4 +- .../sql/execution/command/DDLSuite.scala | 10 +-- .../sql/hive/execution/SQLQuerySuite.scala | 16 ++--- 30 files changed, 205 insertions(+), 197 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala index 51b24ba6dbc2..015a53f7fbdb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala @@ -44,12 +44,12 @@ import org.apache.spark.util.Utils * and the remaining are input arguments to the Java method. */ @ExpressionDescription( - usage = "_FUNC_(class, method[, arg1[, arg2 ..]]) - Calls method with reflection.", + usage = "_FUNC_(class, method[, arg1[, arg2 ..]]) - Calls a method with reflection.", extended = """ Arguments: class - a string literal that represents a fully-qualified class name. method - a string literal that represents a method name. - arg - a boolean, string or numeric expression except decimal that represents arguments for + arg - a boolean, string or numeric expression except decimal that represents an argument for the method. Examples: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index ddd63806be41..34a7f9404545 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -114,11 +114,11 @@ object Cast { /** Cast the child expression to the target data type. */ @ExpressionDescription( - usage = "_FUNC_(expr AS type) - Casts the value expr to the target data type.", + usage = "_FUNC_(expr AS type) - Casts the value `expr` to the target data type `type`.", extended = """ Arguments: expr - an expression of any type. - type - any data type to cast expr into. + type - any data type to cast `expr` into. Examples: > SELECT _FUNC_('10' as int); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala index bb14ebdd06b0..01bb2f5107b1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala @@ -34,11 +34,11 @@ import org.apache.spark.sql.types.{DataType, LongType} */ @ExpressionDescription( usage = """ - _FUNC_() - Returns monotonically increasing 64-bit integers. - The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive. - The current implementation puts the partition ID in the upper 31 bits, and the lower 33 bits - represent the record number within each partition. The assumption is that the data frame has - less than 1 billion partitions, and each partition has less than 8 billion records. + _FUNC_() - Returns monotonically increasing 64-bit integers. The generated ID is guaranteed + to be monotonically increasing and unique, but not consecutive. The current implementation + puts the partition ID in the upper 31 bits, and the lower 33 bits represent the record number + within each partition. The assumption is that the data frame has less than 1 billion + partitions, and each partition has less than 8 billion records. """) case class MonotonicallyIncreasingID() extends LeafExpression with Nondeterministic { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala index b016e34401ae..af4c5e6ca999 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala @@ -52,10 +52,10 @@ import org.apache.spark.sql.types._ usage = """ _FUNC_(col, percentage [, accuracy]) - Returns the approximate percentile value of numeric column `col` at the given percentage. The value of percentage must be between 0.0 - and 1.0. The `accuracy` parameter (default: 10000) is a positive integer literal which + and 1.0. The `accuracy` parameter (default: 10000) is a positive numeric literal which controls approximation accuracy at the cost of memory. Higher value of `accuracy` yields better accuracy, `1.0/accuracy` is the relative error of the approximation. - When percentage is an array, each value of the percentage array must be between 0.0 and 1.0. + When `percentage` is an array, each value of the percentage array must be between 0.0 and 1.0. In this case, returns the approximate percentile array of column `col` at the given percentage array. """, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala index 2672b269af03..5375498705c5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Count.scala @@ -24,11 +24,11 @@ import org.apache.spark.sql.types._ // scalastyle:off line.size.limit @ExpressionDescription( usage = """ - _FUNC_(*) - Returns the total number of retrieved rows, including rows containing NULL. + _FUNC_(*) - Returns the total number of retrieved rows, including rows containing null. - _FUNC_(expr) - Returns the number of rows for which the supplied expression is non-NULL. + _FUNC_(expr) - Returns the number of rows for which the supplied expression is non-null. - _FUNC_(DISTINCT expr[, expr...]) - Returns the number of rows for which the supplied expression(s) are unique and non-NULL. + _FUNC_(DISTINCT expr[, expr...]) - Returns the number of rows for which the supplied expression(s) are unique and non-null. """, extended = """ Arguments: diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala index 779e8f5c7281..038d4125d5d5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/First.scala @@ -31,12 +31,12 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = """ _FUNC_(expr[, isIgnoreNull]) - Returns the first value of `expr` for a group of rows. - If isIgnoreNull is true, returns only non-null values. + If `isIgnoreNull` is true, returns only non-null values. """, extended = """ Arguments: expr - an expression of any type that represents data to collect the first. - isIgnoreNull - a boolean literal. If isIgnoreNull is true, returns only non-null + isIgnoreNull - a boolean literal. If `isIgnoreNull` is true, returns only non-null values. Default is false. """) case class First(child: Expression, ignoreNullsExpr: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala index c17255343565..0aaa6a1a071d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/HyperLogLogPlusPlus.scala @@ -49,12 +49,12 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = """ _FUNC_(expr[, relativeSD]) - Returns the estimated cardinality by HyperLogLog++. - relativeSD defines the maximum estimation error allowed. + `relativeSD` defines the maximum estimation error allowed. """, extended = """ Arguments: expr - an expression of any type that represents data to count. - relativeSD - a numeric literal that defines the maximum estimation error allowed. + relativeSD - a double or decimal literal that defines the maximum estimation error allowed. Default is 0.05. """) case class HyperLogLogPlusPlus( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala index ae34d06145a5..cc176fefaf7e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Last.scala @@ -31,12 +31,12 @@ import org.apache.spark.sql.types._ @ExpressionDescription( usage = """ _FUNC_(expr[, isIgnoreNull]) - Returns the last value of `expr` for a group of rows. - If isIgnoreNull is true, returns only non-null values. + If `isIgnoreNull` is true, returns only non-null values. """, extended = """ Arguments: expr - an expression of any type that represents data to collect the last. - isIgnoreNull - a boolean literal. If isIgnoreNull is true, returns only non-null + isIgnoreNull - a boolean literal. If `isIgnoreNull` is true, returns only non-null values. Default is false. """) case class Last(child: Expression, ignoreNullsExpr: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala index dcaab5239b61..48af54e9d587 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.types._ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the maximum value of expr.", + usage = "_FUNC_(expr) - Returns the maximum value of `expr`.", extended = """ Arguments: expr - an expression of any type. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala index ddaaf9a91d2a..0b02b2ca7a68 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.catalyst.util.TypeUtils import org.apache.spark.sql.types._ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the minimum value of expr.", + usage = "_FUNC_(expr) - Returns the minimum value of `expr`.", extended = """ Arguments: expr - an expression of any type. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index e70ecf58dc02..a062909144fe 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.CalendarInterval @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the negated value of expr.", + usage = "_FUNC_(expr) - Returns the negated value of `expr`.", extended = """ Arguments: expr - a numeric or interval expression. @@ -69,7 +69,7 @@ case class UnaryMinus(child: Expression) extends UnaryExpression } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the value of expr.", + usage = "_FUNC_(expr) - Returns the value of `expr`.", extended = """ Arguments: expr - a numeric or interval expression. @@ -149,7 +149,7 @@ object BinaryArithmetic { } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns expr1+expr2.", + usage = "expr1 _FUNC_ expr2 - Returns `expr1`+`expr2`.", extended = """ Arguments: expr1 - a numeric or interval expression. @@ -189,7 +189,7 @@ case class Add(left: Expression, right: Expression) extends BinaryArithmetic wit } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns expr1-expr2.", + usage = "expr1 _FUNC_ expr2 - Returns `expr1`-`expr2`.", extended = """ Arguments: expr1 - a numeric or interval expression. @@ -230,7 +230,7 @@ case class Subtract(left: Expression, right: Expression) } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns expr1*expr2.", + usage = "expr1 _FUNC_ expr2 - Returns `expr1`*`expr2`.", extended = """ Arguments: expr1 - a numeric expression. @@ -253,8 +253,9 @@ case class Multiply(left: Expression, right: Expression) protected override def nullSafeEval(input1: Any, input2: Any): Any = numeric.times(input1, input2) } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns expr1/expr2. It always performs floating point division.", + usage = "expr1 _FUNC_ expr2 - Returns `expr1`/`expr2`. It always performs floating point division.", extended = """ Arguments: expr1 - a numeric expression. @@ -266,6 +267,7 @@ case class Multiply(left: Expression, right: Expression) > SELECT 2L _FUNC_ 2L; 1.0 """) +// scalastyle:on line.size.limit case class Divide(left: Expression, right: Expression) extends BinaryArithmetic with NullIntolerant { @@ -341,7 +343,7 @@ case class Divide(left: Expression, right: Expression) } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns the remainder after expr1/expr2.", + usage = "expr1 _FUNC_ expr2 - Returns the remainder after `expr1`/`expr2`.", extended = """ Arguments: expr1 - a numeric expression. @@ -431,7 +433,7 @@ case class Remainder(left: Expression, right: Expression) } @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns the positive value of expr1 mod expr2.", + usage = "_FUNC_(expr1, expr2) - Returns the positive value of `expr1` mod `expr2`.", extended = """ Arguments: expr1 - a numeric expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index b097021e93eb..be4693f50e45 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.types._ * Code generation inherited from BinaryArithmetic. */ @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise AND of expr1 and expr2.", + usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise AND of `expr1` and `expr2`.", extended = """ Arguments: expr1 - an integral expression. @@ -63,7 +63,7 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme * Code generation inherited from BinaryArithmetic. */ @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise OR of expr1 and expr2.", + usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise OR of `expr1` and `expr2`.", extended = """ Arguments: expr1 - an integral expression. @@ -99,7 +99,7 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet * Code generation inherited from BinaryArithmetic. */ @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise exclusive OR of expr1 and expr2.", + usage = "expr1 _FUNC_ expr2 - Returns the result of bitwise exclusive OR of `expr1` and `expr2`.", extended = """ Arguments: expr1 - an integral expression. @@ -133,7 +133,7 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme * A function that calculates bitwise not(~) of a number. */ @ExpressionDescription( - usage = "_FUNC_ expr - Returns the result of bitwise NOT of expr.", + usage = "_FUNC_ expr - Returns the result of bitwise NOT of `expr`.", extended = """ Arguments: expr - an integral expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index ef10d8ec3d21..e7be291509df 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -223,11 +223,11 @@ case class SortArray(base: Expression, ascendingOrder: Expression) * Checks if the array (left) has the element (right) */ @ExpressionDescription( - usage = "_FUNC_(array, value) - Returns TRUE if the array contains the value.", + usage = "_FUNC_(array, value) - Returns true if the array contains the value.", extended = """ Arguments: array - an array expression. - value - an expression of the element type of array. + value - an expression of the element type of `array`. Examples: > SELECT _FUNC_(array(1, 2, 3), 2); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index 39769d8e23cf..d4baa8436106 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -433,7 +433,7 @@ case class CreateNamedStructUnsafe(children: Seq[Expression]) extends Expression */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(text[, pairDelim[, keyValueDelim]]) - Creates a map after splitting the text into key/value pairs using delimiters. Default delimiters are ',' for pairDelim and ':' for keyValueDelim.", + usage = "_FUNC_(text[, pairDelim[, keyValueDelim]]) - Creates a map after splitting the text into key/value pairs using delimiters. Default delimiters are ',' for `pairDelim` and ':' for `keyValueDelim`.", extended = """ Arguments: text - a string expression that represents data to convert into map. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala index 9d764d8f966d..0464e69f2db5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/conditionalExpressions.scala @@ -24,12 +24,12 @@ import org.apache.spark.sql.types._ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr1, expr2, expr3) - If expr1 evaluates to true, then IF() returns expr2; otherwise it returns expr3.", + usage = "_FUNC_(expr1, expr2, expr3) - If `expr1` evaluates to true, then returns `expr2`; otherwise it returns `expr3`.", extended = """ Arguments: expr1 - a boolean expression. - expr2 - an expression of any type that represents the return value when expr1 evaluates to true. - expr3 - an expression of any type that represents the return value when expr1 evaluates to false. + expr2 - an expression of any type that represents the return value when `expr1` evaluates to true. + expr3 - an expression of any type that represents the return value when `expr1` evaluates to false. Examples: > SELECT _FUNC_(1 < 2, 'a', 'b'); @@ -172,7 +172,7 @@ abstract class CaseWhenBase( */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "CASE WHEN expr1 THEN expr2 [WHEN expr3 THEN expr4]* [ELSE expr5] END - When expr1 = true, returns expr2; when expr3 = true, return expr4; else return expr5.", + usage = "CASE WHEN expr1 THEN expr2 [WHEN expr3 THEN expr4]* [ELSE expr5] END - When `expr1` = true, returns `expr2`; when `expr3` = true, return `expr4`; else return `expr5`.", extended = """ Arguments: expr1 - a boolean expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 4aa0ed94bcfb..2e479a8174ab 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -75,7 +75,7 @@ case class CurrentTimestamp() extends LeafExpression with CodegenFallback { * Adds a number of days to startdate. */ @ExpressionDescription( - usage = "_FUNC_(start_date, num_days) - Returns the date that is num_days after start_date.", + usage = "_FUNC_(start_date, num_days) - Returns the date that is `num_days` after `start_date`.", extended = """ Arguments: start_date - a date expression. @@ -112,7 +112,7 @@ case class DateAdd(startDate: Expression, days: Expression) * Subtracts a number of days to startdate. */ @ExpressionDescription( - usage = "_FUNC_(start_date, num_days) - Returns the date that is num_days before start_date.", + usage = "_FUNC_(start_date, num_days) - Returns the date that is `num_days` before `start_date`.", extended = """ Arguments: start_date - a date expression. @@ -145,7 +145,7 @@ case class DateSub(startDate: Expression, days: Expression) } @ExpressionDescription( - usage = "_FUNC_(timestamp) - Returns the hour component of the string/timestamp/interval.", + usage = "_FUNC_(timestamp) - Returns the hour component of the string/timestamp.", extended = """ Arguments: timestamp - a timestamp expression. @@ -171,7 +171,7 @@ case class Hour(child: Expression) extends UnaryExpression with ImplicitCastInpu } @ExpressionDescription( - usage = "_FUNC_(timestamp) - Returns the minute component of the string/timestamp/interval.", + usage = "_FUNC_(timestamp) - Returns the minute component of the string/timestamp.", extended = """ Arguments: timestamp - a timestamp expression. @@ -197,7 +197,7 @@ case class Minute(child: Expression) extends UnaryExpression with ImplicitCastIn } @ExpressionDescription( - usage = "_FUNC_(timestamp) - Returns the second component of the string/timestamp/interval.", + usage = "_FUNC_(timestamp) - Returns the second component of the string/timestamp.", extended = """ Arguments: timestamp - a timestamp expression. @@ -249,7 +249,7 @@ case class DayOfYear(child: Expression) extends UnaryExpression with ImplicitCas } @ExpressionDescription( - usage = "_FUNC_(date) - Returns the year component of the date/timestamp/interval.", + usage = "_FUNC_(date) - Returns the year component of the date/timestamp.", extended = """ Arguments: date - a date expression. @@ -301,7 +301,7 @@ case class Quarter(child: Expression) extends UnaryExpression with ImplicitCastI } @ExpressionDescription( - usage = "_FUNC_(date) - Returns the month component of the date/timestamp/interval.", + usage = "_FUNC_(date) - Returns the month component of the date/timestamp.", extended = """ Arguments: date - a date expression. @@ -327,7 +327,7 @@ case class Month(child: Expression) extends UnaryExpression with ImplicitCastInp } @ExpressionDescription( - usage = "_FUNC_(date) - Returns the day of month of date/timestamp, or the day of interval.", + usage = "_FUNC_(date) - Returns the day of month of date/timestamp.", extended = """ Arguments: date - a date expression. @@ -400,11 +400,11 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp, fmt) - Converts timestamp to a value of string in the format specified by the date format fmt.", + usage = "_FUNC_(timestamp, fmt) - Converts `timestamp` to a value of string in the format specified by the date format `fmt`.", extended = """ Arguments: timestamp - a timestamp expression. - fmt - string type expression that represents the date/timestamp format available in java.text.SimpleDateFormat. + fmt - a string type expression that represents the date/timestamp format available in java.text.SimpleDateFormat. Examples: > SELECT _FUNC_('2016-04-08', 'y'); @@ -609,7 +609,7 @@ abstract class UnixTime extends BinaryExpression with ExpectsInputTypes { * Note that hive Language Manual says it returns 0 if fail, but in fact it returns null. */ @ExpressionDescription( - usage = "_FUNC_(unix_time, format) - Returns unix_time in the specified format.", + usage = "_FUNC_(unix_time, format) - Returns `unix_time` in the specified `format`.", extended = """ Arguments: unix_time - a numeric expression. @@ -742,7 +742,7 @@ case class LastDay(startDate: Expression) extends UnaryExpression with ImplicitC */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(start_date, day_of_week) - Returns the first date which is later than start_date and named as indicated.", + usage = "_FUNC_(start_date, day_of_week) - Returns the first date which is later than `start_date` and named as indicated.", extended = """ Arguments: start_date - a date expression. @@ -931,8 +931,9 @@ case class TimeSub(start: Expression, interval: Expression) /** * Returns the date that is num_months after start_date. */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(start_date, num_months) - Returns the date that is num_months after start_date.", + usage = "_FUNC_(start_date, num_months) - Returns the date that is `num_months` after `start_date`.", extended = """ Arguments: start_date - a date expression. @@ -942,6 +943,7 @@ case class TimeSub(start: Expression, interval: Expression) > SELECT _FUNC_('2016-08-31', 1); 2016-09-30 """) +// scalastyle:on line.size.limit case class AddMonths(startDate: Expression, numMonths: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -971,7 +973,7 @@ case class AddMonths(startDate: Expression, numMonths: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(timestamp1, timestamp2) - Returns number of months between dates timestamp1 and timestamp2.", + usage = "_FUNC_(timestamp1, timestamp2) - Returns number of months between `timestamp1` and `timestamp2`.", extended = """ Arguments: timestamp1 - a timestamp expression. @@ -1103,7 +1105,7 @@ case class ToDate(child: Expression) extends UnaryExpression with ImplicitCastIn */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(date, fmt) - Returns returns date with the time portion of the day truncated to the unit specified by the format model fmt.", + usage = "_FUNC_(date, fmt) - Returns returns `date` with the time portion of the day truncated to the unit specified by the format model `fmt`.", extended = """ Arguments: date - a date expression. @@ -1186,7 +1188,7 @@ case class TruncDate(date: Expression, format: Expression) * Returns the number of days from startDate to endDate. */ @ExpressionDescription( - usage = "_FUNC_(date1, date2) - Returns the number of days between date1 and date2.", + usage = "_FUNC_(date1, date2) - Returns the number of days between `date1` and `date2`.", extended = """ Arguments: date1 - a date expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index d54139de82d3..cc32d12b119f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -102,7 +102,7 @@ case class UserDefinedGenerator( * }}} */ @ExpressionDescription( - usage = "_FUNC_(n, expr1, ..., exprk) - Separate expr1, ..., exprk into n rows.", + usage = "_FUNC_(n, expr1, ..., exprk) - Separates `expr1`, ..., `exprk` into `n` rows.", extended = """ Arguments: n - an integer literal that represents the number of output rows. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index 0800c87846b3..b464ae4bca55 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -109,11 +109,11 @@ private[this] object SharedFactory { * of the extracted json object. It will return null if the input json string is invalid. */ @ExpressionDescription( - usage = "_FUNC_(json_txt, path) - Extracts a json object from path.", + usage = "_FUNC_(json_txt, path) - Extracts a json object from `path`.", extended = """ Arguments: - json_txt - a string expression that represents JSON document. - path - a string expression that represents path for JSON document. + json_txt - a string expression that represents a JSON document. + path - a string expression that represents path for a JSON document. Examples: > SELECT _FUNC_('{"a":"b"}', '$.a'); @@ -334,11 +334,11 @@ case class GetJsonObject(json: Expression, path: Expression) // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(jsonStr, p1, p2, ..., pn) - like get_json_object, but it takes multiple names and return a tuple. All the input parameters and output column types are string.", + usage = "_FUNC_(jsonStr, p1, p2, ..., pn) - Return a tuple like the function get_json_object, but it takes multiple names. All the input parameters and output column types are string.", extended = """ Arguments: - json_txt - a string expression that represents JSON document. - p - a string expression that represents key name for JSON document. + json_txt - a string expression that represents a JSON document. + p - a string expression that represents key name for a JSON document. Examples: > SELECT _FUNC_('{"a":1, "b":2}', 'a', 'b'); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index ac8ec7a33521..3ccdca52bac8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -168,7 +168,7 @@ case class Pi() extends LeafMathExpression(math.Pi, "PI") // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the inverse cosine (a.k.a. arccosine) of expr if -1<=expr<=1 or NaN otherwise.", + usage = "_FUNC_(expr) - Returns the inverse cosine (a.k.a. arccosine) of `expr` if -1<=`expr`<=1 or NaN otherwise.", extended = """ Arguments: expr - a numeric expression. @@ -184,7 +184,7 @@ case class Acos(child: Expression) extends UnaryMathExpression(math.acos, "ACOS" // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the inverse sine (a.k.a. arcsine) the arc sin of expr if -1<=expr<=1 or NaN otherwise.", + usage = "_FUNC_(expr) - Returns the inverse sine (a.k.a. arcsine) the arc sin of `expr` if -1<=`expr`<=1 or NaN otherwise.", extended = """ Arguments: expr - a numeric expression. @@ -213,7 +213,7 @@ case class Asin(child: Expression) extends UnaryMathExpression(math.asin, "ASIN" case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the cube root of expr.", + usage = "_FUNC_(expr) - Returns the cube root of `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -225,7 +225,7 @@ case class Atan(child: Expression) extends UnaryMathExpression(math.atan, "ATAN" case class Cbrt(child: Expression) extends UnaryMathExpression(math.cbrt, "CBRT") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the smallest integer not smaller than expr.", + usage = "_FUNC_(expr) - Returns the smallest integer not smaller than `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -263,7 +263,7 @@ case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil, "CEIL" } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the cosine of expr.", + usage = "_FUNC_(expr) - Returns the cosine of `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -275,7 +275,7 @@ case class Ceil(child: Expression) extends UnaryMathExpression(math.ceil, "CEIL" case class Cos(child: Expression) extends UnaryMathExpression(math.cos, "COS") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the hyperbolic cosine of expr.", + usage = "_FUNC_(expr) - Returns the hyperbolic cosine of `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -294,7 +294,7 @@ case class Cosh(child: Expression) extends UnaryMathExpression(math.cosh, "COSH" * @param toBaseExpr to which base */ @ExpressionDescription( - usage = "_FUNC_(num, from_base, to_base) - Convert num from from_base to to_base.", + usage = "_FUNC_(num, from_base, to_base) - Convert `num` from `from_base` to `to_base`.", extended = """ Arguments: num - a string expression. @@ -336,7 +336,7 @@ case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expre } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns e to the power of expr.", + usage = "_FUNC_(expr) - Returns e to the power of `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -348,7 +348,7 @@ case class Conv(numExpr: Expression, fromBaseExpr: Expression, toBaseExpr: Expre case class Exp(child: Expression) extends UnaryMathExpression(math.exp, "EXP") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns exp(expr) - 1.", + usage = "_FUNC_(expr) - Returns exp(`expr`) - 1.", extended = """ Arguments: expr - a numeric expression. @@ -360,7 +360,7 @@ case class Exp(child: Expression) extends UnaryMathExpression(math.exp, "EXP") case class Expm1(child: Expression) extends UnaryMathExpression(math.expm1, "EXPM1") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the largest integer not greater than expr.", + usage = "_FUNC_(expr) - Returns the largest integer not greater than `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -429,7 +429,7 @@ object Factorial { } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns expr factorial for expr is [0..20]. Otherwise, NULL.", + usage = "_FUNC_(expr) - Returns the factorial of `expr`. `expr` is [0..20]. Otherwise, null.", extended = """ Arguments: expr - a numeric expression. @@ -471,7 +471,7 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the natural logarithm (base e) of expr.", + usage = "_FUNC_(expr) - Returns the natural logarithm (base e) of `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -483,7 +483,7 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas case class Log(child: Expression) extends UnaryLogExpression(math.log, "LOG") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the logarithm of expr with base 2.", + usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 2.", extended = """ Arguments: expr - a numeric expression. @@ -508,7 +508,7 @@ case class Log2(child: Expression) } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the logarithm of expr with base 10.", + usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 10.", extended = """ Arguments: expr - a numeric expression. @@ -520,7 +520,7 @@ case class Log2(child: Expression) case class Log10(child: Expression) extends UnaryLogExpression(math.log10, "LOG10") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns log(1 + expr).", + usage = "_FUNC_(expr) - Returns log(1 + `expr`).", extended = """ Arguments: expr - a numeric expression. @@ -550,7 +550,7 @@ case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND } @ExpressionDescription( - usage = "_FUNC_(expr) - Returns -1.0, 0.0 or 1.0 as expr is negative, 0 or positive.", + usage = "_FUNC_(expr) - Returns -1.0, 0.0 or 1.0 as `expr` is negative, 0 or positive.", extended = """ Arguments: expr - a numeric expression. @@ -562,7 +562,7 @@ case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND case class Signum(child: Expression) extends UnaryMathExpression(math.signum, "SIGNUM") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the sine of expr.", + usage = "_FUNC_(expr) - Returns the sine of `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -574,7 +574,7 @@ case class Signum(child: Expression) extends UnaryMathExpression(math.signum, "S case class Sin(child: Expression) extends UnaryMathExpression(math.sin, "SIN") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the hyperbolic sine of expr.", + usage = "_FUNC_(expr) - Returns the hyperbolic sine of `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -586,7 +586,7 @@ case class Sin(child: Expression) extends UnaryMathExpression(math.sin, "SIN") case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the square root of expr.", + usage = "_FUNC_(expr) - Returns the square root of `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -598,7 +598,7 @@ case class Sinh(child: Expression) extends UnaryMathExpression(math.sinh, "SINH" case class Sqrt(child: Expression) extends UnaryMathExpression(math.sqrt, "SQRT") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the tangent of expr.", + usage = "_FUNC_(expr) - Returns the tangent of `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -610,7 +610,7 @@ case class Sqrt(child: Expression) extends UnaryMathExpression(math.sqrt, "SQRT" case class Tan(child: Expression) extends UnaryMathExpression(math.tan, "TAN") @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the hyperbolic tangent of expr.", + usage = "_FUNC_(expr) - Returns the hyperbolic tangent of `expr`.", extended = """ Arguments: expr - a numeric expression. @@ -757,7 +757,7 @@ object Hex { * and returns the resulting STRING. Negative numbers would be treated as two's complement. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Converts the expr to hexadecimal.", + usage = "_FUNC_(expr) - Converts `expr` to hexadecimal.", extended = """ Arguments: expr - a numeric, binary or string expression. @@ -797,7 +797,7 @@ case class Hex(child: Expression) extends UnaryExpression with ImplicitCastInput * Resulting characters are returned as a byte array. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Converts hexadecimal expr to binary.", + usage = "_FUNC_(expr) - Converts hexadecimal `expr` to binary.", extended = """ Arguments: expr - a string expression. @@ -836,7 +836,7 @@ case class Unhex(child: Expression) extends UnaryExpression with ImplicitCastInp // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns the angle in radians between the positive x-axis of a plane and the point given by the coordinates (expr1, expr2).", + usage = "_FUNC_(expr1, expr2) - Returns the angle in radians between the positive x-axis of a plane and the point given by the coordinates (`expr1`, `expr2`).", extended = """ Arguments: expr1 - a numeric expression. @@ -861,7 +861,7 @@ case class Atan2(left: Expression, right: Expression) } @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Raise expr1 to the power of expr2.", + usage = "_FUNC_(expr1, expr2) - Raises `expr1` to the power of `expr2`.", extended = """ Arguments: expr1 - a numeric expression. @@ -889,8 +889,8 @@ case class Pow(left: Expression, right: Expression) usage = "_FUNC_(base, expr) - Bitwise left shift.", extended = """ Arguments: - base - an integral expression that defines the base number to shift. - expr - an integral expression. + base - a numeric expression that defines the base number to shift. + expr - a numeric expression. Examples: > SELECT _FUNC_(2, 1); @@ -927,8 +927,8 @@ case class ShiftLeft(left: Expression, right: Expression) usage = "_FUNC_(base, expr) - Bitwise (signed) right shift.", extended = """ Arguments: - base - an integral expression that defines the base number to shift. - expr - an integral expression. + base - a numeric expression that defines the base number to shift. + expr - a numeric expression. Examples: > SELECT _FUNC_(4, 1); @@ -965,8 +965,8 @@ case class ShiftRight(left: Expression, right: Expression) usage = "_FUNC_(base, expr) - Bitwise unsigned right shift.", extended = """ Arguments: - base - an integral expression that defines the base number to shift. - expr - an integral expression. + base - a numeric expression that defines the base number to shift. + expr - a numeric expression. Examples: > SELECT _FUNC_(4, 1); @@ -993,7 +993,7 @@ case class ShiftRightUnsigned(left: Expression, right: Expression) } @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns sqrt(expr1**2 + expr2**2).", + usage = "_FUNC_(expr1, expr2) - Returns sqrt(`expr1`**2 + `expr2`**2).", extended = """ Arguments: expr1 - a numeric expression. @@ -1014,7 +1014,7 @@ case class Hypot(left: Expression, right: Expression) * @param right the number to compute the logarithm of. */ @ExpressionDescription( - usage = "_FUNC_(base, expr) - Returns the logarithm of expr with base.", + usage = "_FUNC_(base, expr) - Returns the logarithm of `expr` with `base`.", extended = """ Arguments: base - a numeric expression that represents the logarithm base. @@ -1248,7 +1248,7 @@ abstract class RoundBase(child: Expression, scale: Expression, */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr, d) - Returns expr rounded to d decimal places using HALF_UP rounding mode.", + usage = "_FUNC_(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_UP rounding mode.", extended = """ Arguments: expr - a numeric expression. @@ -1272,7 +1272,7 @@ case class Round(child: Expression, scale: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr, d) - Returns expr rounded to d decimal places using HALF_EVEN rounding mode.", + usage = "_FUNC_(expr, d) - Returns `expr` rounded to `d` decimal places using HALF_EVEN rounding mode.", extended = """ Arguments: expr - a numeric expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala index 529ebacd42ef..a8444ed69e2f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala @@ -38,7 +38,7 @@ import org.apache.spark.unsafe.Platform * For input of type [[BinaryType]] */ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns an MD5 128-bit checksum as a hex string of expr.", + usage = "_FUNC_(expr) - Returns an MD5 128-bit checksum as a hex string of `expr`.", extended = """ Arguments: expr - a binary expression. @@ -73,7 +73,7 @@ case class Md5(child: Expression) extends UnaryExpression with ImplicitCastInput // scalastyle:off line.size.limit @ExpressionDescription( usage = """ - _FUNC_(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of expr. + _FUNC_(expr, bitLength) - Returns a checksum of SHA-2 family as a hex string of `expr`. SHA-224, SHA-256, SHA-384, and SHA-512 are supported. Bit length of 0 is equivalent to 256. """, extended = """ @@ -153,7 +153,7 @@ case class Sha2(left: Expression, right: Expression) * For input of type [[BinaryType]] or [[StringType]] */ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns a sha1 hash value as a hex string of the expr.", + usage = "_FUNC_(expr) - Returns a sha1 hash value as a hex string of the `expr`.", extended = """ Arguments: expr - a binary expression. @@ -183,7 +183,7 @@ case class Sha1(child: Expression) extends UnaryExpression with ImplicitCastInpu * For input of type [[BinaryType]] */ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns a cyclic redundancy check value of the expr as a bigint.", + usage = "_FUNC_(expr) - Returns a cyclic redundancy check value of the `expr` as a bigint.", extended = """ Arguments: expr - a binary expression. @@ -583,7 +583,7 @@ case class PrintToStderr(child: Expression) extends UnaryExpression { * A function throws an exception if 'condition' is not true. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Throw an exception if expr is not true.", + usage = "_FUNC_(expr) - Throws an exception if `expr` is not true.", extended = """ Arguments: expr - a boolean expression diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala index bed3ce9b46cd..8333896e8d6b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/nullExpressions.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.types._ */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr1, expr2, ...) - Returns the first non-null argument if exists. Otherwise, NULL.", + usage = "_FUNC_(expr1, expr2, ...) - Returns the first non-null argument if exists. Otherwise, null.", extended = """ Arguments: expr - an expression of any type. @@ -98,7 +98,7 @@ case class Coalesce(children: Seq[Expression]) extends Expression { @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns expr2 if expr1 is null, or expr1 otherwise.", + usage = "_FUNC_(expr1, expr2) - Returns `expr2` if `expr1` is null, or `expr1` otherwise.", extended = """ Arguments: expr1 - an expression of any type. @@ -126,7 +126,7 @@ case class IfNull(left: Expression, right: Expression) extends RuntimeReplaceabl @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns null if expr1 equals to expr2, or expr1 otherwise.", + usage = "_FUNC_(expr1, expr2) - Returns null if `expr1` equals to `expr2`, or `expr1` otherwise.", extended = """ Arguments: expr1 - an expression of any type. @@ -156,7 +156,7 @@ case class NullIf(left: Expression, right: Expression) extends RuntimeReplaceabl @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns expr2 if expr1 is null, or expr1 otherwise.", + usage = "_FUNC_(expr1, expr2) - Returns `expr2` if `expr1` is null, or `expr1` otherwise.", extended = """ Arguments: expr1 - an expression of any type. @@ -185,7 +185,7 @@ case class Nvl(left: Expression, right: Expression) extends RuntimeReplaceable { // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr1, expr2, expr3) - Returns expr2 if expr1 is not null, or expr3 otherwise.", + usage = "_FUNC_(expr1, expr2, expr3) - Returns `expr2` if `expr1` is not null, or `expr3` otherwise.", extended = """ Arguments: expr1 - an expression of any type. @@ -220,7 +220,7 @@ case class Nvl2(expr1: Expression, expr2: Expression, expr3: Expression) * Evaluates to `true` iff it's NaN. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns true if expr is NaN and false otherwise.", + usage = "_FUNC_(expr) - Returns true if `expr` is NaN and false otherwise.", extended = """ Arguments: expr - a numeric expression. @@ -266,7 +266,7 @@ case class IsNaN(child: Expression) extends UnaryExpression * This Expression is useful for mapping NaN values to null. */ @ExpressionDescription( - usage = "_FUNC_(expr1, expr2) - Returns expr1 if it's not NaN, or expr2 otherwise.", + usage = "_FUNC_(expr1, expr2) - Returns `expr1` if it's not NaN, or `expr2` otherwise.", extended = """ Arguments: expr1 - a numeric expression. @@ -330,7 +330,7 @@ case class NaNvl(left: Expression, right: Expression) * An expression that is evaluated to true if the input is null. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns true if expr is NULL and false otherwise.", + usage = "_FUNC_(expr) - Returns true if `expr` is null and false otherwise.", extended = """ Arguments: expr - an expression of any type. @@ -359,7 +359,7 @@ case class IsNull(child: Expression) extends UnaryExpression with Predicate { * An expression that is evaluated to true if the input is not null. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns true if expr is not NULL and false otherwise.", + usage = "_FUNC_(expr) - Returns true if `expr` is not null and false otherwise.", extended = """ Arguments: expr - an expression of any type. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index 7beb5042aca6..984ff0b9bd7a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -118,10 +118,10 @@ case class Not(child: Expression) * Evaluates to `true` if `list` contains `value`. */ @ExpressionDescription( - usage = "expr1 _FUNC_(expr2, expr3, ...) - Returns true if expr equals to any valN.", + usage = "expr1 _FUNC_(expr2, expr3, ...) - Returns true if `expr` equals to any valN.", extended = """ Arguments: - expr - a boolean expression. + expr - an expression of any type. """) case class In(value: Expression, list: Seq[Expression]) extends Predicate with ImplicitCastInputTypes { @@ -427,7 +427,7 @@ object Equality { } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 equals expr2 and false otherwise.", + usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` equals `expr2` and false otherwise.", extended = """ Arguments: expr1 - an expression of any type. @@ -460,7 +460,7 @@ case class EqualTo(left: Expression, right: Expression) @ExpressionDescription( usage = """ expr1 _FUNC_ expr2 - Returns same result as the EQUAL(=) operator for non-null operands, - but returns TRUE if both are NULL, FALSE if one of the them is NULL. + but returns true if both are null, false if one of the them is null. """, extended = """ Arguments: @@ -506,7 +506,7 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is less than expr2.", + usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is less than `expr2`.", extended = """ Arguments: expr1 - an expression of any type. @@ -525,7 +525,7 @@ case class LessThan(left: Expression, right: Expression) } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is less than or equal to expr2.", + usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is less than or equal to `expr2`.", extended = """ Arguments: expr1 - an expression of any type. @@ -544,7 +544,7 @@ case class LessThanOrEqual(left: Expression, right: Expression) } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is greater than expr2.", + usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is greater than `expr2`.", extended = """ Arguments: expr1 - an expression of any type. @@ -563,7 +563,7 @@ case class GreaterThan(left: Expression, right: Expression) } @ExpressionDescription( - usage = "expr1 _FUNC_ expr2 - Returns TRUE if expr1 is greater than or equal to expr2.", + usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is greater than or equal to `expr2`.", extended = """ Arguments: expr1 - an expression of any type. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala index 1574fce8fced..03fe9ff223c7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/randomExpressions.scala @@ -57,10 +57,10 @@ abstract class RDG extends LeafExpression with Nondeterministic { /** Generate a random column with i.i.d. uniformly distributed values in [0, 1). */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.). uniformly distributed values in [0, 1).", + usage = "_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.) uniformly distributed values in [0, 1).", extended = """ Arguments: - seed - a numeric literal. + seed - an integer literal. Examples: > SELECT _FUNC_(); @@ -95,7 +95,7 @@ case class Rand(seed: Long) extends RDG { usage = "_FUNC_([seed]) - Returns a random value with independent and identically distributed (i.i.d.) values drawn from the standard normal distribution.", extended = """ Arguments: - seed - a numeric literal. + seed - an integer literal. Examples: > SELECT _FUNC_(); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 05962f9a0beb..79f14fb368a4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -68,7 +68,7 @@ trait StringRegexExpression extends ImplicitCastInputTypes { * Simple RegEx pattern matching function */ @ExpressionDescription( - usage = "str _FUNC_ pattern - Returns true if str matches pattern and false otherwise.", + usage = "str _FUNC_ pattern - Returns true if `str` matches `pattern` and false otherwise.", extended = """ Arguments: str - a string expression. @@ -126,7 +126,7 @@ case class Like(left: Expression, right: Expression) } @ExpressionDescription( - usage = "str _FUNC_ regexp - Returns true if str matches regexp and false otherwise.", + usage = "str _FUNC_ regexp - Returns true if `str` matches `regexp` and false otherwise.", extended = """ Arguments: str - a string expression. @@ -185,7 +185,7 @@ case class RLike(left: Expression, right: Expression) * Splits str around pat (pattern is a regular expression). */ @ExpressionDescription( - usage = "_FUNC_(str, regex) - Splits str around occurrences that match regex.", + usage = "_FUNC_(str, regex) - Splits `str` around occurrences that match `regex`.", extended = """ Arguments: str - a string expression. @@ -224,8 +224,9 @@ case class StringSplit(str: Expression, pattern: Expression) * * NOTE: this expression is not THREAD-SAFE, as it has some internal mutable status. */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(str, regexp, rep) - Replaces all substrings of str that match regexp with rep.", + usage = "_FUNC_(str, regexp, rep) - Replaces all substrings of `str` that match `regexp` with `rep`.", extended = """ Arguments: str - a string expression. @@ -236,6 +237,7 @@ case class StringSplit(str: Expression, pattern: Expression) > SELECT _FUNC_('100-200', '(\d+)', 'num'); num-num """) +// scalastyle:on line.size.limit case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expression) extends TernaryExpression with ImplicitCastInputTypes { @@ -336,12 +338,12 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio * NOTE: this expression is not THREAD-SAFE, as it has some internal mutable status. */ @ExpressionDescription( - usage = "_FUNC_(str, regexp[, idx]) - Extracts a group that matches regexp.", + usage = "_FUNC_(str, regexp[, idx]) - Extracts a group that matches `regexp`.", extended = """ Arguments: str - a string expression. regexp - a string expression that defines a regular expression pattern. - idx - a numeric expression that defines the index of a capturing group in regexp. + idx - a numeric expression that defines the index of a capturing group in `regexp`. Examples: > SELECT _FUNC_('100-200', '(\d+)-(\d+)', 1); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index c1668b3f2e2f..aed20594630f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -40,8 +40,9 @@ import org.apache.spark.unsafe.types.{ByteArray, UTF8String} * An expression that concatenates multiple input strings into a single string. * If any input is null, concat returns null. */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(str1, str2, ..., strN) - Returns the concatenation of str1, str2, ..., strN.", + usage = "_FUNC_(str1, str2, ..., strN) - Returns the concatenation of `str1`, `str2`, ..., `strN`.", extended = """ Arguments: str - a string expression. @@ -50,6 +51,7 @@ import org.apache.spark.unsafe.types.{ByteArray, UTF8String} > SELECT _FUNC_('Spark','SQL'); SparkSQL """) +// scalastyle:on line.size.limit case class Concat(children: Seq[Expression]) extends Expression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq.fill(children.size)(StringType) @@ -87,7 +89,7 @@ case class Concat(children: Seq[Expression]) extends Expression with ImplicitCas */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(sep, [str | array(str)]+) - Returns the concatenation of the strings separated by sep.", + usage = "_FUNC_(sep, [str | array(str)]+) - Returns the concatenation of the strings separated by `sep`.", extended = """ Arguments: sep - a string expression that represents the separator. @@ -183,17 +185,19 @@ case class ConcatWs(children: Seq[Expression]) } } +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(n, str1, str2, ...) - Returns the n-th string, e.g. returns str2 when n is 2.", + usage = "_FUNC_(n, str1, str2, ...) - Returns the `n`-th string, e.g., returns `str2` when `n` is 2.", extended = """ Arguments: - n - a numeric expression that defines the index of str for a return value. + n - a numeric expression that defines the index of `str` for a return value. str - a string expression. Examples: > SELECT _FUNC_(1, 'scala', 'java'); scala """) +// scalastyle:on line.size.limit case class Elt(children: Seq[Expression]) extends Expression with ImplicitCastInputTypes { @@ -270,7 +274,7 @@ trait String2StringExpression extends ImplicitCastInputTypes { * A function that converts the characters of a string to uppercase. */ @ExpressionDescription( - usage = "_FUNC_(str) - Returns str with all characters changed to uppercase.", + usage = "_FUNC_(str) - Returns `str` with all characters changed to uppercase.", extended = """ Arguments: str - a string expression. @@ -293,7 +297,7 @@ case class Upper(child: Expression) * A function that converts the characters of a string to lowercase. */ @ExpressionDescription( - usage = "_FUNC_(str) - Returns str with all characters changed to lowercase.", + usage = "_FUNC_(str) - Returns `str` with all characters changed to lowercase.", extended = """ Arguments: str - a string expression. @@ -456,7 +460,7 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac @ExpressionDescription( usage = """ _FUNC_(str, str_array) - Returns the index (1-based) of the given string (left) in the comma-delimited list (right). - Returns 0, if the string wasn't found or if the given string (left) contains a comma. + Returns 0, if the string was not found or if the given string (left) contains a comma. """, extended = """ Arguments: @@ -491,7 +495,7 @@ case class FindInSet(left: Expression, right: Expression) extends BinaryExpressi * A function that trim the spaces from both ends for the specified string. */ @ExpressionDescription( - usage = "_FUNC_(str) - Removes the leading and trailing space characters from str.", + usage = "_FUNC_(str) - Removes the leading and trailing space characters from `str`.", extended = """ Arguments: str - a string expression. @@ -516,7 +520,7 @@ case class StringTrim(child: Expression) * A function that trim the spaces from left end for given string. */ @ExpressionDescription( - usage = "_FUNC_(str) - Removes the leading and trailing space characters from str.", + usage = "_FUNC_(str) - Removes the leading and trailing space characters from `str`.", extended = """ Arguments: str - a string expression. @@ -541,7 +545,7 @@ case class StringTrimLeft(child: Expression) * A function that trim the spaces from right end for given string. */ @ExpressionDescription( - usage = "_FUNC_(str) - Removes the trailing space characters from str.", + usage = "_FUNC_(str) - Removes the trailing space characters from `str`.", extended = """ Arguments: str - a string expression. @@ -570,7 +574,7 @@ case class StringTrimRight(child: Expression) * NOTE: that this is not zero based, but 1-based index. The first character in str has index 1. */ @ExpressionDescription( - usage = "_FUNC_(str, substr) - Returns the (1-based) index of the first occurrence of substr in str.", + usage = "_FUNC_(str, substr) - Returns the (1-based) index of the first occurrence of `substr` in `str`.", extended = """ Arguments: str - a string expression. @@ -609,17 +613,17 @@ case class StringInstr(str: Expression, substr: Expression) // scalastyle:off line.size.limit @ExpressionDescription( usage = """ - _FUNC_(str, delim, count) - Returns the substring from str before count occurrences of the delimiter delim. - If count is positive, everything to the left of the final delimiter (counting from the - left) is returned. If count is negative, everything to the right of the final delimiter + _FUNC_(str, delim, count) - Returns the substring from `str` before `count` occurrences of the delimiter `delim`. + If `count` is positive, everything to the left of the final delimiter (counting from the + left) is returned. If `count` is negative, everything to the right of the final delimiter (counting from the right) is returned. The function substring_index performs a case-sensitive match - when searching for delim. + when searching for `delim`. """, extended = """ Arguments: str - a string expression. delim - a string expression that represents the delimiter. - count - a numeric expression that defines the number of occurrences of the delimiter delim . + count - a numeric expression that defines the number of occurrences of the delimiter `delim`. Examples: > SELECT _FUNC_('www.apache.org', '.', 2); @@ -652,8 +656,8 @@ case class SubstringIndex(strExpr: Expression, delimExpr: Expression, countExpr: // scalastyle:off line.size.limit @ExpressionDescription( usage = """ - _FUNC_(substr, str[, pos]) - Returns the position of the first occurrence of substr in str after position pos. - The given pos and return value are 1-based. + _FUNC_(substr, str[, pos]) - Returns the position of the first occurrence of `substr` in `str` after position `pos`. + The given `pos` and return value are 1-based. """, extended = """ Arguments: @@ -740,8 +744,8 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression) */ @ExpressionDescription( usage = """ - _FUNC_(str, len, pad) - Returns str, left-padded with pad to a length of len. - If str is longer than len, the return value is shortened to len characters. + _FUNC_(str, len, pad) - Returns `str`, left-padded with `pad` to a length of `len`. + If `str` is longer than `len`, the return value is shortened to `len` characters. """, extended = """ Arguments: @@ -778,8 +782,8 @@ case class StringLPad(str: Expression, len: Expression, pad: Expression) */ @ExpressionDescription( usage = """ - _FUNC_(str, len, pad) - Returns str, right-padded with pad to a length of len. - If str is longer than len, the return value is shortened to len characters. + _FUNC_(str, len, pad) - Returns `str`, right-padded with `pad` to a length of `len`. + If `str` is longer than `len`, the return value is shortened to `len` characters. """, extended = """ Arguments: @@ -1070,7 +1074,7 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC */ @ExpressionDescription( usage = """ - _FUNC_(str) - Returns str with the first letter of each word in uppercase. + _FUNC_(str) - Returns `str` with the first letter of each word in uppercase. All other letters are in lowercase. Words are delimited by white space. """, extended = """ @@ -1098,7 +1102,7 @@ case class InitCap(child: Expression) extends UnaryExpression with ImplicitCastI * Returns the string which repeat the given string value n times. */ @ExpressionDescription( - usage = "_FUNC_(str, n) - Returns the string which repeat the given string value n times.", + usage = "_FUNC_(str, n) - Returns the string which repeats the given string value n times.", extended = """ Arguments: str - a string expression. @@ -1154,7 +1158,7 @@ case class StringReverse(child: Expression) extends UnaryExpression with String2 * Returns a string consisting of n spaces. */ @ExpressionDescription( - usage = "_FUNC_(n) - Returns a string consisting of n spaces.", + usage = "_FUNC_(n) - Returns a string consisting of `n` spaces.", extended = """ Arguments: n - a numeric expression that defines the number of spaces. @@ -1190,7 +1194,7 @@ case class StringSpace(child: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(str, pos[, len]) - Returns the substring of str that starts at pos and is of length len or the slice of byte array that starts at pos and is of length len.", + usage = "_FUNC_(str, pos[, len]) - Returns the substring of `str` that starts at `pos` and is of length `len` or the slice of byte array that starts at `pos` and is of length `len`.", extended = """ Arguments: str - a string expression. @@ -1244,7 +1248,7 @@ case class Substring(str: Expression, pos: Expression, len: Expression) * A function that return the length of the given string or binary expression. */ @ExpressionDescription( - usage = "_FUNC_(expr) - Returns the length of str or number of bytes in binary data.", + usage = "_FUNC_(expr) - Returns the length of `str` or number of bytes in binary data.", extended = """ Arguments: expr - a string or binary expression. @@ -1329,7 +1333,7 @@ case class SoundEx(child: Expression) extends UnaryExpression with ExpectsInputT * Returns the numeric value of the first character of str. */ @ExpressionDescription( - usage = "_FUNC_(str) - Returns the numeric value of the first character of str.", + usage = "_FUNC_(str) - Returns the numeric value of the first character of `str`.", extended = """ Arguments: str - a string expression. @@ -1372,7 +1376,7 @@ case class Ascii(child: Expression) extends UnaryExpression with ImplicitCastInp * Converts the argument from binary to a base 64 string. */ @ExpressionDescription( - usage = "_FUNC_(bin) - Converts the argument from binary to a base 64 string.", + usage = "_FUNC_(bin) - Converts the argument from a binary `bin` to a base 64 string.", extended = """ Arguments: bin - a binary expression. @@ -1404,7 +1408,7 @@ case class Base64(child: Expression) extends UnaryExpression with ImplicitCastIn * Converts the argument from a base 64 string to BINARY. */ @ExpressionDescription( - usage = "_FUNC_(str) - Convert the argument from a base 64 string to binary.", + usage = "_FUNC_(str) - Convert the argument from a base 64 string `str` to a binary.", extended = """ Arguments: str - a string expression. @@ -1513,8 +1517,8 @@ case class Encode(value: Expression, charset: Expression) */ @ExpressionDescription( usage = """ - _FUNC_(expr1, expr2) - Formats the number expr1 like '#,###,###.##', rounded to expr2 - decimal places. If expr2 is 0, the result has no decimal point or fractional part. + _FUNC_(expr1, expr2) - Formats the number `expr1` like '#,###,###.##', rounded to `expr2` + decimal places. If `expr2` is 0, the result has no decimal point or fractional part. This is supposed to function like MySQL's FORMAT. """, extended = """ @@ -1647,7 +1651,7 @@ case class FormatNumber(x: Expression, d: Expression) * The 'lang' and 'country' arguments are optional, and if omitted, the default locale is used. */ @ExpressionDescription( - usage = "_FUNC_(str[, lang, country]) - Splits str into an array of array of words.", + usage = "_FUNC_(str[, lang, country]) - Splits `str` into an array of array of words.", extended = """ Arguments: str - a string expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala index ea98774f39fb..2be44757d9f4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala @@ -384,11 +384,10 @@ abstract class OffsetWindowFunction @ExpressionDescription( usage = """ _FUNC_(input[, offset[, default]]) - Returns the value of `input` at the `offset`th row - after the current row in the window. - The default value of `offset` is 1 and the default value of `default` is null. - If the value of `input` at the `offset`th row is null, null is returned. - If there is no such an offset row (e.g. when the offset is 1, the last row of the window - does not have any subsequent row), `default` is returned. + after the current row in the window. The default value of `offset` is 1 and the default + value of `default` is null. If the value of `input` at the `offset`th row is null, + null is returned. If there is no such an offset row (e.g., when the offset is 1, the last + row of the window does not have any subsequent row), `default` is returned. """, extended = """ Arguments: @@ -421,11 +420,10 @@ case class Lead(input: Expression, offset: Expression, default: Expression) @ExpressionDescription( usage = """ _FUNC_(input[, offset[, default]]) - Returns the value of `input` at the `offset`th row - before the current row in the window. - The default value of `offset` is 1 and the default value of `default` is null. - If the value of `input` at the `offset`th row is null, null is returned. - If there is no such offset row (e.g. when the offset is 1, the first row of the window - does not have any previous row), `default` is returned. + before the current row in the window. The default value of `offset` is 1 and the default + value of `default` is null. If the value of `input` at the `offset`th row is null, + null is returned. If there is no such offset row (e.g., when the offset is 1, the first + row of the window does not have any previous row), `default` is returned. """, extended = """ Arguments: @@ -519,8 +517,8 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction { } /** - * The NTile function divides the rows for each window partition into 'n' buckets ranging from 1 to - * at most 'n'. Bucket values will differ by at most 1. If the number of rows in the partition does + * The NTile function divides the rows for each window partition into `n` buckets ranging from 1 to + * at most `n`. Bucket values will differ by at most 1. If the number of rows in the partition does * not divide evenly into the number of buckets, then the remainder values are distributed one per * bucket, starting with the first bucket. * @@ -541,8 +539,8 @@ case class CumeDist() extends RowNumberLike with SizeBasedWindowFunction { */ @ExpressionDescription( usage = """ - _FUNC_(n) - Divides the rows for each window partition into 'n' buckets ranging - from 1 to at most 'n'. + _FUNC_(n) - Divides the rows for each window partition into `n` buckets ranging + from 1 to at most `n`. """, extended = """ Arguments: diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala index 0c3c10b3e0ee..5a4647feded4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala @@ -104,21 +104,21 @@ case class DescribeFunctionCommand( case "<>" => Row(s"Function: $functionName") :: Row("Usage: expr1 <> expr2 - " + - "Returns TRUE if expr1 is not equal to expr2") :: Nil + "Returns true if `expr1` is not equal to `expr2`.") :: Nil case "!=" => Row(s"Function: $functionName") :: Row("Usage: expr1 != expr2 - " + - "Returns TRUE if expr1 is not equal to expr2") :: Nil + "Returns true if `expr1` is not equal to `expr2`.") :: Nil case "between" => Row("Function: between") :: Row("Usage: expr1 [NOT] BETWEEN expr2 AND expr3 - " + - "evaluate if expr1 is [not] in between expr2 and expr3") :: Nil + "evaluate if `expr1` is [not] in between `expr2` and `expr3`.") :: Nil case "case" => Row("Function: case") :: Row("Usage: CASE expr1 WHEN expr2 THEN expr3 " + "[WHEN expr4 THEN expr5]* [ELSE expr6] END - " + - "When expr1 = expr2, returns expr3; " + - "when expr1 = expr4, return expr5; else return expr6") :: Nil + "When `expr1` = `expr2`, returns `expr3`; " + + "when `expr1` = `expr4`, return `expr5`; else return `expr6`.") :: Nil case _ => try { val info = sparkSession.sessionState.catalog.lookupFunctionInfo(functionName) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index c9fa60b4540d..607589e3824b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -88,7 +88,7 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { checkKeywordsExist(sql("describe function extended upper"), "Function: upper", "Class: org.apache.spark.sql.catalyst.expressions.Upper", - "Usage: upper(str) - Returns str with all characters changed to uppercase", + "Usage: upper(str) - Returns `str` with all characters changed to uppercase", "Extended Usage:", "Arguments:", "str - a string expression.", @@ -99,7 +99,7 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext { checkKeywordsExist(sql("describe functioN Upper"), "Function: upper", "Class: org.apache.spark.sql.catalyst.expressions.Upper", - "Usage: upper(str) - Returns str with all characters changed to uppercase") + "Usage: upper(str) - Returns `str` with all characters changed to uppercase") checkKeywordsNotExist(sql("describe functioN Upper"), "Extended Usage") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 423f3119f58b..bdaac9db37c6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1455,7 +1455,7 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { sql("DESCRIBE FUNCTION log"), Row("Class: org.apache.spark.sql.catalyst.expressions.Logarithm") :: Row("Function: log") :: - Row("Usage: log(base, expr) - Returns the logarithm of expr with base.") :: Nil + Row("Usage: log(base, expr) - Returns the logarithm of `expr` with `base`.") :: Nil ) // predicate operator checkAnswer( @@ -1475,14 +1475,14 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { sql("DESCRIBE FUNCTION +"), Row("Class: org.apache.spark.sql.catalyst.expressions.Add") :: Row("Function: +") :: - Row("Usage: expr1 + expr2 - Returns expr1+expr2.") :: Nil + Row("Usage: expr1 + expr2 - Returns `expr1`+`expr2`.") :: Nil ) // comparison operators checkAnswer( sql("DESCRIBE FUNCTION <"), Row("Class: org.apache.spark.sql.catalyst.expressions.LessThan") :: Row("Function: <") :: - Row("Usage: expr1 < expr2 - Returns TRUE if expr1 is less than expr2.") :: Nil + Row("Usage: expr1 < expr2 - Returns true if `expr1` is less than `expr2`.") :: Nil ) // STRING checkAnswer( @@ -1490,7 +1490,7 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { Row("Class: org.apache.spark.sql.catalyst.expressions.Concat") :: Row("Function: concat") :: Row("Usage: concat(str1, str2, ..., strN) " + - "- Returns the concatenation of str1, str2, ..., strN.") :: Nil + "- Returns the concatenation of `str1`, `str2`, ..., `strN`.") :: Nil ) // extended mode checkAnswer( @@ -1508,7 +1508,7 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { | """.stripMargin) :: Row("Function: ^") :: Row("Usage: expr1 ^ expr2 - Returns the result of " + - "bitwise exclusive OR of expr1 and expr2.") :: Nil + "bitwise exclusive OR of `expr1` and `expr2`.") :: Nil ) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index c139f2bd806d..debdb7b61278 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -252,7 +252,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { checkKeywordsExist(sql("describe function extended upper"), "Function: upper", "Class: org.apache.spark.sql.catalyst.expressions.Upper", - "Usage: upper(str) - Returns str with all characters changed to uppercase", + "Usage: upper(str) - Returns `str` with all characters changed to uppercase", "Extended Usage:", "Arguments:", "str - a string expression.", @@ -263,7 +263,7 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { checkKeywordsExist(sql("describe functioN Upper"), "Function: upper", "Class: org.apache.spark.sql.catalyst.expressions.Upper", - "Usage: upper(str) - Returns str with all characters changed to uppercase") + "Usage: upper(str) - Returns `str` with all characters changed to uppercase") checkKeywordsNotExist(sql("describe functioN Upper"), "Extended Usage") @@ -274,28 +274,28 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton { checkKeywordsExist(sql("describe functioN `~`"), "Function: ~", "Class: org.apache.spark.sql.catalyst.expressions.BitwiseNot", - "Usage: ~ expr - Returns the result of bitwise NOT of expr.") + "Usage: ~ expr - Returns the result of bitwise NOT of `expr`.") // Hard coded describe functions checkKeywordsExist(sql("describe function `<>`"), "Function: <>", - "Usage: expr1 <> expr2 - Returns TRUE if expr1 is not equal to expr2") + "Usage: expr1 <> expr2 - Returns true if `expr1` is not equal to `expr2`") checkKeywordsExist(sql("describe function `!=`"), "Function: !=", - "Usage: expr1 != expr2 - Returns TRUE if expr1 is not equal to expr2") + "Usage: expr1 != expr2 - Returns true if `expr1` is not equal to `expr2`") checkKeywordsExist(sql("describe function `between`"), "Function: between", "Usage: expr1 [NOT] BETWEEN expr2 AND expr3 - " + - "evaluate if expr1 is [not] in between expr2 and expr3") + "evaluate if `expr1` is [not] in between `expr2` and `expr3`") checkKeywordsExist(sql("describe function `case`"), "Function: case", "Usage: CASE expr1 WHEN expr2 THEN expr3 " + "[WHEN expr4 THEN expr5]* [ELSE expr6] END - " + - "When expr1 = expr2, returns expr3; " + - "when expr1 = expr4, return expr5; else return expr6") + "When `expr1` = `expr2`, returns `expr3`; " + + "when `expr1` = `expr4`, return `expr5`; else return `expr6`") } test("describe functions - user defined functions") { From 2b437fe169080b53215f280c8987ff1d8e779df8 Mon Sep 17 00:00:00 2001 From: hyukjinkwon Date: Sat, 29 Oct 2016 09:29:04 +0900 Subject: [PATCH 47/47] address type and more details --- .../sql/catalyst/expressions/aggregate/Max.scala | 2 +- .../sql/catalyst/expressions/aggregate/Min.scala | 2 +- .../catalyst/expressions/aggregate/collect.scala | 2 +- .../sql/catalyst/expressions/arithmetic.scala | 4 ++-- .../expressions/datetimeExpressions.scala | 2 +- .../sql/catalyst/expressions/predicates.scala | 16 ++++++++-------- .../catalyst/expressions/windowExpressions.scala | 8 ++++---- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala index 48af54e9d587..a363c613c6b5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Max.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.types._ usage = "_FUNC_(expr) - Returns the maximum value of `expr`.", extended = """ Arguments: - expr - an expression of any type. + expr - an expression of any type except map. """) case class Max(child: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala index 0b02b2ca7a68..9cf93d1e9afa 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/Min.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.types._ usage = "_FUNC_(expr) - Returns the minimum value of `expr`.", extended = """ Arguments: - expr - an expression of any type. + expr - an expression of any type except map. """) case class Min(child: Expression) extends DeclarativeAggregate { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala index f20495c2b070..2d0ef0a2445d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/collect.scala @@ -116,7 +116,7 @@ case class CollectList( usage = "_FUNC_(expr) - Collects and returns a set of unique elements.", extended = """ Arguments: - expr - an expression of any type that represents data to collect as a set. + expr - an expression of any type except map that represents data to collect as a set. """) case class CollectSet( child: Expression, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index a062909144fe..f448e5d71ffb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -550,7 +550,7 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic wi usage = "_FUNC_(expr, ...) - Returns the least value of all parameters, skipping null values.", extended = """ Arguments: - expr - an expression of any type. + expr - an expression of any type except map. Examples: > SELECT _FUNC_(10, 9, 2, 4, 3); @@ -618,7 +618,7 @@ case class Least(children: Seq[Expression]) extends Expression { usage = "_FUNC_(expr, ...) - Returns the greatest value of all parameters, skipping null values.", extended = """ Arguments: - expr - an expression of any type. + expr - an expression of any type except map. Examples: > SELECT _FUNC_(10, 9, 2, 4, 3); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 2e479a8174ab..a8bf6cd9bed7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -1105,7 +1105,7 @@ case class ToDate(child: Expression) extends UnaryExpression with ImplicitCastIn */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(date, fmt) - Returns returns `date` with the time portion of the day truncated to the unit specified by the format model `fmt`.", + usage = "_FUNC_(date, fmt) - Returns `date` with the time portion of the day truncated to the unit specified by the format model `fmt`.", extended = """ Arguments: date - a date expression. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala index 984ff0b9bd7a..7a895d4edee8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala @@ -509,8 +509,8 @@ case class EqualNullSafe(left: Expression, right: Expression) extends BinaryComp usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is less than `expr2`.", extended = """ Arguments: - expr1 - an expression of any type. - expr2 - an expression of any type. + expr1 - an expression of any atomic type. + expr2 - an expression of any atomic type. """) case class LessThan(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { @@ -528,8 +528,8 @@ case class LessThan(left: Expression, right: Expression) usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is less than or equal to `expr2`.", extended = """ Arguments: - expr1 - an expression of any type. - expr2 - an expression of any type. + expr1 - an expression of any atomic type. + expr2 - an expression of any atomic type. """) case class LessThanOrEqual(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { @@ -547,8 +547,8 @@ case class LessThanOrEqual(left: Expression, right: Expression) usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is greater than `expr2`.", extended = """ Arguments: - expr1 - an expression of any type. - expr2 - an expression of any type. + expr1 - an expression of any atomic type. + expr2 - an expression of any atomic type. """) case class GreaterThan(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { @@ -566,8 +566,8 @@ case class GreaterThan(left: Expression, right: Expression) usage = "expr1 _FUNC_ expr2 - Returns true if `expr1` is greater than or equal to `expr2`.", extended = """ Arguments: - expr1 - an expression of any type. - expr2 - an expression of any type. + expr1 - an expression of any atomic type. + expr2 - an expression of any atomic type. """) case class GreaterThanOrEqual(left: Expression, right: Expression) extends BinaryComparison with NullIntolerant { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala index 2be44757d9f4..d49ee32fbd1a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/windowExpressions.scala @@ -391,9 +391,9 @@ abstract class OffsetWindowFunction """, extended = """ Arguments: - input - an expression of any type. + input - an expression of any type except map. offset - a numeric expression. Default is 1. - default - an expression of any type. Default is null. + default - an expression of any type except map. Default is null. """) case class Lead(input: Expression, offset: Expression, default: Expression) extends OffsetWindowFunction { @@ -427,9 +427,9 @@ case class Lead(input: Expression, offset: Expression, default: Expression) """, extended = """ Arguments: - input - an expression of any type. + input - an expression of any type except map. offset - a numeric expression. Default is 1. - default - an expression of any type. Default is null. + default - an expression of any type except map. Default is null. """) case class Lag(input: Expression, offset: Expression, default: Expression) extends OffsetWindowFunction {