diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala index ea0ed2e8fa11..59481ce04916 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproximatePercentile.scala @@ -63,7 +63,7 @@ import org.apache.spark.sql.types._ > SELECT _FUNC_(10.0, array(0.5, 0.4, 0.1), 100); [10.0,10.0,10.0] > SELECT _FUNC_(10.0, 0.5, 100); - 10.0 + 10 """, since = "2.1.0") case class ApproximatePercentile( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala index 1870c58c548c..8ce8dfa19c01 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CentralMomentAgg.scala @@ -223,7 +223,7 @@ case class VarianceSamp(child: Expression) extends CentralMomentAgg(child) { examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); - 1.1135657469022013 + 1.1135657469022011 > SELECT _FUNC_(col) FROM VALUES (-1000), (-100), (10), (20) AS tab(col); -1.1135657469022011 """, @@ -245,9 +245,9 @@ case class Skewness(child: Expression) extends CentralMomentAgg(child) { examples = """ Examples: > SELECT _FUNC_(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col); - -0.7014368047529618 + -0.7014368047529627 > SELECT _FUNC_(col) FROM VALUES (1), (10), (100), (10), (1) as tab(col); - 0.19432323191698986 + 0.19432323191699075 """, since = "1.6.0") case class Kurtosis(child: Expression) extends CentralMomentAgg(child) { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala index e4276e33acbd..8d462184ed6c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala @@ -448,7 +448,7 @@ case class IntegralDivide(left: Expression, right: Expression) extends DivModLik usage = "expr1 _FUNC_ expr2 - Returns the remainder after `expr1`/`expr2`.", examples = """ Examples: - > SELECT 2 _FUNC_ 1.8; + > SELECT 2 % 1.8; 0.2 > SELECT MOD(2, 1.8); 0.2 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index 319a7fc87e59..cae3c0528e13 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -422,9 +422,9 @@ case class CreateNamedStructUnsafe(children: Seq[Expression]) extends CreateName examples = """ Examples: > SELECT _FUNC_('a:1,b:2,c:3', ',', ':'); - map("a":"1","b":"2","c":"3") + {"a":"1","b":"2","c":"3"} > SELECT _FUNC_('a'); - map("a":null) + {"a":null} """) // scalastyle:on line.size.limit case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: Expression) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala index 65b10f36373d..67c24f687af0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/csvExpressions.scala @@ -40,8 +40,8 @@ import org.apache.spark.unsafe.types.UTF8String examples = """ Examples: > SELECT _FUNC_('1, 0.8', 'a INT, b DOUBLE'); - {"a":1, "b":0.8} - > SELECT _FUNC_('26/08/2015', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy')) + {"a":1,"b":0.8} + > SELECT _FUNC_('26/08/2015', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy')); {"time":2015-08-26 00:00:00.0} """, since = "3.0.0") @@ -199,7 +199,7 @@ case class SchemaOfCsv( > SELECT _FUNC_(named_struct('a', 1, 'b', 2)); 1,2 > SELECT _FUNC_(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy')); - "26/08/2015" + 26/08/2015 """, since = "3.0.0") // scalastyle:on line.size.limit diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 2543828b6315..0fffffea1d82 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -631,7 +631,7 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti examples = """ Examples: > SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd'); - 1460041200 + 1460098800 """, since = "1.6.0") case class ToUnixTimestamp( @@ -842,7 +842,7 @@ abstract class UnixTime extends ToTimestamp { examples = """ Examples: > SELECT _FUNC_(0, 'yyyy-MM-dd HH:mm:ss'); - 1970-01-01 00:00:00 + 1969-12-31 16:00:00 """, since = "1.5.0") case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[String] = None) @@ -1766,10 +1766,10 @@ case class MakeDate(year: Expression, month: Expression, day: Expression) > SELECT _FUNC_(2014, 12, 28, 6, 30, 45.887); 2014-12-28 06:30:45.887 > SELECT _FUNC_(2014, 12, 28, 6, 30, 45.887, 'CET'); - 2014-12-28 10:30:45.887 - > SELECT _FUNC_(2019, 6, 30, 23, 59, 60) + 2014-12-27 21:30:45.887 + > SELECT _FUNC_(2019, 6, 30, 23, 59, 60); 2019-07-01 00:00:00 - > SELECT _FUNC_(2019, 13, 1, 10, 11, 12, 13); + > SELECT _FUNC_(2019, 13, 1, 10, 11, 12, 'PST'); NULL > SELECT _FUNC_(null, 7, 22, 15, 30, 0); NULL diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index 82a7d9825e30..b0a23c62284d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -127,16 +127,16 @@ case class UserDefinedGenerator( * 3 NULL * }}} */ -// scalastyle:off line.size.limit +// scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(n, expr1, ..., exprk) - Separates `expr1`, ..., `exprk` into `n` rows. Uses column names col0, col1, etc. by default unless specified otherwise.", examples = """ Examples: > SELECT _FUNC_(2, 1, 2, 3); - 1 2 - 3 NULL + 1 2 + 3 NULL """) -// scalastyle:on line.size.limit +// scalastyle:on line.size.limit line.contains.tab case class Stack(children: Seq[Expression]) extends Generator { private lazy val numRows = children.head.eval().asInstanceOf[Int] @@ -375,16 +375,16 @@ case class Explode(child: Expression) extends ExplodeBase { * 1 20 * }}} */ -// scalastyle:off line.size.limit +// scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple rows with positions, or the elements of map `expr` into multiple rows and columns with positions. Unless specified otherwise, uses the column name `pos` for position, `col` for elements of the array or `key` and `value` for elements of the map.", examples = """ Examples: > SELECT _FUNC_(array(10,20)); - 0 10 - 1 20 + 0 10 + 1 20 """) -// scalastyle:on line.size.limit +// scalastyle:on line.size.limit line.contains.tab case class PosExplode(child: Expression) extends ExplodeBase { override val position = true } @@ -392,16 +392,16 @@ case class PosExplode(child: Expression) extends ExplodeBase { /** * Explodes an array of structs into a table. */ -// scalastyle:off line.size.limit +// scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(expr) - Explodes an array of structs into a table. Uses column names col1, col2, etc. by default unless specified otherwise.", examples = """ Examples: > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b'))); - 1 a - 2 b + 1 a + 2 b """) -// scalastyle:on line.size.limit +// scalastyle:on line.size.limit line.contains.tab case class Inline(child: Expression) extends UnaryExpression with CollectionGenerator { override val inline: Boolean = true override val position: Boolean = false diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala index 221b97bdc785..2da3f6c74b0e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/grouping.scala @@ -38,7 +38,7 @@ trait GroupingSet extends Expression with CodegenFallback { override def eval(input: InternalRow): Any = throw new UnsupportedOperationException } -// scalastyle:off line.size.limit +// scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = """ _FUNC_([col1[, col2 ..]]) - create a multi-dimensional cube using the specified columns @@ -47,19 +47,19 @@ trait GroupingSet extends Expression with CodegenFallback { examples = """ Examples: > SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY _FUNC_(name, age); - NULL 2 1 - NULL NULL 2 - Alice 2 1 - Bob 5 1 - NULL 5 1 - Bob NULL 1 - Alice NULL 1 + Bob 5 1 + Alice 2 1 + NULL NULL 2 + NULL 5 1 + Bob NULL 1 + Alice NULL 1 + NULL 2 1 """, since = "2.0.0") -// scalastyle:on line.size.limit +// scalastyle:on line.size.limit line.contains.tab case class Cube(groupByExprs: Seq[Expression]) extends GroupingSet {} -// scalastyle:off line.size.limit +// scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = """ _FUNC_([col1[, col2 ..]]) - create a multi-dimensional rollup using the specified columns @@ -68,21 +68,21 @@ case class Cube(groupByExprs: Seq[Expression]) extends GroupingSet {} examples = """ Examples: > SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY _FUNC_(name, age); - NULL NULL 2 - Alice 2 1 - Bob 5 1 - Bob NULL 1 - Alice NULL 1 + Bob 5 1 + Alice 2 1 + NULL NULL 2 + Bob NULL 1 + Alice NULL 1 """, since = "2.0.0") -// scalastyle:on line.size.limit +// scalastyle:on line.size.limit line.contains.tab case class Rollup(groupByExprs: Seq[Expression]) extends GroupingSet {} /** * Indicates whether a specified column expression in a GROUP BY list is aggregated or not. * GROUPING returns 1 for aggregated or 0 for not aggregated in the result set. */ -// scalastyle:off line.size.limit +// scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = """ _FUNC_(col) - indicates whether a specified column in a GROUP BY is aggregated or @@ -91,12 +91,12 @@ case class Rollup(groupByExprs: Seq[Expression]) extends GroupingSet {} examples = """ Examples: > SELECT name, _FUNC_(name), sum(age) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY cube(name); - Alice 0 2 - NULL 1 7 - Bob 0 5 + Bob 0 5 + Alice 0 2 + NULL 1 7 """, since = "2.0.0") -// scalastyle:on line.size.limit +// scalastyle:on line.size.limit line.contains.tab case class Grouping(child: Expression) extends Expression with Unevaluable { @transient override lazy val references: AttributeSet = @@ -111,7 +111,7 @@ case class Grouping(child: Expression) extends Expression with Unevaluable { * * If groupByExprs is empty, it means all grouping expressions in GroupingSets. */ -// scalastyle:off line.size.limit +// scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = """ _FUNC_([col1[, col2 ..]]) - returns the level of grouping, equals to @@ -120,20 +120,20 @@ case class Grouping(child: Expression) extends Expression with Unevaluable { examples = """ Examples: > SELECT name, _FUNC_(), sum(age), avg(height) FROM VALUES (2, 'Alice', 165), (5, 'Bob', 180) people(age, name, height) GROUP BY cube(name, height); - NULL 2 2 165.0 - Alice 0 2 165.0 - NULL 2 5 180.0 - NULL 3 7 172.5 - Bob 0 5 180.0 - Bob 1 5 180.0 - Alice 1 2 165.0 + NULL 2 5 180.0 + Alice 0 2 165.0 + NULL 3 7 172.5 + NULL 2 2 165.0 + Bob 1 5 180.0 + Alice 1 2 165.0 + Bob 0 5 180.0 """, note = """ Input columns should match with grouping columns exactly, or empty (means all the grouping columns). """, since = "2.0.0") -// scalastyle:on line.size.limit +// scalastyle:on line.size.limit line.contains.tab case class GroupingID(groupByExprs: Seq[Expression]) extends Expression with Unevaluable { @transient override lazy val references: AttributeSet = diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala index ed26bb375de2..3b104e1a78b7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala @@ -463,7 +463,7 @@ case class ArrayExists( > SELECT _FUNC_(array(1, null, 3), x -> x % 2 == 0); false > SELECT _FUNC_(array(2, null, 8), x -> x % 2 == 0); - null + NULL """, since = "3.0.0") case class ArrayForAll( diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala index 655e44e4e491..a13a6836c6be 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala @@ -331,15 +331,15 @@ case class GetJsonObject(json: Expression, path: Expression) } } -// scalastyle:off line.size.limit +// scalastyle:off line.size.limit line.contains.tab @ExpressionDescription( usage = "_FUNC_(jsonStr, p1, p2, ..., pn) - Returns a tuple like the function get_json_object, but it takes multiple names. All the input parameters and output column types are string.", examples = """ Examples: > SELECT _FUNC_('{"a":1, "b":2}', 'a', 'b'); - 1 2 + 1 2 """) -// scalastyle:on line.size.limit +// scalastyle:on line.size.limit line.contains.tab case class JsonTuple(children: Seq[Expression]) extends Generator with CodegenFallback { @@ -502,9 +502,9 @@ case class JsonTuple(children: Seq[Expression]) examples = """ Examples: > SELECT _FUNC_('{"a":1, "b":0.8}', 'a INT, b DOUBLE'); - {"a":1, "b":0.8} + {"a":1,"b":0.8} > SELECT _FUNC_('{"time":"26/08/2015"}', 'time Timestamp', map('timestampFormat', 'dd/MM/yyyy')); - {"time":"2015-08-26 00:00:00.0"} + {"time":2015-08-26 00:00:00.0} """, since = "2.2.0") // scalastyle:on line.size.limit diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala index b9f089ec056c..d5b959b91c23 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala @@ -1291,7 +1291,7 @@ abstract class RoundBase(child: Expression, scale: Expression, examples = """ Examples: > SELECT _FUNC_(2.5, 0); - 3.0 + 3 """) // scalastyle:on line.size.limit case class Round(child: Expression, scale: Expression) @@ -1311,7 +1311,7 @@ case class Round(child: Expression, scale: Expression) examples = """ Examples: > SELECT _FUNC_(2.5, 0); - 2.0 + 2 """) // scalastyle:on line.size.limit case class BRound(child: Expression, scale: Expression) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 9229ef2039fe..79fe05f8087a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -96,7 +96,7 @@ abstract class StringRegexExpression extends BinaryExpression """, examples = """ Examples: - > SELECT '%SystemDrive%\Users\John' _FUNC_ '\%SystemDrive\%\\Users%' + > SELECT '%SystemDrive%\Users\John' _FUNC_ '\%SystemDrive\%\Users%'; true """, note = """ @@ -153,6 +153,7 @@ case class Like(left: Expression, right: Expression) extends StringRegexExpressi } } +// scalastyle:off line.contains.tab @ExpressionDescription( usage = "str _FUNC_ regexp - Returns true if `str` matches `regexp`, or false otherwise.", arguments = """ @@ -170,18 +171,20 @@ case class Like(left: Expression, right: Expression) extends StringRegexExpressi """, examples = """ Examples: - When spark.sql.parser.escapedStringLiterals is disabled (default). - > SELECT '%SystemDrive%\Users\John' _FUNC_ '%SystemDrive%\\Users.*' + > SET spark.sql.parser.escapedStringLiterals=true; + spark.sql.parser.escapedStringLiterals true + > SELECT '%SystemDrive%\Users\John' _FUNC_ '%SystemDrive%\\Users.*'; true - - When spark.sql.parser.escapedStringLiterals is enabled. - > SELECT '%SystemDrive%\Users\John' _FUNC_ '%SystemDrive%\Users.*' + > SET spark.sql.parser.escapedStringLiterals=false; + spark.sql.parser.escapedStringLiterals false + > SELECT '%SystemDrive%\Users\John' _FUNC_ '%SystemDrive%\Users.*'; true """, note = """ Use LIKE to match with simple string pattern. """, since = "1.0.0") +// scalastyle:on line.contains.tab case class RLike(left: Expression, right: Expression) extends StringRegexExpression { override def escape(v: String): String = v diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index e4847e9cec3f..211ae3f02a0d 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -847,7 +847,7 @@ object StringTrimLeft { usage = """ _FUNC_(str) - Removes the leading space characters from `str`. - _FUNC_(trimStr, str) - Removes the leading string contains the characters from the trim string + _FUNC_(str, trimStr) - Removes the leading string contains the characters from the trim string """, arguments = """ Arguments: @@ -858,7 +858,7 @@ object StringTrimLeft { Examples: > SELECT _FUNC_(' SparkSQL '); SparkSQL - > SELECT _FUNC_('Sp', 'SSparkSQLS'); + > SELECT _FUNC_('SparkSQLS', 'Sp'); arkSQLS """, since = "1.5.0") @@ -949,7 +949,7 @@ object StringTrimRight { usage = """ _FUNC_(str) - Removes the trailing space characters from `str`. - _FUNC_(trimStr, str) - Removes the trailing string which contains the characters from the trim string from the `str` + _FUNC_(str, trimStr) - Removes the trailing string which contains the characters from the trim string from the `str` """, arguments = """ Arguments: @@ -960,7 +960,7 @@ object StringTrimRight { Examples: > SELECT _FUNC_(' SparkSQL '); SparkSQL - > SELECT _FUNC_('LQSa', 'SSparkSQLS'); + > SELECT _FUNC_('SSparkSQLS', 'SQLS'); SSpark """, since = "1.5.0") @@ -1299,11 +1299,11 @@ object ParseUrl { usage = "_FUNC_(url, partToExtract[, key]) - Extracts a part from a URL.", examples = """ Examples: - > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST') + > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'HOST'); spark.apache.org - > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY') + > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY'); query=1 - > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY', 'query') + > SELECT _FUNC_('http://spark.apache.org/path?query=1', 'QUERY', 'query'); 1 """, since = "2.0.0") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala index aacf1a44e2ad..073b45af51ca 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/xml/xpath.scala @@ -194,7 +194,7 @@ case class XPathString(xml: Expression, path: Expression) extends XPathExtract { examples = """ Examples: > SELECT _FUNC_('b1b2b3c1c2','a/b/text()'); - ['b1','b2','b3'] + ["b1","b2","b3"] """) // scalastyle:on line.size.limit case class XPathList(xml: Expression, path: Expression) extends XPathExtract { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index b5d021549c7a..9488bbe3a253 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -26,6 +26,7 @@ import org.apache.spark.{AccumulatorSuite, SparkException} import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart} import org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation import org.apache.spark.sql.catalyst.util.StringUtils +import org.apache.spark.sql.execution.HiveResult.hiveResultString import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, SortAggregateExec} import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.execution.datasources.v2.BatchScanExec @@ -127,7 +128,9 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession { // _FUNC_ is replaced by `%` which causes a parsing error on `SELECT %(2, 1.8)` "org.apache.spark.sql.catalyst.expressions.Remainder", // Examples demonstrate alternative names, see SPARK-20749 - "org.apache.spark.sql.catalyst.expressions.Length") + "org.apache.spark.sql.catalyst.expressions.Length", + // Uses settings without _FUNC_ in `SET spark.sql.parser.escapedStringLiterals=true` + "org.apache.spark.sql.catalyst.expressions.RLike") spark.sessionState.functionRegistry.listFunction().foreach { funcId => val info = spark.sessionState.catalog.lookupFunctionInfo(funcId) val className = info.getClassName @@ -140,6 +143,53 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession { } } + test("check outputs of expression examples") { + def unindentAndTrim(s: String): String = { + s.replaceAll("\n\\s+", "\n").trim + } + val beginSqlStmtRe = " > ".r + val endSqlStmtRe = ";\n".r + def checkExampleSyntax(example: String): Unit = { + val beginStmtNum = beginSqlStmtRe.findAllIn(example).length + val endStmtNum = endSqlStmtRe.findAllIn(example).length + assert(beginStmtNum === endStmtNum, + "The number of ` > ` does not match to the number of `;`") + } + val exampleRe = """^(.+);\n(?s)(.+)$""".r + val ignoreSet = Set( + // One of examples shows getting the current timestamp + "org.apache.spark.sql.catalyst.expressions.UnixTimestamp", + // Random output without a seed + "org.apache.spark.sql.catalyst.expressions.Rand", + "org.apache.spark.sql.catalyst.expressions.Randn", + "org.apache.spark.sql.catalyst.expressions.Shuffle", + "org.apache.spark.sql.catalyst.expressions.Uuid", + // The example calls methods that return unstable results. + "org.apache.spark.sql.catalyst.expressions.CallMethodViaReflection") + + withSQLConf(SQLConf.UTC_TIMESTAMP_FUNC_ENABLED.key -> "true") { + spark.sessionState.functionRegistry.listFunction().par.foreach { funcId => + val info = spark.sessionState.catalog.lookupFunctionInfo(funcId) + val className = info.getClassName + if (!ignoreSet.contains(className)) { + withClue(s"Function '${info.getName}', Expression class '$className'") { + val example = info.getExamples + checkExampleSyntax(example) + example.split(" > ").toList.foreach(_ match { + case exampleRe(sql, output) => + val df = spark.sql(sql) + val actual = unindentAndTrim( + hiveResultString(df.queryExecution.executedPlan).mkString("\n")) + val expected = unindentAndTrim(output) + assert(actual === expected) + case _ => + }) + } + } + } + } + } + test("SPARK-6743: no columns from cache") { Seq( (83, 0, 38),