diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index c0e7e7868af34..c7d0eba0964cc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -404,7 +404,7 @@ object FunctionRegistry { expression[Month]("month"), expression[MonthsBetween]("months_between"), expression[NextDay]("next_day"), - expression[CurrentTimestamp]("now", true), + expression[Now]("now"), expression[Quarter]("quarter"), expression[Second]("second"), expression[ParseToTimestamp]("to_timestamp"), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala index 2f8cb26ffaa9c..423f89fefa093 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/UnsupportedOperationChecker.scala @@ -19,13 +19,11 @@ package org.apache.spark.sql.catalyst.analysis import org.apache.spark.internal.Logging import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, AttributeSet, CurrentDate, CurrentTimestamp, MonotonicallyIncreasingID} +import org.apache.spark.sql.catalyst.expressions.{Attribute, CurrentDate, CurrentTimestamp, MonotonicallyIncreasingID, Now} import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression -import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys import org.apache.spark.sql.catalyst.plans._ import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.streaming.InternalOutputModes -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.streaming.OutputMode /** @@ -412,7 +410,7 @@ object UnsupportedOperationChecker extends Logging { subPlan.expressions.foreach { e => if (e.collectLeaves().exists { - case (_: CurrentTimestamp | _: CurrentDate) => true + case (_: CurrentTimestamp | _: Now | _: CurrentDate) => true case _ => false }) { throwError(s"Continuous processing does not support current time operations.") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala index 2e27f1d0df484..7ffed9d32aaeb 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala @@ -28,7 +28,6 @@ import org.apache.commons.text.StringEscapeUtils import org.apache.spark.SparkUpgradeException import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.FunctionRegistry import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.expressions.codegen.Block._ import org.apache.spark.sql.catalyst.util.{DateTimeUtils, LegacyDateFormats, TimestampFormatter} @@ -62,7 +61,21 @@ trait TimeZoneAwareExpression extends Expression { * There is no code generation since this expression should get constant folded by the optimizer. */ @ExpressionDescription( - usage = "_FUNC_() - Returns the current date at the start of query evaluation.", + usage = """ + _FUNC_() - Returns the current date at the start of query evaluation. + + _FUNC_ - Returns the current date at the start of query evaluation. + """, + examples = """ + Examples: + > SELECT _FUNC_(); + 2020-04-25 + > SELECT _FUNC_; + 2020-04-25 + """, + note = """ + The syntax without braces has been supported since 2.0.1. + """, group = "datetime_funcs", since = "1.5.0") case class CurrentDate(timeZoneId: Option[String] = None) @@ -83,6 +96,13 @@ case class CurrentDate(timeZoneId: Option[String] = None) override def prettyName: String = "current_date" } +abstract class CurrentTimestampLike() extends LeafExpression with CodegenFallback { + override def foldable: Boolean = true + override def nullable: Boolean = false + override def dataType: DataType = TimestampType + override def eval(input: InternalRow): Any = currentTimestamp() +} + /** * Returns the current timestamp at the start of query evaluation. * All calls of current_timestamp within the same query return the same value. @@ -90,19 +110,38 @@ case class CurrentDate(timeZoneId: Option[String] = None) * There is no code generation since this expression should get constant folded by the optimizer. */ @ExpressionDescription( - usage = "_FUNC_() - Returns the current timestamp at the start of query evaluation.", + usage = """ + _FUNC_() - Returns the current timestamp at the start of query evaluation. + + _FUNC_ - Returns the current timestamp at the start of query evaluation. + """, + examples = """ + Examples: + > SELECT _FUNC_(); + 2020-04-25 15:49:11.914 + > SELECT _FUNC_; + 2020-04-25 15:49:11.914 + """, + note = """ + The syntax without braces has been supported since 2.0.1. + """, group = "datetime_funcs", since = "1.5.0") -case class CurrentTimestamp() extends LeafExpression with CodegenFallback { - override def foldable: Boolean = true - override def nullable: Boolean = false - - override def dataType: DataType = TimestampType - - override def eval(input: InternalRow): Any = currentTimestamp() +case class CurrentTimestamp() extends CurrentTimestampLike { + override def prettyName: String = "current_timestamp" +} - override def prettyName: String = - getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("current_timestamp") +@ExpressionDescription( + usage = "_FUNC_() - Returns the current timestamp at the start of query evaluation.", + examples = """ + Examples: + > SELECT _FUNC_(); + 2020-04-25 15:49:11.914 + """, + group = "datetime_funcs", + since = "1.6.0") +case class Now() extends CurrentTimestampLike { + override def prettyName: String = "now" } /** diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 3723680ff5db2..82b1e5f0998b0 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -1619,7 +1619,11 @@ case class StringSpace(child: Expression) */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(str, pos[, len]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`.", + usage = """ + _FUNC_(str, pos[, len]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`. + + _FUNC_(str FROM pos[ FOR len]]) - Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of byte array that starts at `pos` and is of length `len`. + """, examples = """ Examples: > SELECT _FUNC_('Spark SQL', 5); @@ -1628,6 +1632,12 @@ case class StringSpace(child: Expression) SQL > SELECT _FUNC_('Spark SQL', 5, 1); k + > SELECT _FUNC_('Spark SQL' FROM 5); + k SQL + > SELECT _FUNC_('Spark SQL' FROM -3); + SQL + > SELECT _FUNC_('Spark SQL' FROM 5 FOR 1); + k """, since = "1.5.0") // scalastyle:on line.size.limit diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala index c79bf3e20b776..80d85827657fd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala @@ -85,7 +85,7 @@ object ComputeCurrentTime extends Rule[LogicalPlan] { LocalDate.now(DateTimeUtils.getZoneId(timeZoneId)), DateType) }) - case CurrentTimestamp() => currentTime + case CurrentTimestamp() | Now() => currentTime } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala index 9a6fe46c48ec7..e18514c6f93f9 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/expressions/ExpressionInfoSuite.scala @@ -120,6 +120,9 @@ class ExpressionInfoSuite extends SparkFunSuite with SharedSparkSession { val ignoreSet = Set( // One of examples shows getting the current timestamp "org.apache.spark.sql.catalyst.expressions.UnixTimestamp", + "org.apache.spark.sql.catalyst.expressions.CurrentDate", + "org.apache.spark.sql.catalyst.expressions.CurrentTimestamp", + "org.apache.spark.sql.catalyst.expressions.Now", // Random output without a seed "org.apache.spark.sql.catalyst.expressions.Rand", "org.apache.spark.sql.catalyst.expressions.Randn", diff --git a/sql/gen-sql-functions-docs.py b/sql/gen-sql-functions-docs.py index 7f0b2ae582f56..c07734e273051 100644 --- a/sql/gen-sql-functions-docs.py +++ b/sql/gen-sql-functions-docs.py @@ -104,9 +104,11 @@ def _make_pretty_usage(infos): result.append(" ") for info in infos: - # Extracts (signature, description) pairs from `info.usage`, e.g., - # the signature is `func(expr)` and the description is `...` in an usage `func(expr) - ...`. - usages = iter(re.split(r"(%s\(.*\)) - " % info.name, info.usage.strip())[1:]) + # Extracts (signature, description) pairs from `info.usage`. + # Expected formats are as follows; + # - `_FUNC_(...) - description`, or + # - `_FUNC_ - description` + usages = iter(re.split(r"(%s.*) - " % info.name, info.usage.strip())[1:]) for (sig, description) in zip(usages, usages): result.append(" ") result.append(" %s" % sig)