From a8b979e72ab30150e83b68e0494c7c739fd626be Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Wed, 25 Aug 2021 17:47:06 +0800 Subject: [PATCH 1/5] audit --- .../sql/catalyst/expressions/bitwiseExpressions.scala | 8 ++++---- .../sql/catalyst/expressions/intervalExpressions.scala | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala index 3940c65593ec5..57ab9e2773e31 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala @@ -233,10 +233,10 @@ object BitwiseGetUtil { @ExpressionDescription( usage = """ - |_FUNC_(expr, pos) - Returns the value of the bit (0 or 1) at the specified position. - | The positions are numbered from right to left, starting at zero. - | The position argument cannot be negative. - """, + _FUNC_(expr, pos) - Returns the value of the bit (0 or 1) at the specified position. + The positions are numbered from right to left, starting at zero. + The position argument cannot be negative. + """, examples = """ Examples: > SELECT _FUNC_(11, 0); diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 5b111d17cdcd6..8ad1ee8f6cb26 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -360,7 +360,7 @@ case class MakeInterval( // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(days, hours, mins, secs) - Make DayTimeIntervalType duration from days, hours, mins and secs.", + usage = "_FUNC_(days[, hours[, mins[, secs]]]) - Make DayTimeIntervalType duration from days, hours, mins and secs.", arguments = """ Arguments: * days - the number of days, positive or negative @@ -372,6 +372,8 @@ case class MakeInterval( Examples: > SELECT _FUNC_(1, 12, 30, 01.001001); 1 12:30:01.001001000 + > SELECT _FUNC_(2); + 2 00:00:00.000000000 > SELECT _FUNC_(100, null, 3); NULL """, @@ -436,7 +438,7 @@ case class MakeDTInterval( } @ExpressionDescription( - usage = "_FUNC_(years, months) - Make year-month interval from years, months.", + usage = "_FUNC_(years[, months]) - Make year-month interval from years, months.", arguments = """ Arguments: * years - the number of years, positive or negative @@ -450,6 +452,8 @@ case class MakeDTInterval( 1-0 > SELECT _FUNC_(-1, 1); -0-11 + > SELECT _FUNC_(2); + 2-0 """, since = "3.2.0", group = "datetime_funcs") From ef063eb76b1cf4e1b54ee82293a0c3d5a69c1155 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Thu, 26 Aug 2021 15:10:11 +0800 Subject: [PATCH 2/5] address comments --- .../main/scala/org/apache/spark/errors/SparkCoreErrors.scala | 2 +- project/SparkBuild.scala | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/errors/SparkCoreErrors.scala b/core/src/main/scala/org/apache/spark/errors/SparkCoreErrors.scala index 7708996777faa..b17a81549cafc 100644 --- a/core/src/main/scala/org/apache/spark/errors/SparkCoreErrors.scala +++ b/core/src/main/scala/org/apache/spark/errors/SparkCoreErrors.scala @@ -30,7 +30,7 @@ import org.apache.spark.storage.{BlockId, BlockManagerId, BlockNotFoundException /** * Object for grouping error messages from (most) exceptions thrown during query execution. */ -private[spark] object SparkCoreErrors { +object SparkCoreErrors { def rddBlockNotFoundError(blockId: BlockId, id: Int): Throwable = { new Exception(s"Could not compute split, block $blockId of RDD $id not found") } diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 8edc1710e8485..93256bc9e0143 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -967,6 +967,7 @@ object Unidoc { .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/internal"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/hive"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/catalog/v2/utils"))) + .map(_.filterNot(_.getCanonicalPath.contains("org.apache.spark.errors"))) .map(_.filterNot(_.getCanonicalPath.contains("org.apache.spark.sql.errors"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/hive"))) .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/v2/avro"))) From 02f7d4ee757914bffe15372a7abc72e7ac21b896 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Thu, 26 Aug 2021 17:35:23 +0800 Subject: [PATCH 3/5] address comment --- .../spark/sql/catalyst/expressions/intervalExpressions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 8ad1ee8f6cb26..6e7ea7c786ec5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -227,7 +227,7 @@ case class DivideInterval( // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(years, months, weeks, days, hours, mins, secs) - Make interval from years, months, weeks, days, hours, mins and secs.", + usage = "_FUNC_(years[, months[, weeks[, days[, hours[, mins[, secs]]]]]]) - Make interval from years, months, weeks, days, hours, mins and secs.", arguments = """ Arguments: * years - the number of years, positive or negative From dddc1fd1e02ef37b37dc6ac522889a780691b273 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Thu, 26 Aug 2021 19:22:21 +0800 Subject: [PATCH 4/5] address --- .../spark/sql/catalyst/expressions/intervalExpressions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 6e7ea7c786ec5..0bd4ae87131cc 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -227,7 +227,7 @@ case class DivideInterval( // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(years[, months[, weeks[, days[, hours[, mins[, secs]]]]]]) - Make interval from years, months, weeks, days, hours, mins and secs.", + usage = "_FUNC_([years[, months[, weeks[, days[, hours[, mins[, secs]]]]]]]) - Make interval from years, months, weeks, days, hours, mins and secs.", arguments = """ Arguments: * years - the number of years, positive or negative From 85e4d981fbadaf9dd3d0011726880791a2171ab1 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Thu, 26 Aug 2021 23:53:22 +0800 Subject: [PATCH 5/5] update --- .../spark/sql/catalyst/expressions/intervalExpressions.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala index 0bd4ae87131cc..c799c69b9a2ce 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/intervalExpressions.scala @@ -360,7 +360,7 @@ case class MakeInterval( // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(days[, hours[, mins[, secs]]]) - Make DayTimeIntervalType duration from days, hours, mins and secs.", + usage = "_FUNC_([days[, hours[, mins[, secs]]]]) - Make DayTimeIntervalType duration from days, hours, mins and secs.", arguments = """ Arguments: * days - the number of days, positive or negative @@ -438,7 +438,7 @@ case class MakeDTInterval( } @ExpressionDescription( - usage = "_FUNC_(years[, months]) - Make year-month interval from years, months.", + usage = "_FUNC_([years[, months]]) - Make year-month interval from years, months.", arguments = """ Arguments: * years - the number of years, positive or negative