diff --git a/R/pkg/R/functions.R b/R/pkg/R/functions.R index 8f425b12937e4..5110d84172378 100644 --- a/R/pkg/R/functions.R +++ b/R/pkg/R/functions.R @@ -3568,6 +3568,8 @@ setMethod("element_at", #' @details #' \code{explode}: Creates a new row for each element in the given array or map column. +#' Uses the default column name \code{col} for elements in the array and +#' \code{key} and \code{value} for elements in the map unless specified otherwise. #' #' @rdname column_collection_functions #' @aliases explode explode,Column-method @@ -3628,7 +3630,9 @@ setMethod("sort_array", #' @details #' \code{posexplode}: Creates a new row for each element with position in the given array -#' or map column. +#' or map column. Uses the default column name \code{pos} for position, and \code{col} +#' for elements in the array and \code{key} and \code{value} for elements in the map +#' unless specified otherwise. #' #' @rdname column_collection_functions #' @aliases posexplode posexplode,Column-method @@ -3769,7 +3773,8 @@ setMethod("repeat_string", #' \code{explode}: Creates a new row for each element in the given array or map column. #' Unlike \code{explode}, if the array/map is \code{null} or empty #' then \code{null} is produced. -#' +#' Uses the default column name \code{col} for elements in the array and +#' \code{key} and \code{value} for elements in the map unless specified otherwise. #' #' @rdname column_collection_functions #' @aliases explode_outer explode_outer,Column-method @@ -3794,6 +3799,9 @@ setMethod("explode_outer", #' \code{posexplode_outer}: Creates a new row for each element with position in the given #' array or map column. Unlike \code{posexplode}, if the array/map is \code{null} or empty #' then the row (\code{null}, \code{null}) is produced. +#' Uses the default column name \code{pos} for position, and \code{col} +#' for elements in the array and \code{key} and \code{value} for elements in the map +#' unless specified otherwise. #' #' @rdname column_collection_functions #' @aliases posexplode_outer posexplode_outer,Column-method diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py index 3c33e2bed92d9..953ac6bb62332 100644 --- a/python/pyspark/sql/functions.py +++ b/python/pyspark/sql/functions.py @@ -2088,7 +2088,10 @@ def array_except(col1, col2): @since(1.4) def explode(col): - """Returns a new row for each element in the given array or map. + """ + Returns a new row for each element in the given array or map. + Uses the default column name `col` for elements in the array and + `key` and `value` for elements in the map unless specified otherwise. >>> from pyspark.sql import Row >>> eDF = spark.createDataFrame([Row(a=1, intlist=[1,2,3], mapfield={"a": "b"})]) @@ -2109,7 +2112,10 @@ def explode(col): @since(2.1) def posexplode(col): - """Returns a new row for each element with position in the given array or map. + """ + Returns a new row for each element with position in the given array or map. + Uses the default column name `pos` for position, and `col` for elements in the + array and `key` and `value` for elements in the map unless specified otherwise. >>> from pyspark.sql import Row >>> eDF = spark.createDataFrame([Row(a=1, intlist=[1,2,3], mapfield={"a": "b"})]) @@ -2130,8 +2136,11 @@ def posexplode(col): @since(2.3) def explode_outer(col): - """Returns a new row for each element in the given array or map. + """ + Returns a new row for each element in the given array or map. Unlike explode, if the array/map is null or empty then null is produced. + Uses the default column name `col` for elements in the array and + `key` and `value` for elements in the map unless specified otherwise. >>> df = spark.createDataFrame( ... [(1, ["foo", "bar"], {"x": 1.0}), (2, [], {}), (3, None, None)], @@ -2163,8 +2172,11 @@ def explode_outer(col): @since(2.3) def posexplode_outer(col): - """Returns a new row for each element with position in the given array or map. + """ + Returns a new row for each element with position in the given array or map. Unlike posexplode, if the array/map is null or empty then the row (null, null) is produced. + Uses the default column name `pos` for position, and `col` for elements in the + array and `key` and `value` for elements in the map unless specified otherwise. >>> df = spark.createDataFrame( ... [(1, ["foo", "bar"], {"x": 1.0}), (2, [], {}), (3, None, None)], diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala index 6b6da1c8b4142..82a7d9825e30a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala @@ -127,14 +127,16 @@ case class UserDefinedGenerator( * 3 NULL * }}} */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(n, expr1, ..., exprk) - Separates `expr1`, ..., `exprk` into `n` rows.", + usage = "_FUNC_(n, expr1, ..., exprk) - Separates `expr1`, ..., `exprk` into `n` rows. Uses column names col0, col1, etc. by default unless specified otherwise.", examples = """ Examples: > SELECT _FUNC_(2, 1, 2, 3); 1 2 3 NULL """) +// scalastyle:on line.size.limit case class Stack(children: Seq[Expression]) extends Generator { private lazy val numRows = children.head.eval().asInstanceOf[Int] @@ -352,7 +354,7 @@ abstract class ExplodeBase extends UnaryExpression with CollectionGenerator with */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple rows, or the elements of map `expr` into multiple rows and columns.", + usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple rows, or the elements of map `expr` into multiple rows and columns. Unless specified otherwise, uses the default column name `col` for elements of the array or `key` and `value` for the elements of the map.", examples = """ Examples: > SELECT _FUNC_(array(10, 20)); @@ -375,7 +377,7 @@ case class Explode(child: Expression) extends ExplodeBase { */ // scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple rows with positions, or the elements of map `expr` into multiple rows and columns with positions.", + usage = "_FUNC_(expr) - Separates the elements of array `expr` into multiple rows with positions, or the elements of map `expr` into multiple rows and columns with positions. Unless specified otherwise, uses the column name `pos` for position, `col` for elements of the array or `key` and `value` for elements of the map.", examples = """ Examples: > SELECT _FUNC_(array(10,20)); @@ -390,14 +392,16 @@ case class PosExplode(child: Expression) extends ExplodeBase { /** * Explodes an array of structs into a table. */ +// scalastyle:off line.size.limit @ExpressionDescription( - usage = "_FUNC_(expr) - Explodes an array of structs into a table.", + usage = "_FUNC_(expr) - Explodes an array of structs into a table. Uses column names col1, col2, etc. by default unless specified otherwise.", examples = """ Examples: > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b'))); 1 a 2 b """) +// scalastyle:on line.size.limit case class Inline(child: Expression) extends UnaryExpression with CollectionGenerator { override val inline: Boolean = true override val position: Boolean = false diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index 1199cd8df6a99..46d134ef199e1 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -3305,6 +3305,8 @@ object functions { /** * Creates a new row for each element in the given array or map column. + * Uses the default column name `col` for elements in the array and + * `key` and `value` for elements in the map unless specified otherwise. * * @group collection_funcs * @since 1.3.0 @@ -3313,6 +3315,8 @@ object functions { /** * Creates a new row for each element in the given array or map column. + * Uses the default column name `col` for elements in the array and + * `key` and `value` for elements in the map unless specified otherwise. * Unlike explode, if the array/map is null or empty then null is produced. * * @group collection_funcs @@ -3322,6 +3326,8 @@ object functions { /** * Creates a new row for each element with position in the given array or map column. + * Uses the default column name `pos` for position, and `col` for elements in the array + * and `key` and `value` for elements in the map unless specified otherwise. * * @group collection_funcs * @since 2.1.0 @@ -3330,6 +3336,8 @@ object functions { /** * Creates a new row for each element with position in the given array or map column. + * Uses the default column name `pos` for position, and `col` for elements in the array + * and `key` and `value` for elements in the map unless specified otherwise. * Unlike posexplode, if the array/map is null or empty then the row (null, null) is produced. * * @group collection_funcs