Skip to content

[SPARK-3485][SQL] Use GenericUDFUtils.ConversionHelper for Simple UDF type conversions #2407

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
adrian-wang wants to merge 2 commits into from
Closed

[SPARK-3485][SQL] Use GenericUDFUtils.ConversionHelper for Simple UDF type conversions #2407

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0d69eb4
another way to pass to hive simple udf
adrian-wang Sep 16, 2014
15762d2
add posmod test which would fail the test but now ok
adrian-wang Sep 17, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ...atibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -801,6 +801,7 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
"udf_or",
"udf_parse_url",
"udf_PI",
"udf_pmod",
"udf_positive",
"udf_pow",
"udf_power",
Expand Down
55 changes: 16 additions & 39 deletions sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUdfs.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.spark.sql.hive

import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils.ConversionHelper

import scala.collection.mutable.ArrayBuffer

import org.apache.hadoop.hive.common.`type`.HiveDecimal
Expand Down Expand Up @@ -104,52 +106,27 @@ private[hive] case class HiveSimpleUdf(functionClassName: String, children: Seq[
function.getResolver.getEvalMethod(children.map(_.dataType.toTypeInfo))

@transient
lazy val dataType = javaClassToDataType(method.getReturnType)
protected lazy val arguments = children.map(c => toInspector(c.dataType)).toArray

protected lazy val wrappers: Array[(Any) => AnyRef] = method.getParameterTypes.map { argClass =>
val primitiveClasses = Seq(
Integer.TYPE, classOf[java.lang.Integer], classOf[java.lang.String], java.lang.Double.TYPE,
classOf[java.lang.Double], java.lang.Long.TYPE, classOf[java.lang.Long],
classOf[HiveDecimal], java.lang.Byte.TYPE, classOf[java.lang.Byte],
classOf[java.sql.Timestamp]
)
val matchingConstructor = argClass.getConstructors.find { c =>
c.getParameterTypes.size == 1 && primitiveClasses.contains(c.getParameterTypes.head)
}
// Create parameter converters
@transient
protected lazy val conversionHelper = new ConversionHelper(method, arguments)

matchingConstructor match {
case Some(constructor) =>
(a: Any) => {
logDebug(
s"Wrapping $a of type ${if (a == null) "null" else a.getClass.getName} $constructor.")
// We must make sure that primitives get boxed java style.
if (a == null) {
null
} else {
constructor.newInstance(a match {
case i: Int => i: java.lang.Integer
case bd: BigDecimal => new HiveDecimal(bd.underlying())
case other: AnyRef => other
}).asInstanceOf[AnyRef]
}
}
case None =>
(a: Any) => a match {
case wrapper => wrap(wrapper)
}
}
@transient
lazy val dataType = javaClassToDataType(method.getReturnType)

def catalystToHive(value: Any): Object = value match {
// TODO need more types here? or can we use wrap()
case bd: BigDecimal => new HiveDecimal(bd.underlying())
case d => d.asInstanceOf[Object]
}

// TODO: Finish input output types.
override def eval(input: Row): Any = {
val evaluatedChildren = children.map(_.eval(input))
// Wrap the function arguments in the expected types.
val args = evaluatedChildren.zip(wrappers).map {
case (arg, wrapper) => wrapper(arg)
}
val evaluatedChildren = children.map(c => catalystToHive(c.eval(input)))

// Invoke the udf and unwrap the result.
unwrap(method.invoke(function, args: _*))
unwrap(FunctionRegistry.invoke(method, function, conversionHelper
.convertIfNecessary(evaluatedChildren: _*): _*))
}
}

Expand Down