From 3a3d77711a0723d61761adb46c3e9df04bbf58c1 Mon Sep 17 00:00:00 2001 From: philo Date: Thu, 19 May 2022 09:34:12 +0800 Subject: [PATCH 1/9] Initial commit --- .../ColumnarExpressionConverter.scala | 8 ++ .../scala/com/intel/oap/expression/UDF.scala | 91 +++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala index e4b49eb51..6ceff353f 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala @@ -399,6 +399,14 @@ object ColumnarExpressionConverter extends Logging { attributeSeq, convertBoundRefToAttrRef = convertBoundRefToAttrRef), expr) + case expr if (UDF.isSupportedUDF(expr.prettyName)) => + val children = expr.children.map { expr => + replaceWithColumnarExpression( + expr, + attributeSeq, + convertBoundRefToAttrRef = convertBoundRefToAttrRef) + } + UDF.create(children, expr) case expr => throw new UnsupportedOperationException( s" --> ${expr.getClass} | ${expr} is not currently supported.") diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala new file mode 100644 index 000000000..93694b9c7 --- /dev/null +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.oap.expression + +import com.google.common.collect.Lists +import org.apache.arrow.gandiva.expression.{TreeBuilder, TreeNode} +import org.apache.arrow.vector.types.pojo.ArrowType +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} +import org.apache.spark.sql.types.StringType +import org.apache.spark.sql.types.DataType + + +case class ColumnarURLDecoder(input: Expression) extends Expression with ColumnarExpression { + def nullable: Boolean = { + true + } + + def children: Seq[Expression] = { + Seq(input) + } + + def dataType: DataType = { + StringType + } + + def eval(input: InternalRow): Any = { + } + + def child: Expression = { + input + } + + override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + throw new UnsupportedOperationException("Should not trigger code gen") + } + + buildCheck + + def buildCheck: Unit = { + val supportedTypes = List(StringType) + if (!supportedTypes.contains(input.dataType)) { + throw new UnsupportedOperationException("Only StringType input is supported!"); + } + } + + override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = { + val (inputNode, _): (TreeNode, ArrowType) = + input.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) + val resultType = new ArrowType.Utf8() + val funcNode = + TreeBuilder.makeFunction( + "url_decoder", + Lists.newArrayList(inputNode), + resultType) + (funcNode, resultType) + } +} + +object UDF { + val supportList = {"TestUDF"} + + def isSupportedUDF(name: String): Boolean = { + return supportList.contains(name) + } + + def create(children: Seq[Expression], original: Expression): Expression = { + original.prettyName match { + case "TestUDF" => + new ColumnarURLDecoder(original.children.head) + case other => + throw new UnsupportedOperationException(s"not currently supported: $other.") + } + } +} From 90bfa7a0f3ea24d30691b8c4b6aacc79355f8ea1 Mon Sep 17 00:00:00 2001 From: philo Date: Thu, 19 May 2022 15:06:32 +0800 Subject: [PATCH 2/9] Fix bugs --- .../oap/expression/ColumnarExpressionConverter.scala | 2 ++ .../src/main/scala/com/intel/oap/expression/UDF.scala | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala index 6ceff353f..b809634e0 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala @@ -468,6 +468,8 @@ object ColumnarExpressionConverter extends Logging { containsSubquery(sr.srcExpr) || containsSubquery(sr.searchExpr) || containsSubquery(sr.replaceExpr) + case expr if (UDF.isSupportedUDF(expr.prettyName)) => + expr.children.map(containsSubquery).exists(_ == true) case expr => throw new UnsupportedOperationException( s" --> ${expr.getClass} | ${expr} is not currently supported.") diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala index 93694b9c7..619d82a89 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala @@ -51,6 +51,10 @@ case class ColumnarURLDecoder(input: Expression) extends Expression with Columna throw new UnsupportedOperationException("Should not trigger code gen") } + protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): ColumnarURLDecoder = { + copy(input = newChildren.head) + } + buildCheck def buildCheck: Unit = { @@ -74,7 +78,7 @@ case class ColumnarURLDecoder(input: Expression) extends Expression with Columna } object UDF { - val supportList = {"TestUDF"} + val supportList = {"testUDF"} def isSupportedUDF(name: String): Boolean = { return supportList.contains(name) @@ -82,8 +86,8 @@ object UDF { def create(children: Seq[Expression], original: Expression): Expression = { original.prettyName match { - case "TestUDF" => - new ColumnarURLDecoder(original.children.head) + case "testUDF" => + new ColumnarURLDecoder(children.head) case other => throw new UnsupportedOperationException(s"not currently supported: $other.") } From 48a0d5e85485c7f76e136a5d3cefedfeacb76f55 Mon Sep 17 00:00:00 2001 From: philo Date: Thu, 19 May 2022 15:28:32 +0800 Subject: [PATCH 3/9] Let supportColumnarCodegen return false --- .../core/src/main/scala/com/intel/oap/expression/UDF.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala index 619d82a89..059ff1a69 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala @@ -64,6 +64,10 @@ case class ColumnarURLDecoder(input: Expression) extends Expression with Columna } } + override def supportColumnarCodegen(args: java.lang.Object): Boolean = { + false + } + override def doColumnarCodeGen(args: Object): (TreeNode, ArrowType) = { val (inputNode, _): (TreeNode, ArrowType) = input.asInstanceOf[ColumnarExpression].doColumnarCodeGen(args) @@ -87,7 +91,7 @@ object UDF { def create(children: Seq[Expression], original: Expression): Expression = { original.prettyName match { case "testUDF" => - new ColumnarURLDecoder(children.head) + ColumnarURLDecoder(children.head) case other => throw new UnsupportedOperationException(s"not currently supported: $other.") } From fa7a444d5199d2b09c116538c48de241ca5942f7 Mon Sep 17 00:00:00 2001 From: philo Date: Thu, 19 May 2022 20:09:07 +0800 Subject: [PATCH 4/9] Align the UDF name with the real --- .../main/scala/com/intel/oap/expression/UDF.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala index 059ff1a69..4c013e420 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala @@ -20,11 +20,12 @@ package com.intel.oap.expression import com.google.common.collect.Lists import org.apache.arrow.gandiva.expression.{TreeBuilder, TreeNode} import org.apache.arrow.vector.types.pojo.ArrowType + import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode} -import org.apache.spark.sql.types.StringType import org.apache.spark.sql.types.DataType +import org.apache.spark.sql.types.StringType case class ColumnarURLDecoder(input: Expression) extends Expression with ColumnarExpression { @@ -41,6 +42,7 @@ case class ColumnarURLDecoder(input: Expression) extends Expression with Columna } def eval(input: InternalRow): Any = { + throw new UnsupportedOperationException("Should not trigger eval!") } def child: Expression = { @@ -48,7 +50,7 @@ case class ColumnarURLDecoder(input: Expression) extends Expression with Columna } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - throw new UnsupportedOperationException("Should not trigger code gen") + throw new UnsupportedOperationException("Should not trigger code gen!") } protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): ColumnarURLDecoder = { @@ -60,7 +62,7 @@ case class ColumnarURLDecoder(input: Expression) extends Expression with Columna def buildCheck: Unit = { val supportedTypes = List(StringType) if (!supportedTypes.contains(input.dataType)) { - throw new UnsupportedOperationException("Only StringType input is supported!"); + throw new UnsupportedOperationException("Only StringType input is supported!") } } @@ -82,7 +84,10 @@ case class ColumnarURLDecoder(input: Expression) extends Expression with Columna } object UDF { - val supportList = {"testUDF"} + // Keep the supported UDF name. The name is specified in registering the + // row based function in spark, e.g., + // CREATE TEMPORARY FUNCTION UrlDecoder AS 'com.intel.test.URLDecoderNew'; + val supportList = {"UrlDecoder"} def isSupportedUDF(name: String): Boolean = { return supportList.contains(name) @@ -90,7 +95,7 @@ object UDF { def create(children: Seq[Expression], original: Expression): Expression = { original.prettyName match { - case "testUDF" => + case "UrlDecoder" => ColumnarURLDecoder(children.head) case other => throw new UnsupportedOperationException(s"not currently supported: $other.") From 72a5fe82a11354a155a05bacb626f3ae422c2104 Mon Sep 17 00:00:00 2001 From: philo Date: Mon, 23 May 2022 17:29:37 +0800 Subject: [PATCH 5/9] Add scala udf support and a unit test --- .../ColumnarExpressionConverter.scala | 18 +++++++++++++++--- .../{UDF.scala => ColumnarUDF.scala} | 14 +++++++++++++- .../spark/sql/DataFrameFunctionsSuite.scala | 10 ++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) rename native-sql-engine/core/src/main/scala/com/intel/oap/expression/{UDF.scala => ColumnarUDF.scala} (89%) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala index b809634e0..00007f4c8 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala @@ -399,14 +399,24 @@ object ColumnarExpressionConverter extends Logging { attributeSeq, convertBoundRefToAttrRef = convertBoundRefToAttrRef), expr) - case expr if (UDF.isSupportedUDF(expr.prettyName)) => + // Scala UDF. + case expr: ScalaUDF if ColumnarUDF.isSupportedUDF(expr.udfName.get) => val children = expr.children.map { expr => replaceWithColumnarExpression( expr, attributeSeq, convertBoundRefToAttrRef = convertBoundRefToAttrRef) } - UDF.create(children, expr) + ColumnarUDF.create(children, expr) + // Hive UDF. + case expr if (ColumnarUDF.isSupportedUDF(expr.prettyName)) => + val children = expr.children.map { expr => + replaceWithColumnarExpression( + expr, + attributeSeq, + convertBoundRefToAttrRef = convertBoundRefToAttrRef) + } + ColumnarUDF.create(children, expr) case expr => throw new UnsupportedOperationException( s" --> ${expr.getClass} | ${expr} is not currently supported.") @@ -468,7 +478,9 @@ object ColumnarExpressionConverter extends Logging { containsSubquery(sr.srcExpr) || containsSubquery(sr.searchExpr) || containsSubquery(sr.replaceExpr) - case expr if (UDF.isSupportedUDF(expr.prettyName)) => + case expr: ScalaUDF if ColumnarUDF.isSupportedUDF(expr.udfName.get) => + expr.children.map(containsSubquery).exists(_ == true) + case expr if (ColumnarUDF.isSupportedUDF(expr.prettyName)) => expr.children.map(containsSubquery).exists(_ == true) case expr => throw new UnsupportedOperationException( diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUDF.scala similarity index 89% rename from native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala rename to native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUDF.scala index 4c013e420..4709ee6eb 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/UDF.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUDF.scala @@ -83,20 +83,32 @@ case class ColumnarURLDecoder(input: Expression) extends Expression with Columna } } -object UDF { +object ColumnarUDF { // Keep the supported UDF name. The name is specified in registering the // row based function in spark, e.g., // CREATE TEMPORARY FUNCTION UrlDecoder AS 'com.intel.test.URLDecoderNew'; val supportList = {"UrlDecoder"} def isSupportedUDF(name: String): Boolean = { + if (name == null) { + return false; + } return supportList.contains(name) } def create(children: Seq[Expression], original: Expression): Expression = { original.prettyName match { + // Hive UDF. case "UrlDecoder" => ColumnarURLDecoder(children.head) + // Scala UDF. + case "scalaudf" => + original.asInstanceOf[ScalaUDF].udfName.get match { + case "UrlDecoder" => + ColumnarURLDecoder(children.head) + case other => + throw new UnsupportedOperationException(s"not currently supported: $other.") + } case other => throw new UnsupportedOperationException(s"not currently supported: $other.") } diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala index aa1678e4f..cfe36616e 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala @@ -3629,6 +3629,16 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession { df.select(map(map_entries($"m"), lit(1))), Row(Map(Seq(Row(1, "a")) -> 1))) } + + test("Columnar UDF") { + // Register a scala UDF. The scala UDF code will not be acutally used. It + // will be replaced by columnar UDF at runtime. + spark.udf.register("UrlDecoder", (s : String) => s) + checkAnswer( + sql("select UrlDecoder('AaBb%23'), UrlDecoder(null)"), + Seq(Row("AaBb#", null)) + ) + } } object DataFrameFunctionsSuite { From 888a30a3e5f285e52c2cb0ad3da418dc6ba19a24 Mon Sep 17 00:00:00 2001 From: philo Date: Mon, 23 May 2022 17:31:27 +0800 Subject: [PATCH 6/9] Change arrow branch for test [will revert at last] --- arrow-data-source/script/build_arrow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arrow-data-source/script/build_arrow.sh b/arrow-data-source/script/build_arrow.sh index d8ec40128..f35843088 100755 --- a/arrow-data-source/script/build_arrow.sh +++ b/arrow-data-source/script/build_arrow.sh @@ -62,7 +62,7 @@ echo "ARROW_SOURCE_DIR=${ARROW_SOURCE_DIR}" echo "ARROW_INSTALL_DIR=${ARROW_INSTALL_DIR}" mkdir -p $ARROW_SOURCE_DIR mkdir -p $ARROW_INSTALL_DIR -git clone https://github.com/oap-project/arrow.git --branch arrow-4.0.0-oap $ARROW_SOURCE_DIR +git clone https://github.com/PHILO-HE/arrow.git --branch url_decoder $ARROW_SOURCE_DIR pushd $ARROW_SOURCE_DIR cmake ./cpp \ From 1865752d2d675f3c784843dfde7133536487154c Mon Sep 17 00:00:00 2001 From: philo Date: Mon, 23 May 2022 20:03:41 +0800 Subject: [PATCH 7/9] Revert "Change arrow branch for test [will revert at last]" This reverts commit 888a30a3e5f285e52c2cb0ad3da418dc6ba19a24. --- arrow-data-source/script/build_arrow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arrow-data-source/script/build_arrow.sh b/arrow-data-source/script/build_arrow.sh index f35843088..d8ec40128 100755 --- a/arrow-data-source/script/build_arrow.sh +++ b/arrow-data-source/script/build_arrow.sh @@ -62,7 +62,7 @@ echo "ARROW_SOURCE_DIR=${ARROW_SOURCE_DIR}" echo "ARROW_INSTALL_DIR=${ARROW_INSTALL_DIR}" mkdir -p $ARROW_SOURCE_DIR mkdir -p $ARROW_INSTALL_DIR -git clone https://github.com/PHILO-HE/arrow.git --branch url_decoder $ARROW_SOURCE_DIR +git clone https://github.com/oap-project/arrow.git --branch arrow-4.0.0-oap $ARROW_SOURCE_DIR pushd $ARROW_SOURCE_DIR cmake ./cpp \ From d91b46dd3da2807cc12f6bb940f8441dc49b0bc4 Mon Sep 17 00:00:00 2001 From: philo Date: Mon, 23 May 2022 20:21:46 +0800 Subject: [PATCH 8/9] Consider None in getting udfName --- .../expression/ColumnarExpressionConverter.scala | 14 ++++++++++++-- .../apache/spark/sql/DataFrameFunctionsSuite.scala | 10 ---------- .../org/apache/spark/sql/DataFrameSuite.scala | 10 ++++++++++ 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala index 00007f4c8..9206be146 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarExpressionConverter.scala @@ -400,7 +400,12 @@ object ColumnarExpressionConverter extends Logging { convertBoundRefToAttrRef = convertBoundRefToAttrRef), expr) // Scala UDF. - case expr: ScalaUDF if ColumnarUDF.isSupportedUDF(expr.udfName.get) => + case expr: ScalaUDF if (expr.udfName match { + case Some(name) => + ColumnarUDF.isSupportedUDF(name) + case None => + false + }) => val children = expr.children.map { expr => replaceWithColumnarExpression( expr, @@ -478,7 +483,12 @@ object ColumnarExpressionConverter extends Logging { containsSubquery(sr.srcExpr) || containsSubquery(sr.searchExpr) || containsSubquery(sr.replaceExpr) - case expr: ScalaUDF if ColumnarUDF.isSupportedUDF(expr.udfName.get) => + case expr: ScalaUDF if (expr.udfName match { + case Some(name) => + ColumnarUDF.isSupportedUDF(name) + case None => + false + }) => expr.children.map(containsSubquery).exists(_ == true) case expr if (ColumnarUDF.isSupportedUDF(expr.prettyName)) => expr.children.map(containsSubquery).exists(_ == true) diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala index cfe36616e..aa1678e4f 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala @@ -3629,16 +3629,6 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession { df.select(map(map_entries($"m"), lit(1))), Row(Map(Seq(Row(1, "a")) -> 1))) } - - test("Columnar UDF") { - // Register a scala UDF. The scala UDF code will not be acutally used. It - // will be replaced by columnar UDF at runtime. - spark.udf.register("UrlDecoder", (s : String) => s) - checkAnswer( - sql("select UrlDecoder('AaBb%23'), UrlDecoder(null)"), - Seq(Row("AaBb#", null)) - ) - } } object DataFrameFunctionsSuite { diff --git a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala index 356b68798..6ef4943f5 100644 --- a/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/native-sql-engine/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -603,6 +603,16 @@ class DataFrameSuite extends QueryTest ) } + test("Columnar UDF") { + // Register a scala UDF. The scala UDF code will not be acutally used. It + // will be replaced by columnar UDF at runtime. + spark.udf.register("UrlDecoder", (s : String) => s) + checkAnswer( + sql("select UrlDecoder('AaBb%23'), UrlDecoder(null)"), + Seq(Row("AaBb#", null)) + ) + } + test("callUDF without Hive Support") { val df = Seq(("id1", 1), ("id2", 4), ("id3", 5)).toDF("id", "value") df.sparkSession.udf.register("simpleUDF", (v: Int) => v * v) From fa7c51ef1de1c83a5a4ddf0a917711fd2cf1c8c9 Mon Sep 17 00:00:00 2001 From: philo Date: Tue, 24 May 2022 11:46:45 +0800 Subject: [PATCH 9/9] Ignore case in matching UDF name --- .../com/intel/oap/expression/ColumnarUDF.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUDF.scala b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUDF.scala index 4709ee6eb..880e425e8 100644 --- a/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUDF.scala +++ b/native-sql-engine/core/src/main/scala/com/intel/oap/expression/ColumnarUDF.scala @@ -87,24 +87,24 @@ object ColumnarUDF { // Keep the supported UDF name. The name is specified in registering the // row based function in spark, e.g., // CREATE TEMPORARY FUNCTION UrlDecoder AS 'com.intel.test.URLDecoderNew'; - val supportList = {"UrlDecoder"} + val supportList = List("urldecoder") def isSupportedUDF(name: String): Boolean = { if (name == null) { - return false; + return false } - return supportList.contains(name) + return supportList.map(s => s.equalsIgnoreCase(name)).exists(_ == true) } def create(children: Seq[Expression], original: Expression): Expression = { - original.prettyName match { + original.prettyName.toLowerCase() match { // Hive UDF. - case "UrlDecoder" => + case "urldecoder" => ColumnarURLDecoder(children.head) // Scala UDF. case "scalaudf" => - original.asInstanceOf[ScalaUDF].udfName.get match { - case "UrlDecoder" => + original.asInstanceOf[ScalaUDF].udfName.get.toLowerCase() match { + case "urldecoder" => ColumnarURLDecoder(children.head) case other => throw new UnsupportedOperationException(s"not currently supported: $other.")