Skip to content

Commit

Permalink
[fix](udf)java udf does not support overloaded evaluate method (apach…
Browse files Browse the repository at this point in the history
…e#22681)

Co-authored-by: morningman <morningman@163.com>
  • Loading branch information
2 people authored and 胥剑旭 committed Dec 14, 2023
1 parent bd33a16 commit ba9b420
Show file tree
Hide file tree
Showing 4 changed files with 192 additions and 34 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,12 @@
import java.net.URLClassLoader;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

// create a user define function
public class CreateFunctionStmt extends DdlStmt {
Expand Down Expand Up @@ -527,45 +529,56 @@ private void analyzeJavaUdf(String clazz) throws AnalysisException {
URL[] urls = {new URL("jar:" + userFile + "!/")};
try (URLClassLoader cl = URLClassLoader.newInstance(urls)) {
Class udfClass = cl.loadClass(clazz);

Method eval = null;
for (Method m : udfClass.getMethods()) {
if (!m.getDeclaringClass().equals(udfClass)) {
continue;
}
String name = m.getName();
if (EVAL_METHOD_KEY.equals(name) && eval == null) {
eval = m;
} else if (EVAL_METHOD_KEY.equals(name)) {
throw new AnalysisException(String.format(
"UDF class '%s' has multiple methods with name '%s' ", udfClass.getCanonicalName(),
EVAL_METHOD_KEY));
}
}
if (eval == null) {
List<Method> evalList = Arrays.stream(udfClass.getMethods())
.filter(m -> m.getDeclaringClass().equals(udfClass) && EVAL_METHOD_KEY.equals(m.getName()))
.collect(Collectors.toList());
if (evalList.size() == 0) {
throw new AnalysisException(String.format(
"No method '%s' in class '%s'!", EVAL_METHOD_KEY, udfClass.getCanonicalName()));
"No method '%s' in class '%s'!", EVAL_METHOD_KEY, udfClass.getCanonicalName()));
}
if (Modifier.isStatic(eval.getModifiers())) {
List<Method> evalNonStaticAndPublicList = evalList.stream()
.filter(m -> !Modifier.isStatic(m.getModifiers()) && Modifier.isPublic(m.getModifiers()))
.collect(Collectors.toList());
if (evalNonStaticAndPublicList.size() == 0) {
throw new AnalysisException(
String.format("Method '%s' in class '%s' should be non-static", eval.getName(),
udfClass.getCanonicalName()));
String.format("Method '%s' in class '%s' should be non-static and public", EVAL_METHOD_KEY,
udfClass.getCanonicalName()));
}
if (!Modifier.isPublic(eval.getModifiers())) {
List<Method> evalArgLengthMatchList = evalNonStaticAndPublicList.stream().filter(
m -> m.getParameters().length == argsDef.getArgTypes().length).collect(Collectors.toList());
if (evalArgLengthMatchList.size() == 0) {
throw new AnalysisException(
String.format("Method '%s' in class '%s' should be public", eval.getName(),
udfClass.getCanonicalName()));
}
if (eval.getParameters().length != argsDef.getArgTypes().length) {
throw new AnalysisException(
String.format("The number of parameters for method '%s' in class '%s' should be %d",
eval.getName(), udfClass.getCanonicalName(), argsDef.getArgTypes().length));
}

checkUdfType(udfClass, eval, returnType.getType(), eval.getReturnType(), "return");
for (int i = 0; i < eval.getParameters().length; i++) {
Parameter p = eval.getParameters()[i];
checkUdfType(udfClass, eval, argsDef.getArgTypes()[i], p.getType(), p.getName());
String.format("The number of parameters for method '%s' in class '%s' should be %d",
EVAL_METHOD_KEY, udfClass.getCanonicalName(), argsDef.getArgTypes().length));
} else if (evalArgLengthMatchList.size() == 1) {
Method method = evalArgLengthMatchList.get(0);
checkUdfType(udfClass, method, returnType.getType(), method.getReturnType(), "return");
for (int i = 0; i < method.getParameters().length; i++) {
Parameter p = method.getParameters()[i];
checkUdfType(udfClass, method, argsDef.getArgTypes()[i], p.getType(), p.getName());
}
} else {
// If multiple methods have the same parameters,
// the error message returned cannot be as specific as a single method
boolean hasError = false;
for (Method method : evalArgLengthMatchList) {
try {
checkUdfType(udfClass, method, returnType.getType(), method.getReturnType(), "return");
for (int i = 0; i < method.getParameters().length; i++) {
Parameter p = method.getParameters()[i];
checkUdfType(udfClass, method, argsDef.getArgTypes()[i], p.getType(), p.getName());
}
hasError = false;
break;
} catch (AnalysisException e) {
hasError = true;
}
}
if (hasError) {
throw new AnalysisException(String.format(
"Multi methods '%s' in class '%s' and no one passed parameter matching verification",
EVAL_METHOD_KEY, udfClass.getCanonicalName()));
}
}
} catch (ClassNotFoundException e) {
throw new AnalysisException("Class [" + clazz + "] not found in file :" + userFile);
Expand Down
23 changes: 23 additions & 0 deletions regression-test/data/javaudf_p0/test_javaudf_multi_evaluate.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select_default --
1 1.11 2 3
2 \N 3 4
3 3.33 2 \N

-- !select --
2.11

-- !select --
\N

-- !select --
5

-- !select --
\N

-- !select --
7

-- !select --
\N
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.udf;
import org.apache.hadoop.hive.ql.exec.UDF;

public class MultiEvaluateTest extends UDF {

public Float evaluate(Float flo) {
return flo == null ? null : flo + 1;
}

public Integer evaluate(Integer value) {
return value == null ? null : value + 1;
}

public Integer evaluate(Integer value1, Integer value2) {
return value1 == null || value2 == null ? null : value1 + value2;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

import org.codehaus.groovy.runtime.IOGroovyMethods

import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.Paths

suite("test_javaudf_multi_evaluate") {
def tableName = "test_javaudf_multi_evaluate"
def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar"""

log.info("Jar path: ${jarPath}".toString())
try {
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
CREATE TABLE IF NOT EXISTS ${tableName} (
`user_id` INT NOT NULL COMMENT "",
`float_1` FLOAT COMMENT "",
`int_1` INT COMMENT "",
`int_2` INT COMMENT ""
)
DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
"""


sql """ INSERT INTO ${tableName} (`user_id`,`float_1`,`int_1`,`int_2`) VALUES
(1,1.11,2,3),
(2,null,3,4),
(3,3.33,2,null);
"""
qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """

File path = new File(jarPath)
if (!path.exists()) {
throw new IllegalStateException("""${jarPath} doesn't exist! """)
}

sql """ CREATE FUNCTION java_udf_multi_evaluate_test(FLOAT) RETURNS FLOAT PROPERTIES (
"file"="file://${jarPath}",
"symbol"="org.apache.doris.udf.MultiEvaluateTest",
"type"="JAVA_UDF"
); """

qt_select """ SELECT java_udf_multi_evaluate_test(float_1) FROM ${tableName} where user_id = 1; """
qt_select """ SELECT java_udf_multi_evaluate_test(float_1) FROM ${tableName} where user_id = 2; """

sql """ CREATE FUNCTION java_udf_multi_evaluate_test(int) RETURNS int PROPERTIES (
"file"="file://${jarPath}",
"symbol"="org.apache.doris.udf.MultiEvaluateTest",
"type"="JAVA_UDF"
); """

qt_select """ SELECT java_udf_multi_evaluate_test(int_2) FROM ${tableName} where user_id = 2; """
qt_select """ SELECT java_udf_multi_evaluate_test(int_2) FROM ${tableName} where user_id = 3; """

sql """ CREATE FUNCTION java_udf_multi_evaluate_test(int,int) RETURNS int PROPERTIES (
"file"="file://${jarPath}",
"symbol"="org.apache.doris.udf.MultiEvaluateTest",
"type"="JAVA_UDF"
); """

qt_select """ SELECT java_udf_multi_evaluate_test(int_1, int_2) FROM ${tableName} where user_id = 2; """
qt_select """ SELECT java_udf_multi_evaluate_test(int_1, int_2) FROM ${tableName} where user_id = 3; """

} finally {
try_sql("DROP FUNCTION IF EXISTS java_udf_multi_evaluate_test(FLOAT);")
try_sql("DROP FUNCTION IF EXISTS java_udf_multi_evaluate_test(int);")
try_sql("DROP FUNCTION IF EXISTS java_udf_multi_evaluate_test(int,int);")
try_sql("DROP TABLE IF EXISTS ${tableName}")
}
}

0 comments on commit ba9b420

Please sign in to comment.