Skip to content

[improve](udf)add some check for udf when result is null #51084

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 2 commits into from
May 22, 2025
Merged

[improve](udf)add some check for udf when result is null #51084

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b2e7ea0
[improve](udf)add some check for udf when result is null
zhangstar333 May 20, 2025
1f65075
update
zhangstar333 May 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 47 additions & 0 deletions ...va-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,16 @@ public void reset() {
}
}

public void checkNullable(Object[] batch, int rows) {
for (int i = 0; i < rows; ++i) {
if (batch[i] == null) {
throw new RuntimeException(
"the result of " + i + " row is null, but the return type is not nullable, please check "
+ "the always_nullable property in create function statement, it's should be true");
}
}
}

public final boolean isNullAt(int rowId) {
if (numNulls == 0 || nullMap == 0) {
return false;
Expand Down Expand Up @@ -454,6 +464,7 @@ public void appendBoolean(Boolean[] batch, boolean isNullable) {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
} else {
checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = (byte) (batch[i] ? 1 : 0);
}
Expand Down Expand Up @@ -511,6 +522,7 @@ public void appendByte(Byte[] batch, boolean isNullable) {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
} else {
checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
Expand Down Expand Up @@ -568,6 +580,7 @@ public void appendShort(Short[] batch, boolean isNullable) {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
} else {
checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
Expand Down Expand Up @@ -625,6 +638,7 @@ public void appendInt(Integer[] batch, boolean isNullable) {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
} else {
checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
Expand Down Expand Up @@ -682,6 +696,7 @@ public void appendFloat(Float[] batch, boolean isNullable) {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
} else {
checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
Expand Down Expand Up @@ -739,6 +754,7 @@ public void appendLong(Long[] batch, boolean isNullable) {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
} else {
checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
Expand Down Expand Up @@ -796,6 +812,7 @@ public void appendDouble(Double[] batch, boolean isNullable) {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
} else {
checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
Expand Down Expand Up @@ -837,6 +854,9 @@ public int appendBigInteger(BigInteger v) {
}

public void appendBigInteger(BigInteger[] batch, boolean isNullable) {
if (!isNullable) {
checkNullable(batch, batch.length);
}
reserve(appendIndex + batch.length);
for (BigInteger v : batch) {
if (v == null) {
Expand Down Expand Up @@ -904,6 +924,9 @@ public int appendInetAddress(InetAddress v) {
}

public void appendInetAddress(InetAddress[] batch, boolean isNullable) {
if (!isNullable) {
checkNullable(batch, batch.length);
}
reserve(appendIndex + batch.length);
for (InetAddress v : batch) {
if (v == null) {
Expand Down Expand Up @@ -933,6 +956,9 @@ public int appendDecimal(BigDecimal v) {
}

public void appendDecimal(BigDecimal[] batch, boolean isNullable) {
if (!isNullable) {
checkNullable(batch, batch.length);
}
reserve(appendIndex + batch.length);
for (BigDecimal v : batch) {
if (v == null) {
Expand Down Expand Up @@ -979,6 +1005,9 @@ public int appendDate(LocalDate v) {
}

public void appendDate(LocalDate[] batch, boolean isNullable) {
if (!isNullable) {
checkNullable(batch, batch.length);
}
reserve(appendIndex + batch.length);
for (LocalDate v : batch) {
if (v == null) {
Expand Down Expand Up @@ -1045,6 +1074,9 @@ public int appendDateTime(LocalDateTime v) {
}

public void appendDateTime(LocalDateTime[] batch, boolean isNullable) {
if (!isNullable) {
checkNullable(batch, batch.length);
}
reserve(appendIndex + batch.length);
for (LocalDateTime v : batch) {
if (v == null) {
Expand Down Expand Up @@ -1146,6 +1178,9 @@ public int appendStringAndOffset(String str) {
}

public void appendStringAndOffset(String[] batch, boolean isNullable) {
if (!isNullable) {
checkNullable(batch, batch.length);
}
reserve(appendIndex + batch.length);
for (String v : batch) {
byte[] bytes;
Expand All @@ -1162,6 +1197,9 @@ public void appendStringAndOffset(String[] batch, boolean isNullable) {
}

public void appendBinaryAndOffset(byte[][] batch, boolean isNullable) {
if (!isNullable) {
checkNullable(batch, batch.length);
}
reserve(appendIndex + batch.length);
for (byte[] v : batch) {
byte[] bytes = v;
Expand Down Expand Up @@ -1215,6 +1253,9 @@ public int appendArray(List<ColumnValue> values) {
}

public void appendArray(List<Object>[] batch, boolean isNullable) {
if (!isNullable) {
checkNullable(batch, batch.length);
}
reserve(appendIndex + batch.length);
int offset = childColumns[0].appendIndex;
for (List<Object> v : batch) {
Expand Down Expand Up @@ -1275,6 +1316,9 @@ public int appendMap(List<ColumnValue> keys, List<ColumnValue> values) {
}

public void appendMap(Map<Object, Object>[] batch, boolean isNullable) {
if (!isNullable) {
checkNullable(batch, batch.length);
}
reserve(appendIndex + batch.length);
int offset = childColumns[0].appendIndex;
for (Map<Object, Object> v : batch) {
Expand Down Expand Up @@ -1341,6 +1385,9 @@ public int appendStruct(List<Integer> structFieldIndex, List<ColumnValue> values
}

public void appendStruct(Map<String, Object>[] batch, boolean isNullable) {
if (!isNullable) {
checkNullable(batch, batch.length);
}
reserve(appendIndex + batch.length);
Object[][] columnData = new Object[childColumns.length][];
Preconditions.checkArgument(this.getColumnType().getChildNames().size() == childColumns.length);
Expand Down
3 changes: 3 additions & 0 deletions regression-test/java-udf-src/src/main/java/org/apache/doris/udf/StringTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@

public class StringTest extends UDF {
public String evaluate(String field, Integer a, Integer b) {
if (field == null || a == null || b == null) {
return null;
}
return field.substring(0, a) + StringUtils.repeat("*", field.length() - a -b) + field.substring(field.length()-b);
}
}
12 changes: 11 additions & 1 deletion regression-test/suites/javaudf_p0/test_javaudf_array.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,19 @@ suite("test_javaudf_array") {
"type"="JAVA_UDF"
); """
qt_select_14 """ SELECT java_udf_array_list_test(array(string_col)), string_col, tinyint_col as result FROM ${tableName} ORDER BY result; """

sql """ CREATE FUNCTION java_udf_array_list_test_not_nullable(array<string>) RETURNS array<string> PROPERTIES (
"file"="file://${jarPath}",
"symbol"="org.apache.doris.udf.ArrayListTest",
"always_nullable"="false",
"type"="JAVA_UDF"
); """
test {
sql """ SELECT java_udf_array_list_test_not_nullable(NULL); """
exception "but the return type is not nullable"
}
} finally {
try_sql("DROP FUNCTION IF EXISTS java_udf_array_int_test(array<int>);")
try_sql("DROP FUNCTION IF EXISTS java_udf_array_list_test_not_nullable(array<string>);")
try_sql("DROP FUNCTION IF EXISTS java_udf_array_return_int_test(array<int>);")
try_sql("DROP FUNCTION IF EXISTS java_udf_array_return_string_test(array<string>);")
try_sql("DROP FUNCTION IF EXISTS java_udf_array_string_test(array<string>);")
Expand Down
26 changes: 26 additions & 0 deletions regression-test/suites/javaudf_p0/test_javaudf_int.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,13 +123,39 @@ suite("test_javaudf_int") {
qt_select_global_3 """ SELECT java_udf_int_test_global(3) result FROM ${tableName} ORDER BY result; """
qt_select_global_4 """ SELECT abs(java_udf_int_test_global(3)) result FROM ${tableName} ORDER BY result; """

sql """ CREATE FUNCTION java_udf_int_test_not_nullable(int) RETURNS int PROPERTIES (
"file"="file://${jarPath}",
"symbol"="org.apache.doris.udf.IntTest",
"always_nullable"="false",
"type"="JAVA_UDF"
); """

test {
sql """ SELECT java_udf_int_test_not_nullable(NULL); """
exception "but the return type is not nullable"
}

sql """ CREATE FUNCTION java_udf_largeint_test_not_nullable(largeint) RETURNS largeint PROPERTIES (
"file"="file://${jarPath}",
"symbol"="org.apache.doris.udf.LargeintTest",
"always_nullable"="false",
"type"="JAVA_UDF"
); """

test {
sql """ SELECT java_udf_largeint_test_not_nullable(NULL); """
exception "but the return type is not nullable"
}

} finally {
try_sql("DROP GLOBAL FUNCTION IF EXISTS java_udf_int_test_global(int);")
try_sql("DROP FUNCTION IF EXISTS java_udf_tinyint_test(tinyint);")
try_sql("DROP FUNCTION IF EXISTS java_udf_smallint_test(smallint);")
try_sql("DROP FUNCTION IF EXISTS java_udf_bigint_test(bigint);")
try_sql("DROP FUNCTION IF EXISTS java_udf_largeint_test(largeint);")
try_sql("DROP FUNCTION IF EXISTS java_udf_int_test(int);")
try_sql("DROP FUNCTION IF EXISTS java_udf_int_test_not_nullable(int);")
try_sql("DROP FUNCTION IF EXISTS java_udf_largeint_test_not_nullable(largeint);")
try_sql("DROP TABLE IF EXISTS ${tableName}")
}
}
13 changes: 13 additions & 0 deletions regression-test/suites/javaudf_p0/test_javaudf_string.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,21 @@ suite("test_javaudf_string") {
}
sql """ insert into tbl1 select random()%10000 * 10000, "5" from tbl1;"""
qt_select_5 """ select count(0) from (select k1, max(k2) as k2 from tbl1 group by k1)v where java_udf_string_test(k2, 0, 1) = "asd" """;

sql """ CREATE FUNCTION java_udf_string_test_not_nullabel(string, int, int) RETURNS string PROPERTIES (
"file"="file://${jarPath}",
"symbol"="org.apache.doris.udf.StringTest",
"always_nullable"="false",
"type"="JAVA_UDF"
); """

test {
sql """ SELECT java_udf_string_test_not_nullabel(NULL,NULL,NULL); """
exception "but the return type is not nullable"
}
} finally {
try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int, int);")
try_sql("DROP FUNCTION IF EXISTS java_udf_string_test_not_nullabel(string, int, int);")
try_sql("DROP TABLE IF EXISTS ${tableName}")
try_sql("DROP TABLE IF EXISTS tbl1")
try_sql("DROP TABLE IF EXISTS test_javaudf_string_2")
Expand Down
Loading