Skip to content

Commit 639c9a0

Browse files
zhangstar333zhangstar333
authored
[cherry-pick](branch-30)add some check for udf when result is null (#51084) (#51198)
Problem Summary: cherry-pick from master (#51084)
1 parent d985f92 commit 639c9a0

File tree

5 files changed

+97
-1
lines changed

5 files changed

+97
-1
lines changed

‎fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,16 @@ public void reset() {
310310
}
311311
}
312312

313+
public void checkNullable(Object[] batch, int rows) {
314+
for (int i = 0; i < rows; ++i) {
315+
if (batch[i] == null) {
316+
throw new RuntimeException(
317+
"the result of " + i + " row is null, but the return type is not nullable, please check "
318+
+ "the always_nullable property in create function statement, it's should be true");
319+
}
320+
}
321+
}
322+
313323
public final boolean isNullAt(int rowId) {
314324
if (numNulls == 0 || nullMap == 0) {
315325
return false;
@@ -410,6 +420,7 @@ public void appendBoolean(Boolean[] batch, boolean isNullable) {
410420
}
411421
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
412422
} else {
423+
checkNullable(batch, rows);
413424
for (int i = 0; i < rows; ++i) {
414425
batchData[i] = (byte) (batch[i] ? 1 : 0);
415426
}
@@ -467,6 +478,7 @@ public void appendByte(Byte[] batch, boolean isNullable) {
467478
}
468479
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
469480
} else {
481+
checkNullable(batch, rows);
470482
for (int i = 0; i < rows; ++i) {
471483
batchData[i] = batch[i];
472484
}
@@ -524,6 +536,7 @@ public void appendShort(Short[] batch, boolean isNullable) {
524536
}
525537
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
526538
} else {
539+
checkNullable(batch, rows);
527540
for (int i = 0; i < rows; ++i) {
528541
batchData[i] = batch[i];
529542
}
@@ -581,6 +594,7 @@ public void appendInt(Integer[] batch, boolean isNullable) {
581594
}
582595
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
583596
} else {
597+
checkNullable(batch, rows);
584598
for (int i = 0; i < rows; ++i) {
585599
batchData[i] = batch[i];
586600
}
@@ -638,6 +652,7 @@ public void appendFloat(Float[] batch, boolean isNullable) {
638652
}
639653
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
640654
} else {
655+
checkNullable(batch, rows);
641656
for (int i = 0; i < rows; ++i) {
642657
batchData[i] = batch[i];
643658
}
@@ -695,6 +710,7 @@ public void appendLong(Long[] batch, boolean isNullable) {
695710
}
696711
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
697712
} else {
713+
checkNullable(batch, rows);
698714
for (int i = 0; i < rows; ++i) {
699715
batchData[i] = batch[i];
700716
}
@@ -752,6 +768,7 @@ public void appendDouble(Double[] batch, boolean isNullable) {
752768
}
753769
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
754770
} else {
771+
checkNullable(batch, rows);
755772
for (int i = 0; i < rows; ++i) {
756773
batchData[i] = batch[i];
757774
}
@@ -793,6 +810,9 @@ public int appendBigInteger(BigInteger v) {
793810
}
794811

795812
public void appendBigInteger(BigInteger[] batch, boolean isNullable) {
813+
if (!isNullable) {
814+
checkNullable(batch, batch.length);
815+
}
796816
reserve(appendIndex + batch.length);
797817
for (BigInteger v : batch) {
798818
if (v == null) {
@@ -839,6 +859,9 @@ public int appendDecimal(BigDecimal v) {
839859
}
840860

841861
public void appendDecimal(BigDecimal[] batch, boolean isNullable) {
862+
if (!isNullable) {
863+
checkNullable(batch, batch.length);
864+
}
842865
reserve(appendIndex + batch.length);
843866
for (BigDecimal v : batch) {
844867
if (v == null) {
@@ -885,6 +908,9 @@ public int appendDate(LocalDate v) {
885908
}
886909

887910
public void appendDate(LocalDate[] batch, boolean isNullable) {
911+
if (!isNullable) {
912+
checkNullable(batch, batch.length);
913+
}
888914
reserve(appendIndex + batch.length);
889915
for (LocalDate v : batch) {
890916
if (v == null) {
@@ -951,6 +977,9 @@ public int appendDateTime(LocalDateTime v) {
951977
}
952978

953979
public void appendDateTime(LocalDateTime[] batch, boolean isNullable) {
980+
if (!isNullable) {
981+
checkNullable(batch, batch.length);
982+
}
954983
reserve(appendIndex + batch.length);
955984
for (LocalDateTime v : batch) {
956985
if (v == null) {
@@ -1052,6 +1081,9 @@ public int appendStringAndOffset(String str) {
10521081
}
10531082

10541083
public void appendStringAndOffset(String[] batch, boolean isNullable) {
1084+
if (!isNullable) {
1085+
checkNullable(batch, batch.length);
1086+
}
10551087
reserve(appendIndex + batch.length);
10561088
for (String v : batch) {
10571089
byte[] bytes;
@@ -1068,6 +1100,9 @@ public void appendStringAndOffset(String[] batch, boolean isNullable) {
10681100
}
10691101

10701102
public void appendBinaryAndOffset(byte[][] batch, boolean isNullable) {
1103+
if (!isNullable) {
1104+
checkNullable(batch, batch.length);
1105+
}
10711106
reserve(appendIndex + batch.length);
10721107
for (byte[] v : batch) {
10731108
byte[] bytes = v;
@@ -1121,6 +1156,9 @@ public int appendArray(List<ColumnValue> values) {
11211156
}
11221157

11231158
public void appendArray(List<Object>[] batch, boolean isNullable) {
1159+
if (!isNullable) {
1160+
checkNullable(batch, batch.length);
1161+
}
11241162
reserve(appendIndex + batch.length);
11251163
int offset = childColumns[0].appendIndex;
11261164
for (List<Object> v : batch) {
@@ -1181,6 +1219,9 @@ public int appendMap(List<ColumnValue> keys, List<ColumnValue> values) {
11811219
}
11821220

11831221
public void appendMap(Map<Object, Object>[] batch, boolean isNullable) {
1222+
if (!isNullable) {
1223+
checkNullable(batch, batch.length);
1224+
}
11841225
reserve(appendIndex + batch.length);
11851226
int offset = childColumns[0].appendIndex;
11861227
for (Map<Object, Object> v : batch) {
@@ -1247,6 +1288,9 @@ public int appendStruct(List<Integer> structFieldIndex, List<ColumnValue> values
12471288
}
12481289

12491290
public void appendStruct(Map<String, Object>[] batch, boolean isNullable) {
1291+
if (!isNullable) {
1292+
checkNullable(batch, batch.length);
1293+
}
12501294
reserve(appendIndex + batch.length);
12511295
Object[][] columnData = new Object[childColumns.length][];
12521296
Preconditions.checkArgument(this.getColumnType().getChildNames().size() == childColumns.length);

‎regression-test/java-udf-src/src/main/java/org/apache/doris/udf/StringTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222

2323
public class StringTest extends UDF {
2424
public String evaluate(String field, Integer a, Integer b) {
25+
if (field == null || a == null || b == null) {
26+
return null;
27+
}
2528
return field.substring(0, a) + StringUtils.repeat("*", field.length() - a -b) + field.substring(field.length()-b);
2629
}
2730
}

‎regression-test/suites/javaudf_p0/test_javaudf_array.groovy‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,19 @@ suite("test_javaudf_array") {
124124
"type"="JAVA_UDF"
125125
); """
126126
qt_select_14 """ SELECT java_udf_array_list_test(array(string_col)), string_col, tinyint_col as result FROM ${tableName} ORDER BY result; """
127-
127+
sql """ CREATE FUNCTION java_udf_array_list_test_not_nullable(array<string>) RETURNS array<string> PROPERTIES (
128+
"file"="file://${jarPath}",
129+
"symbol"="org.apache.doris.udf.ArrayListTest",
130+
"always_nullable"="false",
131+
"type"="JAVA_UDF"
132+
); """
133+
test {
134+
sql """ SELECT java_udf_array_list_test_not_nullable(NULL); """
135+
exception "but the return type is not nullable"
136+
}
128137
} finally {
129138
try_sql("DROP FUNCTION IF EXISTS java_udf_array_int_test(array<int>);")
139+
try_sql("DROP FUNCTION IF EXISTS java_udf_array_list_test_not_nullable(array<string>);")
130140
try_sql("DROP FUNCTION IF EXISTS java_udf_array_return_int_test(array<int>);")
131141
try_sql("DROP FUNCTION IF EXISTS java_udf_array_return_string_test(array<string>);")
132142
try_sql("DROP FUNCTION IF EXISTS java_udf_array_string_test(array<string>);")

‎regression-test/suites/javaudf_p0/test_javaudf_int.groovy‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,13 +123,39 @@ suite("test_javaudf_int") {
123123
qt_select_global_3 """ SELECT java_udf_int_test_global(3) result FROM ${tableName} ORDER BY result; """
124124
qt_select_global_4 """ SELECT abs(java_udf_int_test_global(3)) result FROM ${tableName} ORDER BY result; """
125125

126+
sql """ CREATE FUNCTION java_udf_int_test_not_nullable(int) RETURNS int PROPERTIES (
127+
"file"="file://${jarPath}",
128+
"symbol"="org.apache.doris.udf.IntTest",
129+
"always_nullable"="false",
130+
"type"="JAVA_UDF"
131+
); """
132+
133+
test {
134+
sql """ SELECT java_udf_int_test_not_nullable(NULL); """
135+
exception "but the return type is not nullable"
136+
}
137+
138+
sql """ CREATE FUNCTION java_udf_largeint_test_not_nullable(largeint) RETURNS largeint PROPERTIES (
139+
"file"="file://${jarPath}",
140+
"symbol"="org.apache.doris.udf.LargeintTest",
141+
"always_nullable"="false",
142+
"type"="JAVA_UDF"
143+
); """
144+
145+
test {
146+
sql """ SELECT java_udf_largeint_test_not_nullable(NULL); """
147+
exception "but the return type is not nullable"
148+
}
149+
126150
} finally {
127151
try_sql("DROP GLOBAL FUNCTION IF EXISTS java_udf_int_test_global(int);")
128152
try_sql("DROP FUNCTION IF EXISTS java_udf_tinyint_test(tinyint);")
129153
try_sql("DROP FUNCTION IF EXISTS java_udf_smallint_test(smallint);")
130154
try_sql("DROP FUNCTION IF EXISTS java_udf_bigint_test(bigint);")
131155
try_sql("DROP FUNCTION IF EXISTS java_udf_largeint_test(largeint);")
132156
try_sql("DROP FUNCTION IF EXISTS java_udf_int_test(int);")
157+
try_sql("DROP FUNCTION IF EXISTS java_udf_int_test_not_nullable(int);")
158+
try_sql("DROP FUNCTION IF EXISTS java_udf_largeint_test_not_nullable(largeint);")
133159
try_sql("DROP TABLE IF EXISTS ${tableName}")
134160
}
135161
}

‎regression-test/suites/javaudf_p0/test_javaudf_string.groovy‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,21 @@ suite("test_javaudf_string") {
9898
}
9999
sql """ insert into tbl1 select random()%10000 * 10000, "5" from tbl1;"""
100100
qt_select_5 """ select count(0) from (select k1, max(k2) as k2 from tbl1 group by k1)v where java_udf_string_test(k2, 0, 1) = "asd" """;
101+
102+
sql """ CREATE FUNCTION java_udf_string_test_not_nullabel(string, int, int) RETURNS string PROPERTIES (
103+
"file"="file://${jarPath}",
104+
"symbol"="org.apache.doris.udf.StringTest",
105+
"always_nullable"="false",
106+
"type"="JAVA_UDF"
107+
); """
108+
109+
test {
110+
sql """ SELECT java_udf_string_test_not_nullabel(NULL,NULL,NULL); """
111+
exception "but the return type is not nullable"
112+
}
101113
} finally {
102114
try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int, int);")
115+
try_sql("DROP FUNCTION IF EXISTS java_udf_string_test_not_nullabel(string, int, int);")
103116
try_sql("DROP TABLE IF EXISTS ${tableName}")
104117
try_sql("DROP TABLE IF EXISTS tbl1")
105118
try_sql("DROP TABLE IF EXISTS test_javaudf_string_2")

0 commit comments

Comments
 (0)