Skip to content

Commit 6513f9e

Browse files
zhangstar333zhangstar333
authored
[cherry-pick](branch-21)add some check for udf when result is null (#51084) (#51196)
1 parent 1652101 commit 6513f9e

File tree

5 files changed

+97
-0
lines changed

5 files changed

+97
-0
lines changed

‎fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,16 @@ public void reset() {
305305
}
306306
}
307307

308+
public void checkNullable(Object[] batch, int rows) {
309+
for (int i = 0; i < rows; ++i) {
310+
if (batch[i] == null) {
311+
throw new RuntimeException(
312+
"the result of " + i + " row is null, but the return type is not nullable, please check "
313+
+ "the always_nullable property in create function statement, it's should be true");
314+
}
315+
}
316+
}
317+
308318
public final boolean isNullAt(int rowId) {
309319
if (numNulls == 0 || nullMap == 0) {
310320
return false;
@@ -405,6 +415,7 @@ public void appendBoolean(Boolean[] batch, boolean isNullable) {
405415
}
406416
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
407417
} else {
418+
checkNullable(batch, rows);
408419
for (int i = 0; i < rows; ++i) {
409420
batchData[i] = (byte) (batch[i] ? 1 : 0);
410421
}
@@ -462,6 +473,7 @@ public void appendByte(Byte[] batch, boolean isNullable) {
462473
}
463474
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
464475
} else {
476+
checkNullable(batch, rows);
465477
for (int i = 0; i < rows; ++i) {
466478
batchData[i] = batch[i];
467479
}
@@ -519,6 +531,7 @@ public void appendShort(Short[] batch, boolean isNullable) {
519531
}
520532
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
521533
} else {
534+
checkNullable(batch, rows);
522535
for (int i = 0; i < rows; ++i) {
523536
batchData[i] = batch[i];
524537
}
@@ -576,6 +589,7 @@ public void appendInt(Integer[] batch, boolean isNullable) {
576589
}
577590
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
578591
} else {
592+
checkNullable(batch, rows);
579593
for (int i = 0; i < rows; ++i) {
580594
batchData[i] = batch[i];
581595
}
@@ -633,6 +647,7 @@ public void appendFloat(Float[] batch, boolean isNullable) {
633647
}
634648
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
635649
} else {
650+
checkNullable(batch, rows);
636651
for (int i = 0; i < rows; ++i) {
637652
batchData[i] = batch[i];
638653
}
@@ -690,6 +705,7 @@ public void appendLong(Long[] batch, boolean isNullable) {
690705
}
691706
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
692707
} else {
708+
checkNullable(batch, rows);
693709
for (int i = 0; i < rows; ++i) {
694710
batchData[i] = batch[i];
695711
}
@@ -747,6 +763,7 @@ public void appendDouble(Double[] batch, boolean isNullable) {
747763
}
748764
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET, null, nullMap + appendIndex, rows);
749765
} else {
766+
checkNullable(batch, rows);
750767
for (int i = 0; i < rows; ++i) {
751768
batchData[i] = batch[i];
752769
}
@@ -788,6 +805,9 @@ public int appendBigInteger(BigInteger v) {
788805
}
789806

790807
public void appendBigInteger(BigInteger[] batch, boolean isNullable) {
808+
if (!isNullable) {
809+
checkNullable(batch, batch.length);
810+
}
791811
reserve(appendIndex + batch.length);
792812
for (BigInteger v : batch) {
793813
if (v == null) {
@@ -834,6 +854,9 @@ public int appendDecimal(BigDecimal v) {
834854
}
835855

836856
public void appendDecimal(BigDecimal[] batch, boolean isNullable) {
857+
if (!isNullable) {
858+
checkNullable(batch, batch.length);
859+
}
837860
reserve(appendIndex + batch.length);
838861
for (BigDecimal v : batch) {
839862
if (v == null) {
@@ -880,6 +903,9 @@ public int appendDate(LocalDate v) {
880903
}
881904

882905
public void appendDate(LocalDate[] batch, boolean isNullable) {
906+
if (!isNullable) {
907+
checkNullable(batch, batch.length);
908+
}
883909
reserve(appendIndex + batch.length);
884910
for (LocalDate v : batch) {
885911
if (v == null) {
@@ -946,6 +972,9 @@ public int appendDateTime(LocalDateTime v) {
946972
}
947973

948974
public void appendDateTime(LocalDateTime[] batch, boolean isNullable) {
975+
if (!isNullable) {
976+
checkNullable(batch, batch.length);
977+
}
949978
reserve(appendIndex + batch.length);
950979
for (LocalDateTime v : batch) {
951980
if (v == null) {
@@ -1047,6 +1076,9 @@ public int appendStringAndOffset(String str) {
10471076
}
10481077

10491078
public void appendStringAndOffset(String[] batch, boolean isNullable) {
1079+
if (!isNullable) {
1080+
checkNullable(batch, batch.length);
1081+
}
10501082
reserve(appendIndex + batch.length);
10511083
for (String v : batch) {
10521084
byte[] bytes;
@@ -1063,6 +1095,9 @@ public void appendStringAndOffset(String[] batch, boolean isNullable) {
10631095
}
10641096

10651097
public void appendBinaryAndOffset(byte[][] batch, boolean isNullable) {
1098+
if (!isNullable) {
1099+
checkNullable(batch, batch.length);
1100+
}
10661101
reserve(appendIndex + batch.length);
10671102
for (byte[] v : batch) {
10681103
byte[] bytes = v;
@@ -1116,6 +1151,9 @@ public int appendArray(List<ColumnValue> values) {
11161151
}
11171152

11181153
public void appendArray(List<Object>[] batch, boolean isNullable) {
1154+
if (!isNullable) {
1155+
checkNullable(batch, batch.length);
1156+
}
11191157
reserve(appendIndex + batch.length);
11201158
int offset = childColumns[0].appendIndex;
11211159
for (List<Object> v : batch) {
@@ -1175,6 +1213,9 @@ public int appendMap(List<ColumnValue> keys, List<ColumnValue> values) {
11751213
}
11761214

11771215
public void appendMap(Map<Object, Object>[] batch, boolean isNullable) {
1216+
if (!isNullable) {
1217+
checkNullable(batch, batch.length);
1218+
}
11781219
reserve(appendIndex + batch.length);
11791220
int offset = childColumns[0].appendIndex;
11801221
for (Map<Object, Object> v : batch) {
@@ -1239,6 +1280,9 @@ public int appendStruct(List<Integer> structFieldIndex, List<ColumnValue> values
12391280
}
12401281

12411282
public void appendStruct(Map<String, Object>[] batch, boolean isNullable) {
1283+
if (!isNullable) {
1284+
checkNullable(batch, batch.length);
1285+
}
12421286
reserve(appendIndex + batch.length);
12431287
Object[][] columnData = new Object[childColumns.length][];
12441288
for (int j = 0; j < childColumns.length; ++j) {

‎regression-test/java-udf-src/src/main/java/org/apache/doris/udf/StringTest.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222

2323
public class StringTest extends UDF {
2424
public String evaluate(String field, Integer a, Integer b) {
25+
if (field == null || a == null || b == null) {
26+
return null;
27+
}
2528
return field.substring(0, a) + StringUtils.repeat("*", field.length() - a -b) + field.substring(field.length()-b);
2629
}
2730
}

‎regression-test/suites/javaudf_p0/test_javaudf_array.groovy‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,19 @@ suite("test_javaudf_array") {
117117
); """
118118
qt_select_13 """ SELECT java_udf_array_date_test(array(datev2_col)), tinyint_col as result FROM ${tableName} ORDER BY result; """
119119

120+
sql """ CREATE FUNCTION java_udf_array_list_test_not_nullable(array<string>) RETURNS array<string> PROPERTIES (
121+
"file"="file://${jarPath}",
122+
"symbol"="org.apache.doris.udf.ArrayReturnArrayStringTest",
123+
"always_nullable"="false",
124+
"type"="JAVA_UDF"
125+
); """
126+
test {
127+
sql """ SELECT java_udf_array_list_test_not_nullable(NULL); """
128+
exception "but the return type is not nullable"
129+
}
120130
} finally {
121131
try_sql("DROP FUNCTION IF EXISTS java_udf_array_int_test(array<int>);")
132+
try_sql("DROP FUNCTION IF EXISTS java_udf_array_list_test_not_nullable(array<string>);")
122133
try_sql("DROP FUNCTION IF EXISTS java_udf_array_return_int_test(array<int>);")
123134
try_sql("DROP FUNCTION IF EXISTS java_udf_array_return_string_test(array<string>);")
124135
try_sql("DROP FUNCTION IF EXISTS java_udf_array_string_test(array<string>);")

‎regression-test/suites/javaudf_p0/test_javaudf_int.groovy‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,13 +123,39 @@ suite("test_javaudf_int") {
123123
qt_select_global_3 """ SELECT java_udf_int_test_global(3) result FROM ${tableName} ORDER BY result; """
124124
qt_select_global_4 """ SELECT abs(java_udf_int_test_global(3)) result FROM ${tableName} ORDER BY result; """
125125

126+
sql """ CREATE FUNCTION java_udf_int_test_not_nullable(int) RETURNS int PROPERTIES (
127+
"file"="file://${jarPath}",
128+
"symbol"="org.apache.doris.udf.IntTest",
129+
"always_nullable"="false",
130+
"type"="JAVA_UDF"
131+
); """
132+
133+
test {
134+
sql """ SELECT java_udf_int_test_not_nullable(NULL); """
135+
exception "but the return type is not nullable"
136+
}
137+
138+
sql """ CREATE FUNCTION java_udf_largeint_test_not_nullable(largeint) RETURNS largeint PROPERTIES (
139+
"file"="file://${jarPath}",
140+
"symbol"="org.apache.doris.udf.LargeintTest",
141+
"always_nullable"="false",
142+
"type"="JAVA_UDF"
143+
); """
144+
145+
test {
146+
sql """ SELECT java_udf_largeint_test_not_nullable(NULL); """
147+
exception "but the return type is not nullable"
148+
}
149+
126150
} finally {
127151
try_sql("DROP GLOBAL FUNCTION IF EXISTS java_udf_int_test_global(int);")
128152
try_sql("DROP FUNCTION IF EXISTS java_udf_tinyint_test(tinyint);")
129153
try_sql("DROP FUNCTION IF EXISTS java_udf_smallint_test(smallint);")
130154
try_sql("DROP FUNCTION IF EXISTS java_udf_bigint_test(bigint);")
131155
try_sql("DROP FUNCTION IF EXISTS java_udf_largeint_test(largeint);")
132156
try_sql("DROP FUNCTION IF EXISTS java_udf_int_test(int);")
157+
try_sql("DROP FUNCTION IF EXISTS java_udf_int_test_not_nullable(int);")
158+
try_sql("DROP FUNCTION IF EXISTS java_udf_largeint_test_not_nullable(largeint);")
133159
try_sql("DROP TABLE IF EXISTS ${tableName}")
134160
}
135161
}

‎regression-test/suites/javaudf_p0/test_javaudf_string.groovy‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,8 +98,21 @@ suite("test_javaudf_string") {
9898
}
9999
sql """ insert into tbl1 select random()%10000 * 10000, "5" from tbl1;"""
100100
qt_select_5 """ select count(0) from (select k1, max(k2) as k2 from tbl1 group by k1)v where java_udf_string_test(k2, 0, 1) = "asd" """;
101+
102+
sql """ CREATE FUNCTION java_udf_string_test_not_nullabel(string, int, int) RETURNS string PROPERTIES (
103+
"file"="file://${jarPath}",
104+
"symbol"="org.apache.doris.udf.StringTest",
105+
"always_nullable"="false",
106+
"type"="JAVA_UDF"
107+
); """
108+
109+
test {
110+
sql """ SELECT java_udf_string_test_not_nullabel(NULL,NULL,NULL); """
111+
exception "but the return type is not nullable"
112+
}
101113
} finally {
102114
try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int, int);")
115+
try_sql("DROP FUNCTION IF EXISTS java_udf_string_test_not_nullabel(string, int, int);")
103116
try_sql("DROP TABLE IF EXISTS ${tableName}")
104117
try_sql("DROP TABLE IF EXISTS tbl1")
105118
try_sql("DROP TABLE IF EXISTS test_javaudf_string_2")

0 commit comments

Comments
 (0)