Skip to content

branch-3.0: [fix](hive)fix querying hive text table with NULL DEFINED AS '' (#55626) #55702

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 1 commit into from
Sep 13, 2025
Merged

branch-3.0: [fix](hive)fix querying hive text table with NULL DEFINED AS '' (#55626) #55702

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions .../thirdparties/docker-compose/hive/scripts/data/regression/serde_prop/some_serde_table.hql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,3 +170,33 @@ INSERT INTO TABLE test_open_csv_standard_prop VALUES
INSERT INTO TABLE test_open_csv_custom_prop VALUES
(1, 'John Doe', 28, 50000.75, true, '2022-01-15', '2023-10-21 14:30:00', 4.5, 'Senior Developer'),
(2, 'Jane,Smith', NULL, NULL, false, '2020-05-20', NULL, NULL, '\"Project Manager\"');

CREATE TABLE test_empty_null_format_text (
id INT,
name STRING
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
STORED AS TEXTFILE
TBLPROPERTIES (
"serialization.null.format"=""
);

INSERT INTO TABLE test_empty_null_format_text VALUES
(1, 'Alice'),
(2, NULL),
(3, '');

CREATE TABLE test_empty_null_defined_text (
id INT,
name STRING
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY '\t'
NULL DEFINED AS ''
STORED AS TEXTFILE;

INSERT INTO TABLE test_empty_null_defined_text VALUES
(1, 'Alice'),
(2, NULL),
(3, '');
8 changes: 5 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -851,7 +851,7 @@ public static Optional<String> getSerdeProperty(Table table, String key) {

private static Optional<String> firstNonNullable(String... values) {
for (String value : values) {
if (!Strings.isNullOrEmpty(value)) {
if (value != null) {
return Optional.of(value);
}
}
Expand All @@ -872,15 +872,17 @@ public static String firstPresentOrDefault(String defaultValue, Optional<String>
*
* @param altValue
* The string containing a number.
* @param defValue
* The default value to return if altValue is invalid.
*/
public static String getByte(String altValue) {
public static String getByte(String altValue, String defValue) {
if (altValue != null && altValue.length() > 0) {
try {
return Character.toString((char) ((Byte.parseByte(altValue) + 256) % 256));
} catch (NumberFormatException e) {
return altValue.substring(0, 1);
}
}
return null;
return defValue;
}
}
15 changes: 5 additions & 10 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ public static String getFieldDelimiter(Table table) {
Optional<String> fieldDelim = HiveMetaStoreClientHelper.getSerdeProperty(table, PROP_FIELD_DELIMITER);
Optional<String> serFormat = HiveMetaStoreClientHelper.getSerdeProperty(table, PROP_SERIALIZATION_FORMAT);
return HiveMetaStoreClientHelper.getByte(HiveMetaStoreClientHelper.firstPresentOrDefault(
DEFAULT_FIELD_DELIMITER, fieldDelim, serFormat));
"", fieldDelim, serFormat), DEFAULT_FIELD_DELIMITER);
}

public static String getSeparatorChar(Table table) {
Expand All @@ -97,13 +97,13 @@ public static String getSeparatorChar(Table table) {
public static String getLineDelimiter(Table table) {
Optional<String> lineDelim = HiveMetaStoreClientHelper.getSerdeProperty(table, PROP_LINE_DELIMITER);
return HiveMetaStoreClientHelper.getByte(HiveMetaStoreClientHelper.firstPresentOrDefault(
DEFAULT_LINE_DELIMITER, lineDelim));
"", lineDelim), DEFAULT_LINE_DELIMITER);
}

public static String getMapKvDelimiter(Table table) {
Optional<String> mapkvDelim = HiveMetaStoreClientHelper.getSerdeProperty(table, PROP_MAP_KV_DELIMITER);
return HiveMetaStoreClientHelper.getByte(HiveMetaStoreClientHelper.firstPresentOrDefault(
DEFAULT_MAP_KV_DELIMITER, mapkvDelim));
"", mapkvDelim), DEFAULT_MAP_KV_DELIMITER);
}

public static String getCollectionDelimiter(Table table) {
Expand All @@ -112,18 +112,13 @@ public static String getCollectionDelimiter(Table table) {
Optional<String> collectionDelimHive3 = HiveMetaStoreClientHelper.getSerdeProperty(table,
PROP_COLLECTION_DELIMITER_HIVE3);
return HiveMetaStoreClientHelper.getByte(HiveMetaStoreClientHelper.firstPresentOrDefault(
DEFAULT_COLLECTION_DELIMITER, collectionDelimHive2, collectionDelimHive3));
"", collectionDelimHive2, collectionDelimHive3), DEFAULT_COLLECTION_DELIMITER);
}

public static Optional<String> getEscapeDelimiter(Table table) {
Optional<String> escapeDelim = HiveMetaStoreClientHelper.getSerdeProperty(table, PROP_ESCAPE_DELIMITER);
if (escapeDelim.isPresent()) {
String escape = HiveMetaStoreClientHelper.getByte(escapeDelim.get());
if (escape != null) {
return Optional.of(escape);
} else {
return Optional.of(DEFAULT_ESCAPE_DELIMIER);
}
return Optional.of(HiveMetaStoreClientHelper.getByte(escapeDelim.get(), DEFAULT_ESCAPE_DELIMIER));
}
return Optional.empty();
}
Expand Down
44 changes: 44 additions & 0 deletions regression-test/data/external_table_p0/hive/test_hive_serde_prop.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,28 @@ b 2.2
1 John Doe 28 50000.75 TRUE 2022-01-15 2023-10-21 14:30:00 4.5 Senior Developer
2 Jane,Smith 2020-05-20 "Project Manager"

-- !test_empty_null_format_text --
1 Alice
2 \N
3 \N

-- !test_empty_null_format_text2 --
2 \N
3 \N

-- !test_empty_null_format_text3 --

-- !test_empty_null_defined_text --
1 Alice
2 \N
3 \N

-- !test_empty_null_defined_text2 --
2 \N
3 \N

-- !test_empty_null_defined_text3 --

-- !1 --
a 1.1
b 2.2
Expand Down Expand Up @@ -103,3 +125,25 @@ b 2.2
1 John Doe 28 50000.75 TRUE 2022-01-15 2023-10-21 14:30:00 4.5 Senior Developer
2 Jane,Smith FALSE 2020-05-20 "Project Manager"

-- !test_empty_null_format_text --
1 Alice
2 \N
3 \N

-- !test_empty_null_format_text2 --
2 \N
3 \N

-- !test_empty_null_format_text3 --

-- !test_empty_null_defined_text --
1 Alice
2 \N
3 \N

-- !test_empty_null_defined_text2 --
2 \N
3 \N

-- !test_empty_null_defined_text3 --

8 changes: 8 additions & 0 deletions regression-test/suites/external_table_p0/hive/test_hive_serde_prop.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,14 @@ suite("test_hive_serde_prop", "external_docker,hive,external_docker_hive,p0,exte
qt_test_open_csv_default_prop """select * from ${catalog_name}.regression.test_open_csv_default_prop order by id;"""
qt_test_open_csv_standard_prop """select * from ${catalog_name}.regression.test_open_csv_standard_prop order by id;"""
qt_test_open_csv_custom_prop """select * from ${catalog_name}.regression.test_open_csv_custom_prop order by id;"""

qt_test_empty_null_format_text """select * from ${catalog_name}.regression.test_empty_null_format_text order by id;"""
qt_test_empty_null_format_text2 """select * from ${catalog_name}.regression.test_empty_null_format_text where name is null order by id;"""
qt_test_empty_null_format_text3 """select * from ${catalog_name}.regression.test_empty_null_format_text where name = '' order by id;"""

qt_test_empty_null_defined_text """select * from ${catalog_name}.regression.test_empty_null_defined_text order by id;"""
qt_test_empty_null_defined_text2 """select * from ${catalog_name}.regression.test_empty_null_defined_text where name is null order by id;"""
qt_test_empty_null_defined_text3 """select * from ${catalog_name}.regression.test_empty_null_defined_text where name = '' order by id;"""
}
}