Skip to content

[fix](paimon)Handle oversized CHAR/VARCHAR fields in Paimon to Doris type mapping #55051

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 8 commits into from
Aug 28, 2025
Merged

[fix](paimon)Handle oversized CHAR/VARCHAR fields in Paimon to Doris type mapping #55051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
070c241
modify paimon data type mapping
Aug 13, 2025
bf7d851
fix
Aug 18, 2025
fda7d36
add regression test
Aug 18, 2025
cdfa2b9
modify case
Aug 20, 2025
7aeb0da
add regression test
Aug 20, 2025
e850ef2
modify regression test result
Aug 20, 2025
1bd69c4
fix
Aug 20, 2025
bc6fad5
modify regression test result
Aug 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions .../thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run03.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@ drop table if exists test_varchar_char_type;
create table test_varchar_char_type (
c1 int,
c2 char(1),
c3 char(2147483647),
c4 varchar(1),
c6 varchar(2147483646),
c5 varchar(2147483647)
c3 char(254),
c4 char(2147483647),
c5 varchar(1),
c6 varchar(65533),
c7 varchar(2147483646),
c8 varchar(2147483647),
c9 string
);
12 changes: 10 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -225,9 +225,17 @@ private static Type paimonPrimitiveTypeToDorisType(org.apache.paimon.types.DataT
case TINYINT:
return Type.TINYINT;
case VARCHAR:
return ScalarType.createVarcharType(((VarCharType) dataType).getLength());
int varcharLen = ((VarCharType) dataType).getLength();
if (varcharLen > 65533) {
return ScalarType.createStringType();
}
return ScalarType.createVarcharType(varcharLen);
case CHAR:
return ScalarType.createCharType(((CharType) dataType).getLength());
int charLen = ((CharType) dataType).getLength();
if (charLen > 255) {
return ScalarType.createStringType();
}
return ScalarType.createCharType(charLen);
case BINARY:
case VARBINARY:
return Type.STRING;
Expand Down
30 changes: 15 additions & 15 deletions regression-test/data/external_table_p0/paimon/paimon_system_table.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@
-- !desc_direct_query_ctl_db_table --
snapshot_id bigint No true \N NONE
schema_id bigint No false \N NONE
commit_user varchar(2147483647) No false \N NONE
commit_user text No false \N NONE
commit_identifier bigint No false \N NONE
commit_kind varchar(2147483647) No false \N NONE
commit_kind text No false \N NONE
commit_time datetime(3) No false \N NONE
base_manifest_list varchar(2147483647) No false \N NONE
delta_manifest_list varchar(2147483647) No false \N NONE
changelog_manifest_list varchar(2147483647) Yes false \N NONE
base_manifest_list text No false \N NONE
delta_manifest_list text No false \N NONE
changelog_manifest_list text Yes false \N NONE
total_record_count bigint Yes false \N NONE
delta_record_count bigint Yes false \N NONE
changelog_record_count bigint Yes false \N NONE
Expand All @@ -56,13 +56,13 @@ watermark bigint Yes false \N NONE
-- !desc_direct_query_db_table --
snapshot_id bigint No true \N NONE
schema_id bigint No false \N NONE
commit_user varchar(2147483647) No false \N NONE
commit_user text No false \N NONE
commit_identifier bigint No false \N NONE
commit_kind varchar(2147483647) No false \N NONE
commit_kind text No false \N NONE
commit_time datetime(3) No false \N NONE
base_manifest_list varchar(2147483647) No false \N NONE
delta_manifest_list varchar(2147483647) No false \N NONE
changelog_manifest_list varchar(2147483647) Yes false \N NONE
base_manifest_list text No false \N NONE
delta_manifest_list text No false \N NONE
changelog_manifest_list text Yes false \N NONE
total_record_count bigint Yes false \N NONE
delta_record_count bigint Yes false \N NONE
changelog_record_count bigint Yes false \N NONE
Expand All @@ -71,13 +71,13 @@ watermark bigint Yes false \N NONE
-- !desc_direct_query_table --
snapshot_id bigint No true \N NONE
schema_id bigint No false \N NONE
commit_user varchar(2147483647) No false \N NONE
commit_user text No false \N NONE
commit_identifier bigint No false \N NONE
commit_kind varchar(2147483647) No false \N NONE
commit_kind text No false \N NONE
commit_time datetime(3) No false \N NONE
base_manifest_list varchar(2147483647) No false \N NONE
delta_manifest_list varchar(2147483647) No false \N NONE
changelog_manifest_list varchar(2147483647) Yes false \N NONE
base_manifest_list text No false \N NONE
delta_manifest_list text No false \N NONE
changelog_manifest_list text Yes false \N NONE
total_record_count bigint Yes false \N NONE
delta_record_count bigint Yes false \N NONE
changelog_record_count bigint Yes false \N NONE
Expand Down
11 changes: 7 additions & 4 deletions regression-test/data/external_table_p0/paimon/test_paimon_char_varchar_type.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@
-- !q0 --
c1 int Yes true \N
c2 char(1) Yes true \N
c3 char(2147483647) Yes true \N
c4 varchar(1) Yes true \N
c6 varchar(2147483646) Yes true \N
c5 varchar(2147483647) Yes true \N
c3 char(254) Yes true \N
c4 text Yes true \N
c5 varchar(1) Yes true \N
c6 varchar(65533) Yes true \N
c7 text Yes true \N
c8 text Yes true \N
c9 text Yes true \N

20 changes: 10 additions & 10 deletions regression-test/data/external_table_p0/paimon/test_paimon_schema_change.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !desc_1 --
full_name varchar(2147483647) Yes true \N
full_name text Yes true \N
id bigint Yes true \N
location varchar(2147483647) Yes true \N
location text Yes true \N
salary decimal(12,2) Yes true \N

-- !parquet_pk_1 --
Expand Down Expand Up @@ -74,9 +74,9 @@ Ken 11 Austin 9000.00
Laura 12 Portland 10000.00

-- !desc_2 --
full_name varchar(2147483647) Yes true \N
full_name text Yes true \N
id bigint Yes true \N
location varchar(2147483647) Yes true \N
location text Yes true \N
salary decimal(12,2) Yes true \N

-- !orc_pk_1 --
Expand Down Expand Up @@ -184,9 +184,9 @@ Laura 20 Portland 10000.00

-- !desc_3 --
k int Yes true \N
vv varchar(2147483647) Yes true \N
new_col3 map<varchar(2147483647),int> Yes true \N
new_col2 struct<a:int,b:varchar(2147483647)> Yes true \N
vv text Yes true \N
new_col3 map<text,int> Yes true \N
new_col2 struct<a:int,b:text> Yes true \N
new_col1 array<int> Yes true \N

-- !parquet_1 --
Expand All @@ -202,9 +202,9 @@ new_col1 array<int> Yes true \N

-- !desc_4 --
k int Yes true \N
vv varchar(2147483647) Yes true \N
new_col3 map<varchar(2147483647),int> Yes true \N
new_col2 struct<a:int,b:varchar(2147483647)> Yes true \N
vv text Yes true \N
new_col3 map<text,int> Yes true \N
new_col2 struct<a:int,b:text> Yes true \N
new_col1 array<int> Yes true \N

-- !orc_1 --
Expand Down