diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run03.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run03.sql new file mode 100644 index 00000000000000..bd8d673636c270 --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run03.sql @@ -0,0 +1,15 @@ +use paimon; + +create database if not exists test_paimon_spark; +use test_paimon_spark; + +drop table if exists test_varchar_char_type; + +create table test_varchar_char_type ( + c1 int, + c2 char(1), + c3 char(2147483647), + c4 varchar(1), + c6 varchar(2147483646), + c5 varchar(2147483647) +); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 4119f978d24709..0d65d53ba40048 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -42,10 +42,12 @@ import org.apache.paimon.table.Table; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Projection; @@ -165,8 +167,10 @@ private static Type paimonPrimitiveTypeToDorisType(org.apache.paimon.types.DataT case TINYINT: return Type.TINYINT; case VARCHAR: - case BINARY: + return ScalarType.createVarcharType(((VarCharType) dataType).getLength()); case CHAR: + return ScalarType.createCharType(((CharType) dataType).getLength()); + case BINARY: case VARBINARY: return Type.STRING; case DECIMAL: diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java new file mode 100644 index 00000000000000..449cf4c70dce80 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.catalog.Type; + +import org.apache.paimon.types.CharType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.VarCharType; +import org.junit.Assert; +import org.junit.Test; + +public class PaimonUtilTest { + @Test + public void testSchemaForVarcharAndChar() { + DataField c1 = new DataField(1, "c1", new VarCharType(32)); + DataField c2 = new DataField(2, "c2", new CharType(14)); + Type type1 = PaimonUtil.paimonTypeToDorisType(c1.type()); + Type type2 = PaimonUtil.paimonTypeToDorisType(c2.type()); + Assert.assertTrue(type1.isVarchar()); + Assert.assertEquals(32, type1.getLength()); + Assert.assertEquals(14, type2.getLength()); + } +} diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_char_varchar_type.out b/regression-test/data/external_table_p0/paimon/test_paimon_char_varchar_type.out new file mode 100644 index 00000000000000..80fca7ed4024b9 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_char_varchar_type.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q0 -- +c1 int Yes true \N +c2 char(1) Yes true \N +c3 char(2147483647) Yes true \N +c4 varchar(1) Yes true \N +c6 varchar(2147483646) Yes true \N +c5 varchar(2147483647) Yes true \N + diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_schema_change.out b/regression-test/data/external_table_p0/paimon/test_paimon_schema_change.out index 5d33ed9f7e6066..fed740a880378d 100644 --- a/regression-test/data/external_table_p0/paimon/test_paimon_schema_change.out +++ b/regression-test/data/external_table_p0/paimon/test_paimon_schema_change.out @@ -1,8 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !desc_1 -- -full_name text Yes true \N +full_name varchar(2147483647) Yes true \N id bigint Yes true \N -location text Yes true \N +location varchar(2147483647) Yes true \N salary decimal(12,2) Yes true \N -- !parquet_pk_1 -- @@ -74,9 +74,9 @@ Ken 11 Austin 9000.00 Laura 12 Portland 10000.00 -- !desc_2 -- -full_name text Yes true \N +full_name varchar(2147483647) Yes true \N id bigint Yes true \N -location text Yes true \N +location varchar(2147483647) Yes true \N salary decimal(12,2) Yes true \N -- !orc_pk_1 -- @@ -184,9 +184,9 @@ Laura 20 Portland 10000.00 -- !desc_3 -- k int Yes true \N -vv text Yes true \N -new_col3 map Yes true \N -new_col2 struct Yes true \N +vv varchar(2147483647) Yes true \N +new_col3 map Yes true \N +new_col2 struct Yes true \N new_col1 array Yes true \N -- !parquet_1 -- @@ -202,9 +202,9 @@ new_col1 array Yes true \N -- !desc_4 -- k int Yes true \N -vv text Yes true \N -new_col3 map Yes true \N -new_col2 struct Yes true \N +vv varchar(2147483647) Yes true \N +new_col3 map Yes true \N +new_col2 struct Yes true \N new_col1 array Yes true \N -- !orc_1 -- diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_char_varchar_type.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_char_varchar_type.groovy new file mode 100644 index 00000000000000..5ebef0b85ec770 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_char_varchar_type.groovy @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_char_varchar_type", "p0,external,doris,external_docker,external_docker_doris") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + String catalog_name = "test_paimon_char_varchar_type" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalog_name}""" + + sql """ + CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type' = 'paimon', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minio_port}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true' + ); + """ + sql """switch `${catalog_name}`""" + sql """show databases; """ + sql """use `${catalog_name}`.`test_paimon_spark`""" + qt_q0 """ desc test_varchar_char_type """ + } +} + +