diff --git a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp index 98ff1eb7f81b9b..014e8b0d5ebd24 100644 --- a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp @@ -135,12 +135,14 @@ Status DataTypeNullableSerDe::deserialize_column_from_fixed_json( if (!st.ok()) { return st; } - auto& null_map = col.get_null_map_data(); - auto& nested_column = col.get_nested_column(); - - null_map.resize_fill( - rows, null_map.back()); // data_type_nullable::insert_column_last_value_multiple_times() if (rows - 1 != 0) { + auto& null_map = col.get_null_map_data(); + auto& nested_column = col.get_nested_column(); + + uint8_t val = null_map.back(); + size_t new_sz = null_map.size() + rows - 1; + null_map.resize_fill(new_sz, + val); // data_type_nullable::insert_column_last_value_multiple_times() nested_serde->insert_column_last_value_multiple_times(nested_column, rows - 1); } *num_deserialized = rows; diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run65.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run65.hql new file mode 100644 index 00000000000000..2c17d743d5cdd8 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run65.hql @@ -0,0 +1,28 @@ +use default; + + +CREATE TABLE orc_partition_multi_stripe ( + col1 STRING, + col2 INT, + col3 DOUBLE +) PARTITIONED BY ( + partition_col1 STRING, + partition_col2 INT +) +STORED AS ORC +LOCATION '/user/doris/preinstalled_data/orc_table/orc_partition_multi_stripe'; +; +msck repair table orc_partition_multi_stripe; + +CREATE TABLE parquet_partition_multi_row_group ( + col1 STRING, + col2 INT, + col3 DOUBLE +) PARTITIONED BY ( + partition_col1 STRING, + partition_col2 INT +) +STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_partition_multi_row_group'; +; +msck repair table parquet_partition_multi_row_group; diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_10 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_10 new file mode 100644 index 00000000000000..46ebfc96e7eef0 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_10 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_11 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_11 new file mode 100644 index 00000000000000..77c18939ffc18a Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_11 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_12 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_12 new file mode 100644 index 00000000000000..e1327b620f2d0e Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_12 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_13 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_13 new file mode 100644 index 00000000000000..0302b81ef349ea Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_13 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_10 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_10 new file mode 100644 index 00000000000000..fbe3d0ce52a024 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_10 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_11 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_11 new file mode 100644 index 00000000000000..cb71b6314720f5 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_11 differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_9 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_9 new file mode 100644 index 00000000000000..f3c7d4fe72d98e Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_9 differ diff --git a/regression-test/data/external_table_p0/hive/test_hive_opt_fill_partition.out b/regression-test/data/external_table_p0/hive/test_hive_opt_fill_partition.out new file mode 100644 index 00000000000000..7979586e459b99 --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_opt_fill_partition.out @@ -0,0 +1,101 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !parquet_1 -- +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 + +-- !parquet_2 -- +1792 + +-- !parquet_3 -- +1792 + +-- !parquet_4 -- +1792 + +-- !parquet_5 -- +1792 + +-- !parquet_6 -- +1792 + +-- !parquet_7 -- +word 1792 + +-- !parquet_8 -- +hello 1792 + +-- !parquet_9 -- +1 1792 + +-- !parquet_10 -- +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 + +-- !parquet_11 -- +1792 + +-- !parquet_12 -- +1792 + +-- !parquet_13 -- +1792 + +-- !parquet_14 -- +0 + +-- !orc_1 -- +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 + +-- !orc_2 -- +7680 + +-- !orc_3 -- +7680 + +-- !orc_4 -- +7680 + +-- !orc_5 -- +7680 + +-- !orc_6 -- +7680 + +-- !orc_7 -- +word 7680 + +-- !orc_8 -- +hello 7680 + +-- !orc_9 -- +1 7680 + +-- !orc_10 -- +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 +word 2 2.3 hello 1 + +-- !orc_11 -- +7680 + +-- !orc_12 -- +7680 + +-- !orc_13 -- +7680 + +-- !orc_14 -- +0 + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_opt_fill_partition.groovy b/regression-test/suites/external_table_p0/hive/test_hive_opt_fill_partition.groovy new file mode 100644 index 00000000000000..0bb5249e262df0 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_opt_fill_partition.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_hive_opt_fill_partition", "p0,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String hivePrefix ="hive3"; + setHivePrefix(hivePrefix) + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + + String catalog_name = "test_hive_opt_fill_partition" + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}' + ); + """ + + sql """ switch ${catalog_name} """ + sql """ use `default` """ + + qt_parquet_1 """ select * from parquet_partition_multi_row_group limit 5; """ + qt_parquet_2 """ select count(col1) from parquet_partition_multi_row_group ; """ + qt_parquet_3 """ select count(col2) from parquet_partition_multi_row_group ; """ + qt_parquet_4 """ select count(col3) from parquet_partition_multi_row_group ; """ + qt_parquet_5 """ select count(partition_col1) from parquet_partition_multi_row_group ; """ + qt_parquet_6 """ select count(partition_col1) from parquet_partition_multi_row_group ; """ + qt_parquet_7 """ select col1,count(*) from parquet_partition_multi_row_group group by col1; """ + qt_parquet_8 """ select partition_col1,count(*) from parquet_partition_multi_row_group group by partition_col1; """ + qt_parquet_9 """ select partition_col2,count(*) from parquet_partition_multi_row_group group by partition_col2; """ + qt_parquet_10 """ select * from parquet_partition_multi_row_group where col1 = 'word' limit 5; """ + qt_parquet_11 """ select count(*) from parquet_partition_multi_row_group where col2 != 100; """ + qt_parquet_12 """ select count(*) from parquet_partition_multi_row_group where partition_col1 = 'hello' limit 5; """ + qt_parquet_13 """ select count(*) from parquet_partition_multi_row_group where partition_col2 = 1 limit 5; """ + qt_parquet_14 """ select count(*) from parquet_partition_multi_row_group where partition_col2 != 1 ; """ + + + qt_orc_1 """ select * from orc_partition_multi_stripe limit 5; """ + qt_orc_2 """ select count(col1) from orc_partition_multi_stripe ; """ + qt_orc_3 """ select count(col2) from orc_partition_multi_stripe ; """ + qt_orc_4 """ select count(col3) from orc_partition_multi_stripe ; """ + qt_orc_5 """ select count(partition_col1) from orc_partition_multi_stripe ; """ + qt_orc_6 """ select count(partition_col1) from orc_partition_multi_stripe ; """ + qt_orc_7 """ select col1,count(*) from orc_partition_multi_stripe group by col1; """ + qt_orc_8 """ select partition_col1,count(*) from orc_partition_multi_stripe group by partition_col1; """ + qt_orc_9 """ select partition_col2,count(*) from orc_partition_multi_stripe group by partition_col2; """ + qt_orc_10 """ select * from orc_partition_multi_stripe where col1 = 'word' limit 5; """ + qt_orc_11 """ select count(*) from orc_partition_multi_stripe where col2 != 100; """ + qt_orc_12 """ select count(*) from orc_partition_multi_stripe where partition_col1 = 'hello' limit 5; """ + qt_orc_13 """ select count(*) from orc_partition_multi_stripe where partition_col2 = 1 limit 5; """ + qt_orc_14 """ select count(*) from orc_partition_multi_stripe where partition_col2 != 1 ; """ + + } +} \ No newline at end of file