diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index e40759456907de..0ec5153a415169 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -75,6 +75,13 @@ public class HMSExternalCatalog extends ExternalCatalog { public static final String FILE_META_CACHE_TTL_SECOND = "file.meta.cache.ttl-second"; // broker name for file split and query scan. public static final String BIND_BROKER_NAME = "broker.name"; + // Default is false, if set to true, will get table schema from "remoteTable" instead of from hive metastore. + // This is because for some forward compatiblity issue of hive metastore, there maybe + // "storage schema reading not support" error being thrown. + // set this to true can avoid this error. + // But notice that if set to true, the default value of column will be ignored because we cannot get default value + // from remoteTable object. + public static final String GET_SCHEMA_FROM_TABLE = "get_schema_from_table"; // -1 means file cache no ttl set public static final int FILE_META_CACHE_NO_TTL = -1; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index dfd7d6fe4b8c0b..9a1ec5f8067562 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -78,6 +78,7 @@ import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; +import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.internal.schema.InternalSchema; @@ -629,9 +630,18 @@ private Optional getHudiSchema() { } private Optional getHiveSchema() { - HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient(); - List schema = client.getSchema(dbName, name); - Map colDefaultValues = client.getDefaultColumnValues(dbName, name); + boolean getFromTable = catalog.getCatalogProperty() + .getOrDefault(HMSExternalCatalog.GET_SCHEMA_FROM_TABLE, "false") + .equalsIgnoreCase("true"); + List schema = null; + Map colDefaultValues = Maps.newHashMap(); + if (getFromTable) { + schema = getSchemaFromRemoteTable(remoteTable); + } else { + HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient(); + schema = client.getSchema(dbName, name); + colDefaultValues = client.getDefaultColumnValues(dbName, name); + } List columns = Lists.newArrayListWithCapacity(schema.size()); for (FieldSchema field : schema) { String fieldName = field.getName().toLowerCase(Locale.ROOT); @@ -644,6 +654,13 @@ private Optional getHiveSchema() { return Optional.of(new HMSSchemaCacheValue(columns, partitionColumns)); } + private static List getSchemaFromRemoteTable(Table table) { + List schema = Lists.newArrayList(); + schema.addAll(table.getSd().getCols()); + schema.addAll(table.getPartitionKeys()); + return schema; + } + @Override public long fetchRowCount() { makeSureInitialized(); diff --git a/regression-test/data/external_table_p0/hive/test_hive_get_schema_from_table.out b/regression-test/data/external_table_p0/hive/test_hive_get_schema_from_table.out new file mode 100644 index 00000000000000..2e190d329f10b0 --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_get_schema_from_table.out @@ -0,0 +1,55 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !schema_1 -- +1 7706 1 155190 17.00 21168.23 0.04 0.02 N O 1996-03-13 1996-02-12 1996-03-22 DELIVER IN PERSON TRUCK egular courts above the cn beijing + +-- !schema_2 -- +6374628540732951412 -77 -65 -70 -107 -215 65 0 -526 -1309 3750 8827 -19795 34647 57042 -1662 -138248 -890685 -228568 1633079 -2725524 6163040 -10491702 697237 74565050 127767368 93532213 -209675435 -32116110 -3624917040 -2927805617 15581947241 21893441661 24075494509 -116822110531 -59683724667 -146210393388 114424524398 1341560771667 -1638742564263 520137948334 -2927347587131 7415137351179 -7963937754617 52157548982266 140803519083304 -294675355729619 -868076759504942 181128508165910 -91753231238823 -3511241416682881 -11545256318348796 -1952917510863468 -5161099825338866 -59726090170689781 287170105829528178 607326725526282735 1253194074103207461 -162443950414676064 -2964036188567341159 2602201580810990248 5581917084094110764 111739292249520611 -315687754593838642 -2804420462762366976 -2078683524 + +-- !schema_3 -- +false 5 5 5 50 5.5 50.5 7295 12/31/10 5 2010-12-31T12:05:13.600 2010 12 +false 7 7 7 70 7.7 70.7 7297 12/31/10 7 2010-12-31T12:07:13.710 2010 12 +false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 +true 6 6 6 60 6.6 60.599999999999994 7296 12/31/10 6 2010-12-31T12:06:13.650 2010 12 +true 8 8 8 80 8.8 80.8 7298 12/31/10 8 2010-12-31T12:08:13.780 2010 12 + +-- !schema_4 -- +2 24 15314771 999319712124142303 true 6.009337E8 4.8177228079770208E16 \N northern rural 2022-08-30T23:21:08 407186.2849 phones int_col 2019-01-01 [2.5954339078494106e+17, 5.88165568758352e+17, 4.780259987226574e+17, 6.926622881251557e+17, 9.86405645575228e+17] \N phones int_col +5 59 317349992 998913039814974432 false 5.6584864E8 9.900861328269033E17 Handling man satisfy firework descent top. Racing closed county set-up crown cave. Correctly front duration pure. \N 2022-09-02T19:52:57 372765.2493 desktops tinyint_col 2021-10-03 [9.983261252571983e+17, 3.612076153030643e+17, 9.969131496509435e+17, 8.991290717923475e+17, 1.1955893747098878e+17] ["CrySxz", "FMXGRcaGbahSVqhp", "oRKqPmhM", "VdODasEdDWFSRIQf"] desktops tinyint_col +6 62 915699741 999653836472045196 true 4.51937536E8 8.7961505445021914E17 Tale get speed platform august curved. Ease grass neighbour landlord. Baby genetic youth. \N 2022-08-07T09:30:56 875620.2176 phones smallint_col \N [9.423540715161855e+17, 4.8332499920295616e+17, 9.167007747789834e+17] ["zNfbLeFx", "GNTJOmWJyRmOK", "hwvfhSQGsaaMEqUrWCK", "cQrQsROKLARA", "nONj", "oepXBFB", "IPtUql"] phones smallint_col + +-- !schema_5 -- +00cwjIryUv EXHwpeK2Nl hv2PYEMYMM eo69nyw4Yv K6797tgjFg LlFNd8Kyy5 wkpLCO3uo1 AIXCj1MfeD ni0HxZbiUO 6IjRdM8Gqi qsTMK6A2eC 1wu7v9OPwW qavArd9tDc sU88hZADLj lyzWlwLOCx 2022-11-25 + +-- !schema_6 -- +"" "test" + +-- !schema_7 -- +\N \N \N \N \N \N \N \N \N test test test 1 2 3 4 5.1 6.2 true false 2011-05-06 2011-05-06T07:08:09.123 -1.2 12.30 -1234.5678 123456789.12340000 -1234567890.12345678 1234567890123456789012.1234567800000000 dGVzdDI= + +-- !schema_1 -- +1 7706 1 155190 17.00 21168.23 0.04 0.02 N O 1996-03-13 1996-02-12 1996-03-22 DELIVER IN PERSON TRUCK egular courts above the cn beijing + +-- !schema_2 -- +6374628540732951412 -77 -65 -70 -107 -215 65 0 -526 -1309 3750 8827 -19795 34647 57042 -1662 -138248 -890685 -228568 1633079 -2725524 6163040 -10491702 697237 74565050 127767368 93532213 -209675435 -32116110 -3624917040 -2927805617 15581947241 21893441661 24075494509 -116822110531 -59683724667 -146210393388 114424524398 1341560771667 -1638742564263 520137948334 -2927347587131 7415137351179 -7963937754617 52157548982266 140803519083304 -294675355729619 -868076759504942 181128508165910 -91753231238823 -3511241416682881 -11545256318348796 -1952917510863468 -5161099825338866 -59726090170689781 287170105829528178 607326725526282735 1253194074103207461 -162443950414676064 -2964036188567341159 2602201580810990248 5581917084094110764 111739292249520611 -315687754593838642 -2804420462762366976 -2078683524 + +-- !schema_3 -- +false 5 5 5 50 5.5 50.5 7295 12/31/10 5 2010-12-31T12:05:13.600 2010 12 +false 7 7 7 70 7.7 70.7 7297 12/31/10 7 2010-12-31T12:07:13.710 2010 12 +false 9 9 9 90 9.9 90.89999999999999 7299 12/31/10 9 2010-12-31T12:09:13.860 2010 12 +true 6 6 6 60 6.6 60.599999999999994 7296 12/31/10 6 2010-12-31T12:06:13.650 2010 12 +true 8 8 8 80 8.8 80.8 7298 12/31/10 8 2010-12-31T12:08:13.780 2010 12 + +-- !schema_4 -- +2 24 15314771 999319712124142303 true 6.009337E8 4.8177228079770208E16 \N northern rural 2022-08-30T23:21:08 407186.2849 phones int_col 2019-01-01 [2.5954339078494106e+17, 5.88165568758352e+17, 4.780259987226574e+17, 6.926622881251557e+17, 9.86405645575228e+17] \N phones int_col +5 59 317349992 998913039814974432 false 5.6584864E8 9.900861328269033E17 Handling man satisfy firework descent top. Racing closed county set-up crown cave. Correctly front duration pure. \N 2022-09-02T19:52:57 372765.2493 desktops tinyint_col 2021-10-03 [9.983261252571983e+17, 3.612076153030643e+17, 9.969131496509435e+17, 8.991290717923475e+17, 1.1955893747098878e+17] ["CrySxz", "FMXGRcaGbahSVqhp", "oRKqPmhM", "VdODasEdDWFSRIQf"] desktops tinyint_col +6 62 915699741 999653836472045196 true 4.51937536E8 8.7961505445021914E17 Tale get speed platform august curved. Ease grass neighbour landlord. Baby genetic youth. \N 2022-08-07T09:30:56 875620.2176 phones smallint_col \N [9.423540715161855e+17, 4.8332499920295616e+17, 9.167007747789834e+17] ["zNfbLeFx", "GNTJOmWJyRmOK", "hwvfhSQGsaaMEqUrWCK", "cQrQsROKLARA", "nONj", "oepXBFB", "IPtUql"] phones smallint_col + +-- !schema_5 -- +00cwjIryUv EXHwpeK2Nl hv2PYEMYMM eo69nyw4Yv K6797tgjFg LlFNd8Kyy5 wkpLCO3uo1 AIXCj1MfeD ni0HxZbiUO 6IjRdM8Gqi qsTMK6A2eC 1wu7v9OPwW qavArd9tDc sU88hZADLj lyzWlwLOCx 2022-11-25 + +-- !schema_6 -- +"" "test" + +-- !schema_7 -- +\N \N \N \N \N \N \N \N \N test test test 1 2 3 4 5.1 6.2 true false 2011-05-06 2011-05-06T07:08:09.123 -1.2 12.30 -1234.5678 123456789.12340000 -1234567890.12345678 1234567890123456789012.1234567800000000 dGVzdDI= + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_get_schema_from_table.groovy b/regression-test/suites/external_table_p0/hive/test_hive_get_schema_from_table.groovy new file mode 100644 index 00000000000000..c07a0a763b0379 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_get_schema_from_table.groovy @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_get_schema_from_table", "external_docker,hive,external_docker_hive,p0,external") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + + // test get scheam from table + for (String hivePrefix : ["hive2", "hive3"]) { + String catalog_name = "test_${hivePrefix}_get_schema" + String ex_db_name = "`default`" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + + sql """drop catalog if exists ${catalog_name} """ + + sql """CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', + 'hadoop.username' = 'hive', + 'get_schema_from_table' = 'true' + );""" + + sql """switch ${catalog_name}""" + + def res_dbs_log = sql "show databases;" + for (int i = 0; i < res_dbs_log.size(); i++) { + def tbs = sql "show tables from `${res_dbs_log[i][0]}`" + log.info("database = ${res_dbs_log[i][0]} => tables = " + tbs.toString()) + } + + order_qt_schema_1 """select * from ${catalog_name}.${ex_db_name}.parquet_partition_table order by l_orderkey limit 1;""" + order_qt_schema_2 """select * from ${catalog_name}.${ex_db_name}.parquet_delta_binary_packed order by int_value limit 1;""" + order_qt_schema_3 """select * from ${catalog_name}.${ex_db_name}.parquet_alltypes_tiny_pages order by id desc limit 5;""" + order_qt_schema_4 """select * from ${catalog_name}.${ex_db_name}.orc_all_types_partition order by bigint_col desc limit 3;""" + order_qt_schema_5 """select * from ${catalog_name}.${ex_db_name}.csv_partition_table order by k1 limit 1;""" + order_qt_schema_6 """select * from ${catalog_name}.${ex_db_name}.csv_all_types limit 1;""" + order_qt_schema_7 """select * except(t_varchar_max_length) from ${catalog_name}.${ex_db_name}.text_all_types limit 1;""" + + //sql """drop catalog if exists ${catalog_name} """ + + } +} +