[opt](hive) add option to get schema from table object

fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java

@@ -75,6 +75,13 @@ public class HMSExternalCatalog extends ExternalCatalog {
     public static final String FILE_META_CACHE_TTL_SECOND = "file.meta.cache.ttl-second";
     // broker name for file split and query scan.
     public static final String BIND_BROKER_NAME = "broker.name";
+    // Default is false, if set to true, will get table schema from "remoteTable" instead of from hive metastore.
+    // This is because for some forward compatiblity issue of hive metastore, there maybe
+    // "storage schema reading not support" error being thrown.
+    // set this to true can avoid this error.
+    // But notice that if set to true, the default value of column will be ignored because we cannot get default value
+    // from remoteTable object.
+    public static final String GET_SCHEMA_FROM_TABLE = "get_schema_from_table";
 
     // -1 means file cache no ttl set
     public static final int FILE_META_CACHE_NO_TTL = -1;

fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java

@@ -78,6 +78,7 @@
 import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.internal.schema.InternalSchema;
@@ -629,9 +630,18 @@ private Optional<SchemaCacheValue> getHudiSchema() {
     }
 
     private Optional<SchemaCacheValue> getHiveSchema() {
-        HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient();
-        List<FieldSchema> schema = client.getSchema(dbName, name);
-        Map<String, String> colDefaultValues = client.getDefaultColumnValues(dbName, name);
+        boolean getFromTable = catalog.getCatalogProperty()
+                .getOrDefault(HMSExternalCatalog.GET_SCHEMA_FROM_TABLE, "false")
+                .equalsIgnoreCase("true");
+        List<FieldSchema> schema = null;
+        Map<String, String> colDefaultValues = Maps.newHashMap();
+        if (getFromTable) {
+            schema = getSchemaFromRemoteTable(remoteTable);
+        } else {
+            HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient();
+            schema = client.getSchema(dbName, name);
+            colDefaultValues = client.getDefaultColumnValues(dbName, name);
+        }
         List<Column> columns = Lists.newArrayListWithCapacity(schema.size());
         for (FieldSchema field : schema) {
             String fieldName = field.getName().toLowerCase(Locale.ROOT);
@@ -644,6 +654,13 @@ private Optional<SchemaCacheValue> getHiveSchema() {
         return Optional.of(new HMSSchemaCacheValue(columns, partitionColumns));
     }
 
+    private static List<FieldSchema> getSchemaFromRemoteTable(Table table) {
+        List<FieldSchema> schema = Lists.newArrayList();
+        schema.addAll(table.getSd().getCols());
+        schema.addAll(table.getPartitionKeys());
+        return schema;
+    }
+
     @Override
     public long fetchRowCount() {
         makeSureInitialized();

regression-test/data/external_table_p0/hive/test_hive_get_schema_from_table.out

@@ -0,0 +1,55 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !schema_1 --
+1	7706	1	155190	17.00	21168.23	0.04	0.02	N	O	1996-03-13	1996-02-12	1996-03-22	DELIVER IN PERSON	TRUCK	egular courts above the	cn	beijing
+
+-- !schema_2 --
+6374628540732951412	-77	-65	-70	-107	-215	65	0	-526	-1309	3750	8827	-19795	34647	57042	-1662	-138248	-890685	-228568	1633079	-2725524	6163040	-10491702	697237	74565050	127767368	93532213	-209675435	-32116110	-3624917040	-2927805617	15581947241	21893441661	24075494509	-116822110531	-59683724667	-146210393388	114424524398	1341560771667	-1638742564263	520137948334	-2927347587131	7415137351179	-7963937754617	52157548982266	140803519083304	-294675355729619	-868076759504942	181128508165910	-91753231238823	-3511241416682881	-11545256318348796	-1952917510863468	-5161099825338866	-59726090170689781	287170105829528178	607326725526282735	1253194074103207461	-162443950414676064	-2964036188567341159	2602201580810990248	5581917084094110764	111739292249520611	-315687754593838642	-2804420462762366976	-2078683524
+
+-- !schema_3 --
+false	5	5	5	50	5.5	50.5	7295	12/31/10	5	2010-12-31T12:05:13.600	2010	12
+false	7	7	7	70	7.7	70.7	7297	12/31/10	7	2010-12-31T12:07:13.710	2010	12
+false	9	9	9	90	9.9	90.89999999999999	7299	12/31/10	9	2010-12-31T12:09:13.860	2010	12
+true	6	6	6	60	6.6	60.599999999999994	7296	12/31/10	6	2010-12-31T12:06:13.650	2010	12
+true	8	8	8	80	8.8	80.8	7298	12/31/10	8	2010-12-31T12:08:13.780	2010	12
+
+-- !schema_4 --
+2	24	15314771	999319712124142303	true	6.009337E8	4.8177228079770208E16	\N	northern rural	2022-08-30T23:21:08	407186.2849	phones	int_col	2019-01-01	[2.5954339078494106e+17, 5.88165568758352e+17, 4.780259987226574e+17, 6.926622881251557e+17, 9.86405645575228e+17]	\N	phones	int_col
+5	59	317349992	998913039814974432	false	5.6584864E8	9.900861328269033E17	Handling man satisfy firework descent top. Racing closed county set-up crown cave. Correctly front duration pure.	\N	2022-09-02T19:52:57	372765.2493	desktops	tinyint_col	2021-10-03	[9.983261252571983e+17, 3.612076153030643e+17, 9.969131496509435e+17, 8.991290717923475e+17, 1.1955893747098878e+17]	["CrySxz", "FMXGRcaGbahSVqhp", "oRKqPmhM", "VdODasEdDWFSRIQf"]	desktops	tinyint_col
+6	62	915699741	999653836472045196	true	4.51937536E8	8.7961505445021914E17	Tale get speed platform august curved. Ease grass neighbour landlord. Baby genetic youth.	\N	2022-08-07T09:30:56	875620.2176	phones	smallint_col	\N	[9.423540715161855e+17, 4.8332499920295616e+17, 9.167007747789834e+17]	["zNfbLeFx", "GNTJOmWJyRmOK", "hwvfhSQGsaaMEqUrWCK", "cQrQsROKLARA", "nONj", "oepXBFB", "IPtUql"]	phones	smallint_col
+
+-- !schema_5 --
+00cwjIryUv	EXHwpeK2Nl	hv2PYEMYMM	eo69nyw4Yv	K6797tgjFg	LlFNd8Kyy5	wkpLCO3uo1	AIXCj1MfeD	ni0HxZbiUO	6IjRdM8Gqi	qsTMK6A2eC	1wu7v9OPwW	qavArd9tDc	sU88hZADLj	lyzWlwLOCx	2022-11-25
+
+-- !schema_6 --
+""	"test"
+
+-- !schema_7 --
+\N	\N	\N	\N	\N	\N	\N	\N	\N		test		test	test      	1	2	3	4	5.1	6.2	true	false	2011-05-06	2011-05-06T07:08:09.123	-1.2	12.30	-1234.5678	123456789.12340000	-1234567890.12345678	1234567890123456789012.1234567800000000	dGVzdDI=
+
+-- !schema_1 --
+1	7706	1	155190	17.00	21168.23	0.04	0.02	N	O	1996-03-13	1996-02-12	1996-03-22	DELIVER IN PERSON	TRUCK	egular courts above the	cn	beijing
+
+-- !schema_2 --
+6374628540732951412	-77	-65	-70	-107	-215	65	0	-526	-1309	3750	8827	-19795	34647	57042	-1662	-138248	-890685	-228568	1633079	-2725524	6163040	-10491702	697237	74565050	127767368	93532213	-209675435	-32116110	-3624917040	-2927805617	15581947241	21893441661	24075494509	-116822110531	-59683724667	-146210393388	114424524398	1341560771667	-1638742564263	520137948334	-2927347587131	7415137351179	-7963937754617	52157548982266	140803519083304	-294675355729619	-868076759504942	181128508165910	-91753231238823	-3511241416682881	-11545256318348796	-1952917510863468	-5161099825338866	-59726090170689781	287170105829528178	607326725526282735	1253194074103207461	-162443950414676064	-2964036188567341159	2602201580810990248	5581917084094110764	111739292249520611	-315687754593838642	-2804420462762366976	-2078683524
+
+-- !schema_3 --
+false	5	5	5	50	5.5	50.5	7295	12/31/10	5	2010-12-31T12:05:13.600	2010	12
+false	7	7	7	70	7.7	70.7	7297	12/31/10	7	2010-12-31T12:07:13.710	2010	12
+false	9	9	9	90	9.9	90.89999999999999	7299	12/31/10	9	2010-12-31T12:09:13.860	2010	12
+true	6	6	6	60	6.6	60.599999999999994	7296	12/31/10	6	2010-12-31T12:06:13.650	2010	12
+true	8	8	8	80	8.8	80.8	7298	12/31/10	8	2010-12-31T12:08:13.780	2010	12
+
+-- !schema_4 --
+2	24	15314771	999319712124142303	true	6.009337E8	4.8177228079770208E16	\N	northern rural	2022-08-30T23:21:08	407186.2849	phones	int_col	2019-01-01	[2.5954339078494106e+17, 5.88165568758352e+17, 4.780259987226574e+17, 6.926622881251557e+17, 9.86405645575228e+17]	\N	phones	int_col
+5	59	317349992	998913039814974432	false	5.6584864E8	9.900861328269033E17	Handling man satisfy firework descent top. Racing closed county set-up crown cave. Correctly front duration pure.	\N	2022-09-02T19:52:57	372765.2493	desktops	tinyint_col	2021-10-03	[9.983261252571983e+17, 3.612076153030643e+17, 9.969131496509435e+17, 8.991290717923475e+17, 1.1955893747098878e+17]	["CrySxz", "FMXGRcaGbahSVqhp", "oRKqPmhM", "VdODasEdDWFSRIQf"]	desktops	tinyint_col
+6	62	915699741	999653836472045196	true	4.51937536E8	8.7961505445021914E17	Tale get speed platform august curved. Ease grass neighbour landlord. Baby genetic youth.	\N	2022-08-07T09:30:56	875620.2176	phones	smallint_col	\N	[9.423540715161855e+17, 4.8332499920295616e+17, 9.167007747789834e+17]	["zNfbLeFx", "GNTJOmWJyRmOK", "hwvfhSQGsaaMEqUrWCK", "cQrQsROKLARA", "nONj", "oepXBFB", "IPtUql"]	phones	smallint_col
+
+-- !schema_5 --
+00cwjIryUv	EXHwpeK2Nl	hv2PYEMYMM	eo69nyw4Yv	K6797tgjFg	LlFNd8Kyy5	wkpLCO3uo1	AIXCj1MfeD	ni0HxZbiUO	6IjRdM8Gqi	qsTMK6A2eC	1wu7v9OPwW	qavArd9tDc	sU88hZADLj	lyzWlwLOCx	2022-11-25
+
+-- !schema_6 --
+""	"test"
+
+-- !schema_7 --
+\N	\N	\N	\N	\N	\N	\N	\N	\N		test		test	test      	1	2	3	4	5.1	6.2	true	false	2011-05-06	2011-05-06T07:08:09.123	-1.2	12.30	-1234.5678	123456789.12340000	-1234567890.12345678	1234567890123456789012.1234567800000000	dGVzdDI=
+

regression-test/suites/external_table_p0/hive/test_hive_get_schema_from_table.groovy

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_get_schema_from_table", "external_docker,hive,external_docker_hive,p0,external") {
+    String enabled = context.config.otherConfigs.get("enableHiveTest")
+    if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+        logger.info("diable Hive test.")
+        return;
+    }
+
+    // test get scheam from table
+    for (String hivePrefix : ["hive2", "hive3"]) {
+       String catalog_name = "test_${hivePrefix}_get_schema"
+       String ex_db_name = "`default`"
+       String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+       String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
+       String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort")
+
+       sql """drop catalog if exists ${catalog_name} """
+
+       sql """CREATE CATALOG ${catalog_name} PROPERTIES (
+           'type'='hms',
+           'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+           'hadoop.username' = 'hive',
+           'get_schema_from_table' = 'true'
+       );"""
+
+       sql """switch ${catalog_name}"""
+
+       def res_dbs_log = sql "show databases;"
+       for (int i = 0; i < res_dbs_log.size(); i++) {
+           def tbs = sql "show tables from  `${res_dbs_log[i][0]}`"
+           log.info("database = ${res_dbs_log[i][0]} => tables = " + tbs.toString())
+       }
+
+       order_qt_schema_1 """select * from ${catalog_name}.${ex_db_name}.parquet_partition_table order by l_orderkey limit 1;"""
+       order_qt_schema_2 """select * from ${catalog_name}.${ex_db_name}.parquet_delta_binary_packed order by int_value limit 1;"""
+       order_qt_schema_3 """select * from ${catalog_name}.${ex_db_name}.parquet_alltypes_tiny_pages  order by id desc  limit 5;"""
+       order_qt_schema_4 """select * from ${catalog_name}.${ex_db_name}.orc_all_types_partition order by bigint_col desc limit 3;"""
+       order_qt_schema_5 """select * from ${catalog_name}.${ex_db_name}.csv_partition_table order by k1 limit 1;"""
+       order_qt_schema_6 """select * from ${catalog_name}.${ex_db_name}.csv_all_types limit 1;"""
+       order_qt_schema_7 """select * except(t_varchar_max_length) from ${catalog_name}.${ex_db_name}.text_all_types limit 1;"""
+
+       //sql """drop catalog if exists ${catalog_name} """
+
+    }
+}
+