From 129df153b901e30008f3069084f7c9fd461bba95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E7=BF=94=E5=AE=87?= Date: Thu, 29 Jun 2023 14:42:15 +0800 Subject: [PATCH] [Enhancement](multi-catalog) expose config: hms client pool size. --- .../src/main/java/org/apache/doris/common/Config.java | 6 +++++- .../org/apache/doris/datasource/HMSExternalCatalog.java | 3 +-- .../apache/doris/datasource/hive/HiveMetaStoreCache.java | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index fd4e19f574dabf..42eec3171612ce 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1694,7 +1694,7 @@ public class Config extends ConfigBase { @ConfField(mutable = false, masterOnly = false, description = {"Hive表到分区名列表缓存的最大数量。", "Max cache number of hive table to partition names list."}) - public static long max_hive_table_catch_num = 1000; + public static long max_hive_table_cache_num = 1000; @ConfField(mutable = false, masterOnly = false, description = {"获取Hive分区值时候的最大返回数量,-1代表没有限制。", "Max number of hive partition values to return while list partitions, -1 means no limitation."}) @@ -1707,6 +1707,10 @@ public class Config extends ConfigBase { @ConfField(mutable = false, masterOnly = false) public static int max_external_cache_loader_thread_pool_size = 10; + @ConfField(mutable = false, masterOnly = false, description = {"Hive Metastore 池实例数。", + "Max number of hive metastore client pool."}) + public static short max_hms_client_pool_size = 8; + /** * Max cache num of external catalog's file * Decrease this value if FE's memory is small diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java index 4d6c8b86fe14f4..2c92897ae3e746 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java @@ -53,7 +53,6 @@ public class HMSExternalCatalog extends ExternalCatalog { private static final Logger LOG = LogManager.getLogger(HMSExternalCatalog.class); - private static final int MAX_CLIENT_POOL_SIZE = 8; protected PooledHiveMetaStoreClient client; // Record the latest synced event id when processing hive events // Must set to -1 otherwise client.getNextNotification will throw exception @@ -156,7 +155,7 @@ protected void initLocalObjectsImpl() { } } - client = new PooledHiveMetaStoreClient(hiveConf, MAX_CLIENT_POOL_SIZE); + client = new PooledHiveMetaStoreClient(hiveConf, Config.max_hms_client_pool_size); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index d64cdf477b2164..520eb84f2e53c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -129,7 +129,7 @@ public HiveMetaStoreCache(HMSExternalCatalog catalog, Executor executor) { } private void init() { - partitionValuesCache = CacheBuilder.newBuilder().maximumSize(Config.max_hive_table_catch_num) + partitionValuesCache = CacheBuilder.newBuilder().maximumSize(Config.max_hive_table_cache_num) .expireAfterAccess(Config.external_cache_expire_time_minutes_after_access, TimeUnit.MINUTES) .build(CacheLoader.asyncReloading( new CacheLoader() {