From c77115abd15778e9d4f00ec32d8d2b7a4a2ee670 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Fri, 2 Aug 2024 15:31:22 -0700 Subject: [PATCH] HDDS-11225. Increase ipc.server.read.threadpool.size (#7007) --- .../src/main/resources/ozone-default.xml | 9 +++++++++ .../apache/hadoop/ozone/om/OMConfigKeys.java | 3 +++ .../apache/hadoop/ozone/om/OzoneManager.java | 18 +++++++++++------- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 6901ce3f6b6..40fe929708f 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -593,6 +593,15 @@ The number of RPC handler threads for OM service endpoints. + + ozone.om.read.threadpool + 10 + OM, PERFORMANCE + + The number of threads in RPC server reading from the socket for OM service endpoints. + This config overrides Hadoop configuration "ipc.server.read.threadpool.size" for Ozone Manager. + + ozone.om.http-address 0.0.0.0:9874 diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index da7dab5c683..ccc936f385c 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -52,6 +52,9 @@ private OMConfigKeys() { public static final String OZONE_OM_HANDLER_COUNT_KEY = "ozone.om.handler.count.key"; public static final int OZONE_OM_HANDLER_COUNT_DEFAULT = 100; + public static final String OZONE_OM_READ_THREADPOOL_KEY = + "ozone.om.read.threadpool"; + public static final int OZONE_OM_READ_THREADPOOL_DEFAULT = 10; public static final String OZONE_OM_INTERNAL_SERVICE_ID = "ozone.om.internal.service.id"; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 387b96dcb4f..03896a042ec 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -286,6 +286,8 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_READ_THREADPOOL_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_READ_THREADPOOL_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_S3_GPRC_SERVER_ENABLED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_S3_GRPC_SERVER_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_NAMESPACE_STRICT_S3; @@ -1241,8 +1243,6 @@ private RPC.Server getRpcServer(OzoneConfiguration conf) throws IOException { OzoneNetUtils.getAddressWithHostNameLocal(omNodeRpcAddr); } - final int handlerCount = conf.getInt(OZONE_OM_HANDLER_COUNT_KEY, - OZONE_OM_HANDLER_COUNT_DEFAULT); RPC.setProtocolEngine(configuration, OzoneManagerProtocolPB.class, ProtobufRpcEngine.class); @@ -1271,8 +1271,8 @@ private RPC.Server getRpcServer(OzoneConfiguration conf) throws IOException { ReconfigureProtocolService.newReflectiveBlockingService( reconfigureServerProtocol); - return startRpcServer(configuration, omNodeRpcAddr, omService, - omInterService, omAdminService, reconfigureService, handlerCount); + return startRpcServer(conf, omNodeRpcAddr, omService, + omInterService, omAdminService, reconfigureService); } /** @@ -1285,7 +1285,6 @@ private RPC.Server getRpcServer(OzoneConfiguration conf) throws IOException { * (OMInterServiceProtocolPB impl) * @param reconfigureProtocolService RPC protocol for reconfigure * * (ReconfigureProtocolPB impl) - * @param handlerCount RPC server handler count * @return RPC server * @throws IOException if there is an I/O error while creating RPC server */ @@ -1293,16 +1292,21 @@ private RPC.Server startRpcServer(OzoneConfiguration conf, InetSocketAddress addr, BlockingService clientProtocolService, BlockingService interOMProtocolService, BlockingService omAdminProtocolService, - BlockingService reconfigureProtocolService, - int handlerCount) + BlockingService reconfigureProtocolService) throws IOException { + final int handlerCount = conf.getInt(OZONE_OM_HANDLER_COUNT_KEY, + OZONE_OM_HANDLER_COUNT_DEFAULT); + final int readThreads = conf.getInt(OZONE_OM_READ_THREADPOOL_KEY, + OZONE_OM_READ_THREADPOOL_DEFAULT); + RPC.Server rpcServer = preserveThreadName(() -> new RPC.Builder(conf) .setProtocol(OzoneManagerProtocolPB.class) .setInstance(clientProtocolService) .setBindAddress(addr.getHostString()) .setPort(addr.getPort()) .setNumHandlers(handlerCount) + .setNumReaders(readThreads) .setVerbose(false) .setSecretManager(delegationTokenMgr) .build());