From 17089dde867b7c6076e543f3408ea55645d401d5 Mon Sep 17 00:00:00 2001 From: Yi Cheng <74173148+iycheng@users.noreply.github.com> Date: Wed, 8 Feb 2023 10:46:45 -0800 Subject: [PATCH 1/2] [core] Fix the GCS memory usage high issue It's not because of leak. The root cause is because we allocate more requests when start. This PR fixed it by making the number of call constant. --- src/ray/rpc/grpc_server.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ray/rpc/grpc_server.cc b/src/ray/rpc/grpc_server.cc index d38e02d57151..208950652a71 100644 --- a/src/ray/rpc/grpc_server.cc +++ b/src/ray/rpc/grpc_server.cc @@ -148,7 +148,7 @@ void GrpcServer::Run() { if (entry->GetMaxActiveRPCs() != -1) { buffer_size = entry->GetMaxActiveRPCs(); } - for (int j = 0; j < buffer_size; j++) { + for (int j = 0; j < (buffer_size / num_threads_); j++) { entry->CreateCall(); } } From 37790b38ae0de2ae6be475c357b3c50c2574a440 Mon Sep 17 00:00:00 2001 From: Yi Cheng <74173148+iycheng@users.noreply.github.com> Date: Wed, 8 Feb 2023 13:49:57 -0800 Subject: [PATCH 2/2] [core] Fix comments and a corner case in #32302 (#32323) This is a corner case where buffer could be 0 and a comments needs to be fixed in the previous PR. --- src/ray/rpc/grpc_server.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ray/rpc/grpc_server.cc b/src/ray/rpc/grpc_server.cc index 208950652a71..09556cc7ac5d 100644 --- a/src/ray/rpc/grpc_server.cc +++ b/src/ray/rpc/grpc_server.cc @@ -40,6 +40,7 @@ GrpcServer::GrpcServer(std::string name, is_closed_(true), num_threads_(num_threads), keepalive_time_ms_(keepalive_time_ms) { + RAY_CHECK(num_threads_ > 0) << "Num of threads in gRPC must be greater than 0"; cqs_.resize(num_threads_); // Enable built in health check implemented by gRPC: // https://github.com/grpc/grpc/blob/master/doc/health-checking.md @@ -148,7 +149,7 @@ void GrpcServer::Run() { if (entry->GetMaxActiveRPCs() != -1) { buffer_size = entry->GetMaxActiveRPCs(); } - for (int j = 0; j < (buffer_size / num_threads_); j++) { + for (int j = 0; j < std::max(1, buffer_size / num_threads_); j++) { entry->CreateCall(); } }