bugfix for core of GRPCCompQueue when tiflash shutdown (#5483)

close #5480
pingcap · Jul 27, 2022 · 245e2fb · 245e2fb
1 parent 72f7481
commit 245e2fb
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 1 deletion.
diff --git a/dbms/src/Flash/Mpp/GRPCCompletionQueuePool.cpp b/dbms/src/Flash/Mpp/GRPCCompletionQueuePool.cpp
@@ -44,7 +44,10 @@ void GRPCCompletionQueuePool::thread(size_t index)
 {
     GET_METRIC(tiflash_thread_count, type_threads_of_client_cq_pool).Increment();
     SCOPE_EXIT({
-        GET_METRIC(tiflash_thread_count, type_threads_of_client_cq_pool).Decrement();
+        if (!is_shutdown)
+        {
+            GET_METRIC(tiflash_thread_count, type_threads_of_client_cq_pool).Decrement();
+        }
     });
 
     auto & q = queues[index];

diff --git a/dbms/src/Flash/Mpp/GRPCCompletionQueuePool.h b/dbms/src/Flash/Mpp/GRPCCompletionQueuePool.h
@@ -39,10 +39,16 @@ class GRPCCompletionQueuePool
 
     ::grpc::CompletionQueue & pickQueue();
 
+    void markShutdown()
+    {
+        is_shutdown = true;
+    }
+
 private:
     void thread(size_t index);
 
     std::atomic<size_t> next = 0;
+    std::atomic<bool> is_shutdown{false};
     std::vector<::grpc::CompletionQueue> queues;
     std::vector<std::thread> workers;
 };

diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp
@@ -679,6 +679,8 @@ class Server::FlashGrpcServerHolder
             thread_manager->wait();
             flash_grpc_server->Wait();
             flash_grpc_server.reset();
+            if (GRPCCompletionQueuePool::global_instance)
+                GRPCCompletionQueuePool::global_instance->markShutdown();
             LOG_FMT_INFO(log, "Shut down flash grpc server");
 
             /// Close flash service.