Try disabling grpc server auto flow control

Was turned on by default during 1.26.0->1.31.1 grpc-java bump It seems that it may be causing errors in RBE: io.grpc.StatusRuntimeException: RESOURCE_EXHAUSTED: Bandwidth exhausted HTTP/2 error code: ENHANCE_YOUR_CALM Received Goaway too_many_pings bazelbuild#12264 grpc/grpc-java#7302 Closes bazelbuild#12266. PiperOrigin-RevId: 337254515
coeuvre · Oct 15, 2020 · 6e94b05 · 6e94b05
1 parent 0218270
commit 6e94b05
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 1 deletion.
diff --git a/src/main/java/com/google/devtools/build/lib/server/GrpcServerImpl.java b/src/main/java/com/google/devtools/build/lib/server/GrpcServerImpl.java
@@ -388,14 +388,22 @@ public void serve() throws AbruptExitException {
         throw new IOException("ipv6 is not preferred on the system.");
       }
       server =
-          NettyServerBuilder.forAddress(address).addService(this).directExecutor().build().start();
+          NettyServerBuilder.forAddress(address)
+              .addService(this)
+              .directExecutor()
+              // disable auto flow control https://github.com/bazelbuild/bazel/issues/12264
+              .flowControlWindow(NettyServerBuilder.DEFAULT_FLOW_CONTROL_WINDOW)
+              .build()
+              .start();
     } catch (IOException ipv6Exception) {
       address = new InetSocketAddress("127.0.0.1", port);
       try {
         server =
             NettyServerBuilder.forAddress(address)
                 .addService(this)
                 .directExecutor()
+                // disable auto flow control https://github.com/bazelbuild/bazel/issues/12264
+                .flowControlWindow(NettyServerBuilder.DEFAULT_FLOW_CONTROL_WINDOW)
                 .build()
                 .start();
       } catch (IOException ipv4Exception) {

diff --git a/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java b/src/tools/remote/src/main/java/com/google/devtools/build/remote/worker/RemoteWorker.java
@@ -168,6 +168,9 @@ public Server startServer() throws IOException {
       logger.atInfo().log("Execution disabled, only serving cache requests");
     }
 
+    // disable auto flow control https://github.com/bazelbuild/bazel/issues/12264
+    b.flowControlWindow(NettyServerBuilder.DEFAULT_FLOW_CONTROL_WINDOW);
+
     Server server = b.build();
     logger.atInfo().log("Starting gRPC server on port %d", workerOptions.listenPort);
     server.start();