Limit max reset frames to mitigate HTTP/2 RST floods (#5232)

Motivation: To mitigate against the "HTTP/2 Rapid Reset" attack, it is recommended that HTTP/2 servers should close connections that exceed the concurrent stream limit. Reference: - https://blog.cloudflare.com/technical-breakdown-http2-rapid-reset-ddos-attack/ - https://www.cve.org/CVERecord?id=CVE-2023-44487 - netty/netty@58f75f6#diff-82f568a075ff63e9727ce8622f3a2b1553099182edf1fd0b4f857226252b05adR47 Modifications: - Add `ServerBuilder.http2MaxRestFramesPerWindow()` option and `-Dcom.linecorp.armeria.defaultHttp2MaxResetFramesPerMinute<integer>` property to limit the maximum allowed RST frames. - If not set, 400 RST frames per minute are allowed by default. - Bump Netty version to 4.1.100 from 4.1.96 Result: You can now protect your server against DDOS caused by RST floods. ```java Server .builder() .http2MaxResetFramesPerWindow(100, 10) .build(); ```
line · Oct 18, 2023 · 4121cf5 · 4121cf5
1 parent 95bb20a
commit 4121cf5
Show file tree

Hide file tree

Showing 12 changed files with 222 additions and 4 deletions.
diff --git a/core/src/main/java/com/linecorp/armeria/common/DefaultFlagsProvider.java b/core/src/main/java/com/linecorp/armeria/common/DefaultFlagsProvider.java
@@ -78,6 +78,7 @@ final class DefaultFlagsProvider implements FlagsProvider {
     // parameter values, thus anything greater than 0x7FFFFFFF will break them or make them unhappy.
     static final long DEFAULT_HTTP2_MAX_STREAMS_PER_CONNECTION = Integer.MAX_VALUE;
     static final long DEFAULT_HTTP2_MAX_HEADER_LIST_SIZE = 8192L; // from Netty default maxHeaderSize
+    static final int DEFAULT_HTTP2_MAX_RESET_FRAMES_PER_MINUTE = 400; // Netty default is 200 for 30 seconds
     static final String DEFAULT_BACKOFF_SPEC = "exponential=200:10000,jitter=0.2";
     static final int DEFAULT_MAX_TOTAL_ATTEMPTS = 10;
     static final long DEFAULT_REQUEST_AUTO_ABORT_DELAY_MILLIS = 0; // No delay.
@@ -322,6 +323,11 @@ public Long defaultHttp2MaxHeaderListSize() {
         return DEFAULT_HTTP2_MAX_HEADER_LIST_SIZE;
     }
 
+    @Override
+    public Integer defaultHttp2MaxResetFramesPerMinute() {
+        return DEFAULT_HTTP2_MAX_RESET_FRAMES_PER_MINUTE;
+    }
+
     @Override
     public String defaultBackoffSpec() {
         return DEFAULT_BACKOFF_SPEC;

diff --git a/core/src/main/java/com/linecorp/armeria/common/Flags.java b/core/src/main/java/com/linecorp/armeria/common/Flags.java
@@ -285,6 +285,10 @@ private static boolean validateTransportType(TransportType transportType, String
             getValue(FlagsProvider::defaultHttp2MaxHeaderListSize, "defaultHttp2MaxHeaderListSize",
                      value -> value > 0 && value <= 0xFFFFFFFFL);
 
+    private static final int DEFAULT_HTTP2_MAX_RESET_FRAMES_PER_MINUTE =
+            getValue(FlagsProvider::defaultHttp2MaxResetFramesPerMinute,
+                     "defaultHttp2MaxResetFramesPerMinute", value -> value >= 0);
+
     private static final int DEFAULT_MAX_HTTP1_INITIAL_LINE_LENGTH =
             getValue(FlagsProvider::defaultHttp1MaxInitialLineLength, "defaultHttp1MaxInitialLineLength",
                      value -> value >= 0);
@@ -1052,6 +1056,22 @@ public static long defaultHttp2MaxHeaderListSize() {
         return DEFAULT_HTTP2_MAX_HEADER_LIST_SIZE;
     }
 
+    /**
+     * Returns the default maximum number of RST frames that are allowed per window before the connection is
+     * closed. This allows to protect against the remote peer flooding us with such frames and using up a lot
+     * of CPU. Note that this flag has no effect if a user specified the value explicitly via
+     * {@link ServerBuilder#http2MaxResetFramesPerWindow(int, int)}.
+     *
+     * <p>The default value of this flag is
+     * {@value DefaultFlagsProvider#DEFAULT_HTTP2_MAX_RESET_FRAMES_PER_MINUTE}.
+     * Specify the {@code -Dcom.linecorp.armeria.defaultHttp2MaxResetFramesPerMinute=<integer>} JVM option
+     * to override the default value. {@code 0} means no protection should be applied.
+     */
+    @UnstableApi
+    public static int defaultHttp2MaxResetFramesPerMinute() {
+        return DEFAULT_HTTP2_MAX_RESET_FRAMES_PER_MINUTE;
+    }
+
     /**
      * Returns the {@linkplain Backoff#of(String) Backoff specification string} of the default {@link Backoff}
      * returned by {@link Backoff#ofDefault()}. Note that this flag has no effect if a user specified the

diff --git a/core/src/main/java/com/linecorp/armeria/common/FlagsProvider.java b/core/src/main/java/com/linecorp/armeria/common/FlagsProvider.java
@@ -711,6 +711,22 @@ default Long defaultHttp2MaxHeaderListSize() {
         return null;
     }
 
+    /**
+     * Returns the default maximum number of RST frames that are allowed per window before the connection is
+     * closed. This allows to protect against the remote peer flooding us with such frames and using up a lot
+     * of CPU. Note that this flag has no effect if a user specified the value explicitly via
+     * {@link ServerBuilder#http2MaxResetFramesPerWindow(int, int)}.
+     *
+     * <p>The default value of this flag is
+     * {@value DefaultFlagsProvider#DEFAULT_HTTP2_MAX_RESET_FRAMES_PER_MINUTE}.
+     * Specify the {@code -Dcom.linecorp.armeria.defaultHttp2MaxResetFramesPerMinute=<integer>} JVM option
+     * to override the default value. {@code 0} means no protection should be applied.
+     */
+    @Nullable
+    default Integer defaultHttp2MaxResetFramesPerMinute() {
+        return null;
+    }
+
     /**
      * Returns the {@linkplain Backoff#of(String) Backoff specification string} of the default {@link Backoff}
      * returned by {@link Backoff#ofDefault()}. Note that this flag has no effect if a user specified the

diff --git a/core/src/main/java/com/linecorp/armeria/common/SystemPropertyFlagsProvider.java b/core/src/main/java/com/linecorp/armeria/common/SystemPropertyFlagsProvider.java
@@ -298,6 +298,11 @@ public Long defaultHttp2MaxHeaderListSize() {
         return getLong("defaultHttp2MaxHeaderListSize");
     }
 
+    @Override
+    public Integer defaultHttp2MaxResetFramesPerMinute() {
+        return getInt("defaultHttp2MaxResetFramesPerMinute");
+    }
+
     @Override
     public String defaultBackoffSpec() {
         return getNormalized("defaultBackoffSpec");

diff --git a/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java b/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java
@@ -85,6 +85,8 @@ final class DefaultServerConfig implements ServerConfig {
     private final long http2MaxStreamsPerConnection;
     private final int http2MaxFrameSize;
     private final long http2MaxHeaderListSize;
+    private final int http2MaxResetFramesPerWindow;
+    private final int http2MaxResetFramesWindowSeconds;
     private final int http1MaxInitialLineLength;
     private final int http1MaxHeaderSize;
     private final int http1MaxChunkSize;
@@ -129,8 +131,9 @@ final class DefaultServerConfig implements ServerConfig {
             long maxConnectionAgeMillis,
             int maxNumRequestsPerConnection, long connectionDrainDurationMicros,
             int http2InitialConnectionWindowSize, int http2InitialStreamWindowSize,
-            long http2MaxStreamsPerConnection, int http2MaxFrameSize,
-            long http2MaxHeaderListSize, int http1MaxInitialLineLength, int http1MaxHeaderSize,
+            long http2MaxStreamsPerConnection, int http2MaxFrameSize, long http2MaxHeaderListSize,
+            int http2MaxResetFramesPerWindow, int http2MaxResetFramesWindowSeconds,
+            int http1MaxInitialLineLength, int http1MaxHeaderSize,
             int http1MaxChunkSize, Duration gracefulShutdownQuietPeriod, Duration gracefulShutdownTimeout,
             BlockingTaskExecutor blockingTaskExecutor,
             MeterRegistry meterRegistry, int proxyProtocolMaxTlvSize,
@@ -171,6 +174,8 @@ final class DefaultServerConfig implements ServerConfig {
         this.http2MaxStreamsPerConnection = http2MaxStreamsPerConnection;
         this.http2MaxFrameSize = http2MaxFrameSize;
         this.http2MaxHeaderListSize = http2MaxHeaderListSize;
+        this.http2MaxResetFramesPerWindow = http2MaxResetFramesPerWindow;
+        this.http2MaxResetFramesWindowSeconds = http2MaxResetFramesWindowSeconds;
         this.http1MaxInitialLineLength = validateNonNegative(
                 http1MaxInitialLineLength, "http1MaxInitialLineLength");
         this.http1MaxHeaderSize = validateNonNegative(
@@ -568,6 +573,16 @@ public long http2MaxHeaderListSize() {
         return http2MaxHeaderListSize;
     }
 
+    @Override
+    public int http2MaxResetFramesPerWindow() {
+        return http2MaxResetFramesPerWindow;
+    }
+
+    @Override
+    public int http2MaxResetFramesWindowSeconds() {
+        return http2MaxResetFramesWindowSeconds;
+    }
+
     @Override
     public Duration gracefulShutdownQuietPeriod() {
         return gracefulShutdownQuietPeriod;

diff --git a/core/src/main/java/com/linecorp/armeria/server/Http2ServerConnectionHandlerBuilder.java b/core/src/main/java/com/linecorp/armeria/server/Http2ServerConnectionHandlerBuilder.java
@@ -42,6 +42,8 @@ final class Http2ServerConnectionHandlerBuilder
         // Disable graceful shutdown timeout in a super class. Server-side HTTP/2 graceful shutdown is
         // handled by Armeria's HTTP/2 server handler.
         gracefulShutdownTimeoutMillis(-1);
+        decoderEnforceMaxRstFramesPerWindow(config.http2MaxResetFramesPerWindow(),
+                                            config.http2MaxResetFramesWindowSeconds());
     }
 
     @Override

diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerBuilder.java b/core/src/main/java/com/linecorp/armeria/server/ServerBuilder.java
@@ -230,6 +230,8 @@ public final class ServerBuilder implements TlsSetters, ServiceConfigsBuilder {
     private long unhandledExceptionsReportIntervalMillis =
             Flags.defaultUnhandledExceptionsReportIntervalMillis();
     private final List<ShutdownSupport> shutdownSupports = new ArrayList<>();
+    private int http2MaxResetFramesPerWindow = Flags.defaultHttp2MaxResetFramesPerMinute();
+    private int http2MaxResetFramesWindowSeconds = 60;
 
     ServerBuilder() {
         // Set the default host-level properties.
@@ -771,6 +773,26 @@ public ServerBuilder http2MaxStreamsPerConnection(long http2MaxStreamsPerConnect
         return this;
     }
 
+    /**
+     * Sets the maximum number of RST frames that are allowed per window before the connection is closed. This
+     * allows to protect against the remote peer flooding us with such frames and using up a lot of CPU.
+     * Defaults to {@link Flags#defaultHttp2MaxResetFramesPerMinute()}.
+     *
+     * <p>Note that {@code 0} for any of the parameters means no protection should be applied.
+     */
+    @UnstableApi
+    public ServerBuilder http2MaxResetFramesPerWindow(int http2MaxResetFramesPerWindow,
+                                                      int http2MaxResetFramesWindowSeconds) {
+        checkArgument(http2MaxResetFramesPerWindow >= 0, "http2MaxResetFramesPerWindow: %s (expected: >= 0)",
+                      http2MaxResetFramesPerWindow);
+        checkArgument(http2MaxResetFramesWindowSeconds >= 0,
+                      "http2MaxResetFramesWindowSeconds: %s (expected: >= 0)",
+                      http2MaxResetFramesWindowSeconds);
+        this.http2MaxResetFramesPerWindow = http2MaxResetFramesPerWindow;
+        this.http2MaxResetFramesWindowSeconds = http2MaxResetFramesWindowSeconds;
+        return this;
+    }
+
     /**
      * Sets the maximum size of HTTP/2 frame that can be received. Defaults to
      * {@link Flags#defaultHttp2MaxFrameSize()}.
@@ -2168,7 +2190,9 @@ ports, setSslContextIfAbsent(defaultVirtualHost, defaultSslContext),
                 maxNumRequestsPerConnection,
                 connectionDrainDurationMicros, http2InitialConnectionWindowSize,
                 http2InitialStreamWindowSize, http2MaxStreamsPerConnection,
-                http2MaxFrameSize, http2MaxHeaderListSize, http1MaxInitialLineLength, http1MaxHeaderSize,
+                http2MaxFrameSize, http2MaxHeaderListSize,
+                http2MaxResetFramesPerWindow, http2MaxResetFramesWindowSeconds,
+                http1MaxInitialLineLength, http1MaxHeaderSize,
                 http1MaxChunkSize, gracefulShutdownQuietPeriod, gracefulShutdownTimeout,
                 blockingTaskExecutor,
                 meterRegistry, proxyProtocolMaxTlvSize, channelOptions, newChildChannelOptions,

diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerConfig.java b/core/src/main/java/com/linecorp/armeria/server/ServerConfig.java
@@ -213,6 +213,20 @@ public interface ServerConfig {
      */
     long http2MaxHeaderListSize();
 
+    /**
+     * Returns the maximum number of RST frames that are allowed per
+     * {@link #http2MaxResetFramesWindowSeconds()}.
+     */
+    @UnstableApi
+    int http2MaxResetFramesPerWindow();
+
+    /**
+     * Returns the number of seconds during which {@link #http2MaxResetFramesPerWindow()} RST frames are
+     * allowed.
+     */
+    @UnstableApi
+    int http2MaxResetFramesWindowSeconds();
+
     /**
      * Returns the number of milliseconds to wait for active requests to go end before shutting down.
      * {@code 0} means the server will stop right away without waiting.

diff --git a/core/src/main/java/com/linecorp/armeria/server/UpdatableServerConfig.java b/core/src/main/java/com/linecorp/armeria/server/UpdatableServerConfig.java
@@ -220,6 +220,16 @@ public long http2MaxHeaderListSize() {
         return delegate.http2MaxHeaderListSize();
     }
 
+    @Override
+    public int http2MaxResetFramesPerWindow() {
+        return delegate.http2MaxResetFramesPerWindow();
+    }
+
+    @Override
+    public int http2MaxResetFramesWindowSeconds() {
+        return delegate.http2MaxResetFramesWindowSeconds();
+    }
+
     @Override
     public Duration gracefulShutdownQuietPeriod() {
         return delegate.gracefulShutdownQuietPeriod();

diff --git a/core/src/test/java/com/linecorp/armeria/server/MaxResetFramesTest.java b/core/src/test/java/com/linecorp/armeria/server/MaxResetFramesTest.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2023 LINE Corporation
+ *
+ * LINE Corporation licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.linecorp.armeria.server;
+
+import static com.google.common.collect.ImmutableList.toImmutableList;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.IntStream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import com.spotify.futures.CompletableFutures;
+
+import com.linecorp.armeria.client.ClientFactory;
+import com.linecorp.armeria.client.CountingConnectionPoolListener;
+import com.linecorp.armeria.client.WebClient;
+import com.linecorp.armeria.common.AggregatedHttpResponse;
+import com.linecorp.armeria.common.HttpMethod;
+import com.linecorp.armeria.common.HttpObject;
+import com.linecorp.armeria.common.HttpRequest;
+import com.linecorp.armeria.common.HttpResponse;
+import com.linecorp.armeria.common.RequestHeaders;
+import com.linecorp.armeria.common.SessionProtocol;
+import com.linecorp.armeria.common.stream.StreamMessage;
+import com.linecorp.armeria.testing.junit5.server.ServerExtension;
+
+class MaxResetFramesTest {
+    @RegisterExtension
+    static final ServerExtension server = new ServerExtension() {
+        @Override
+        protected void configure(ServerBuilder sb) {
+            sb.idleTimeoutMillis(0);
+            sb.http2MaxResetFramesPerWindow(10, 60);
+            sb.service("/", (ctx, req) -> {
+                return HttpResponse.of(req.aggregate().thenApply(unused -> HttpResponse.of(200)));
+            });
+        }
+    };
+
+    @Test
+    void shouldCloseConnectionWhenExceedingMaxResetFrames() {
+        final CountingConnectionPoolListener listener = new CountingConnectionPoolListener();
+        try (ClientFactory factory = ClientFactory.builder()
+                                                  .connectionPoolListener(listener)
+                                                  .idleTimeoutMillis(0)
+                                                  .build()) {
+            final WebClient client = WebClient.builder(server.uri(SessionProtocol.H2C))
+                                              .factory(factory)
+                                              .build();
+            final List<CompletableFuture<AggregatedHttpResponse>> futures =
+                    IntStream.range(0, 11)
+                             .mapToObj(unused -> HttpRequest.of(RequestHeaders.of(HttpMethod.POST, "/"),
+                                                                StreamMessage.of(InvalidHttpObject.INSTANCE)))
+                             .map(client::execute)
+                             .map(HttpResponse::aggregate)
+                             .collect(toImmutableList());
+
+            CompletableFutures.successfulAsList(futures, cause -> null).join();
+            assertThat(listener.opened()).isEqualTo(1);
+            await().untilAsserted(() -> assertThat(listener.closed()).isEqualTo(1));
+        }
+    }
+
+    /**
+     * {@link WebClient} resets a stream when it receives an invalid {@link HttpObject} from
+     * {@link HttpRequest}.
+     */
+    private enum InvalidHttpObject implements HttpObject {
+
+        INSTANCE;
+
+        @Override
+        public boolean isEndOfStream() {
+            return false;
+        }
+    }
+}
diff --git a/core/src/test/java/com/linecorp/armeria/server/ServerBuilderTest.java b/core/src/test/java/com/linecorp/armeria/server/ServerBuilderTest.java
@@ -705,4 +705,15 @@ void multipleDomainSocketAddresses() {
                 new ServerPort(DomainSocketAddress.of("/tmp/bar"),
                                SessionProtocol.HTTP, SessionProtocol.HTTPS));
     }
+
+    @Test
+    void httpMaxResetFramesPerMinute() {
+        final ServerConfig config = Server.builder()
+                                          .service("/", (ctx, req) -> HttpResponse.of(HttpStatus.OK))
+                                          .http2MaxResetFramesPerWindow(99, 2)
+                                          .build()
+                                          .config();
+        assertThat(config.http2MaxResetFramesPerWindow()).isEqualTo(99);
+        assertThat(config.http2MaxResetFramesWindowSeconds()).isEqualTo(2);
+    }
 }
diff --git a/dependencies.toml b/dependencies.toml
@@ -84,7 +84,7 @@ micrometer13 = "1.3.20"
 mockito = "4.11.0"
 monix = "3.4.1"
 munit = "0.7.29"
-netty = "4.1.96.Final"
+netty = "4.1.100.Final"
 netty-incubator-transport-native-io_uring = "0.0.21.Final"
 nexus-publish = "1.3.0"
 node-gradle-plugin = "5.0.0"