support zstd compression (#1950)

LICENSES/LICENSE.zstd-jni.txt

@@ -0,0 +1,26 @@
+Zstd-jni: JNI bindings to Zstd Library
+
+Copyright (c) 2015-present, Luben Karavelov/ All rights reserved.
+
+BSD License
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

client/src/main/java/org/asynchttpclient/netty/request/NettyRequestFactory.java

@@ -18,6 +18,7 @@
 import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import io.netty.handler.codec.compression.Brotli;
+import io.netty.handler.codec.compression.Zstd;
 import io.netty.handler.codec.http.DefaultFullHttpRequest;
 import io.netty.handler.codec.http.DefaultHttpRequest;
 import io.netty.handler.codec.http.HttpHeaderValues;
@@ -67,6 +68,7 @@
 import static org.asynchttpclient.util.HttpUtils.ACCEPT_ALL_HEADER_VALUE;
 import static org.asynchttpclient.util.HttpUtils.GZIP_DEFLATE;
 import static org.asynchttpclient.util.HttpUtils.filterOutBrotliFromAcceptEncoding;
+import static org.asynchttpclient.util.HttpUtils.filterOutZstdFromAcceptEncoding;
 import static org.asynchttpclient.util.HttpUtils.hostHeader;
 import static org.asynchttpclient.util.HttpUtils.originHeader;
 import static org.asynchttpclient.util.HttpUtils.urlEncodeFormParams;
@@ -182,13 +184,21 @@ public NettyRequest newNettyRequest(Request request, boolean performConnectReque
                         // For manual decompression by user, any encoding may suite, so leave untouched
                         headers.set(ACCEPT_ENCODING, filterOutBrotliFromAcceptEncoding(userDefinedAcceptEncoding));
                     }
+                    if (!Zstd.isAvailable()) {
+                        // zstd is not available.
+                        // For manual decompression by user, any encoding may suit, so leave untouched
+                        headers.set(ACCEPT_ENCODING, filterOutZstdFromAcceptEncoding(userDefinedAcceptEncoding));
+                    }
                 }
             } else if (config.isCompressionEnforced()) {
                 // Add Accept Encoding header if compression is enforced
                 headers.set(ACCEPT_ENCODING, GZIP_DEFLATE);
                 if (Brotli.isAvailable()) {
                     headers.add(ACCEPT_ENCODING, HttpHeaderValues.BR);
                 }
+                if (Zstd.isAvailable()) {
+                    headers.add(ACCEPT_ENCODING, HttpHeaderValues.ZSTD);
+                }
             }
         }
 

client/src/main/java/org/asynchttpclient/util/HttpUtils.java

@@ -40,6 +40,7 @@ public final class HttpUtils {
     private static final String CONTENT_TYPE_CHARSET_ATTRIBUTE = "charset=";
     private static final String CONTENT_TYPE_BOUNDARY_ATTRIBUTE = "boundary=";
     private static final String BROTLY_ACCEPT_ENCODING_SUFFIX = ", br";
+    private static final String ZSTD_ACCEPT_ENCODING_SUFFIX = ", zstd";
 
     private HttpUtils() {
         // Prevent outside initialization
@@ -173,4 +174,12 @@ public static CharSequence filterOutBrotliFromAcceptEncoding(String acceptEncodi
         }
         return acceptEncoding;
     }
+
+    public static CharSequence filterOutZstdFromAcceptEncoding(String acceptEncoding) {
+        // we don't support zstd ATM
+        if (acceptEncoding.endsWith(ZSTD_ACCEPT_ENCODING_SUFFIX)) {
+            return acceptEncoding.subSequence(0, acceptEncoding.length() - ZSTD_ACCEPT_ENCODING_SUFFIX.length());
+        }
+        return acceptEncoding;
+    }
 }

client/src/test/java/org/asynchttpclient/AutomaticDecompressionTest.java

@@ -0,0 +1,161 @@
+/*
+ *    Copyright (c) 2015-2024 AsyncHttpClient Project. All rights reserved.
+ *
+ *    Licensed under the Apache License, Version 2.0 (the "License");
+ *    you may not use this file except in compliance with the License.
+ *    You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS,
+ *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *    See the License for the specific language governing permissions and
+ *    limitations under the License.
+ */
+package org.asynchttpclient;
+
+import com.aayushatharva.brotli4j.encoder.BrotliOutputStream;
+import com.aayushatharva.brotli4j.encoder.Encoder;
+import com.sun.net.httpserver.Headers;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.zip.GZIPOutputStream;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import com.github.luben.zstd.Zstd;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class AutomaticDecompressionTest {
+  private static final String UNCOMPRESSED_PAYLOAD = "a".repeat(500);
+
+  private static HttpServer HTTP_SERVER;
+
+  private static AsyncHttpClient createClient() {
+    AsyncHttpClientConfig config = new DefaultAsyncHttpClientConfig.Builder()
+        .setEnableAutomaticDecompression(true)
+        .setCompressionEnforced(true)
+        .build();
+    return new DefaultAsyncHttpClient(config);
+  }
+
+  @BeforeAll
+  static void setupServer() throws Exception {
+    HTTP_SERVER = HttpServer.create(new InetSocketAddress(0), 0);
+
+    HTTP_SERVER.createContext("/br").setHandler(new HttpHandler() {
+      @Override
+      public void handle(HttpExchange exchange)
+          throws IOException {
+        validateAcceptEncodingHeader(exchange);
+        exchange.getResponseHeaders().set("Content-Encoding", "br");
+        exchange.sendResponseHeaders(200, 0);
+        OutputStream out = exchange.getResponseBody();
+        Encoder.Parameters params = new Encoder.Parameters();
+        BrotliOutputStream brotliOutputStream = new BrotliOutputStream(out, params);
+        brotliOutputStream.write(UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8));
+        brotliOutputStream.flush();
+        brotliOutputStream.close();
+      }
+    });
+
+    HTTP_SERVER.createContext("/zstd").setHandler(new HttpHandler() {
+      @Override
+      public void handle(HttpExchange exchange)
+          throws IOException {
+        validateAcceptEncodingHeader(exchange);
+        exchange.getResponseHeaders().set("Content-Encoding", "zstd");
+        byte[] compressedData = new byte[UNCOMPRESSED_PAYLOAD.length()];
+        long n = Zstd.compress(compressedData, UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8), 2, true);
+        exchange.sendResponseHeaders(200, n);
+        OutputStream out = exchange.getResponseBody();
+        out.write(compressedData, 0, (int) n);
+        out.flush();
+        out.close();
+      }
+    });
+
+    HTTP_SERVER.createContext("/gzip").setHandler(new HttpHandler() {
+      @Override
+      public void handle(HttpExchange exchange)
+          throws IOException {
+        validateAcceptEncodingHeader(exchange);
+        exchange.getResponseHeaders().set("Content-Encoding", "gzip");
+        exchange.sendResponseHeaders(200, 0);
+        OutputStream out = exchange.getResponseBody();
+        GZIPOutputStream gzip = new GZIPOutputStream(out);
+        gzip.write(UNCOMPRESSED_PAYLOAD.getBytes(StandardCharsets.UTF_8));
+        gzip.flush();
+        gzip.close();
+      }
+    });
+
+    HTTP_SERVER.start();
+  }
+
+  private static void validateAcceptEncodingHeader(HttpExchange exchange) {
+    Headers requestHeaders = exchange.getRequestHeaders();
+    List<String> acceptEncodingList = requestHeaders.get("Accept-Encoding")
+        .stream()
+        .flatMap(x -> Arrays.asList(x.split(",")).stream())
+        .collect(Collectors.toList());
+    assertEquals(List.of("gzip", "deflate", "br", "zstd"), acceptEncodingList);
+  }
+
+  @AfterAll
+  static void stopServer() {
+    if (HTTP_SERVER != null) {
+      HTTP_SERVER.stop(0);
+    }
+  }
+
+  @Test
+  void zstd() throws Throwable {
+    io.netty.handler.codec.compression.Zstd.ensureAvailability();
+    try (AsyncHttpClient client = createClient()) {
+      Request request = new RequestBuilder("GET")
+          .setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/zstd")
+          .build();
+      Response response = client.executeRequest(request).get();
+      assertEquals(200, response.getStatusCode());
+      assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody());
+    }
+  }
+
+  @Test
+  void brotli() throws Throwable {
+    io.netty.handler.codec.compression.Brotli.ensureAvailability();
+    try (AsyncHttpClient client = createClient()) {
+      Request request = new RequestBuilder("GET")
+          .setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/br")
+          .build();
+      Response response = client.executeRequest(request).get();
+      assertEquals(200, response.getStatusCode());
+      assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody());
+    }
+  }
+
+  @Test
+  void gzip() throws Throwable {
+    try (AsyncHttpClient client = createClient()) {
+      Request request = new RequestBuilder("GET")
+          .setUrl("http://localhost:" + HTTP_SERVER.getAddress().getPort() + "/gzip")
+          .build();
+      Response response = client.executeRequest(request).get();
+      assertEquals(200, response.getStatusCode());
+      assertEquals(UNCOMPRESSED_PAYLOAD, response.getResponseBody());
+    }
+  }
+
+
+}

client/src/test/java/org/asynchttpclient/netty/NettyTest.java

@@ -3,6 +3,7 @@
 import io.netty.channel.epoll.Epoll;
 import io.netty.channel.kqueue.KQueue;
 import io.netty.handler.codec.compression.Brotli;
+import io.netty.handler.codec.compression.Zstd;
 import io.netty.incubator.channel.uring.IOUring;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.EnabledOnOs;
@@ -40,4 +41,16 @@ public void brotliIsAvailableOnLinux() {
     public void brotliIsAvailableOnMac() {
         assertTrue(Brotli.isAvailable());
     }
+
+    @Test
+    @EnabledOnOs(value = OS.LINUX)
+    public void zstdIsAvailableOnLinux() {
+        assertTrue(Zstd.isAvailable());
+    }
+
+    @Test
+    @EnabledOnOs(value = OS.MAC)
+    public void zstdIsAvailableOnMac() {
+        assertTrue(Zstd.isAvailable());
+    }
 }

pom.xml

@@ -62,6 +62,7 @@
         <netty.iouring>0.0.25.Final</netty.iouring>
         <brotli4j.version>1.16.0</brotli4j.version>
         <slf4j.version>2.0.13</slf4j.version>
+        <zstd-jni.version>1.5.6-3</zstd-jni.version>
         <activation.version>2.0.1</activation.version>
         <logback.version>1.4.11</logback.version>
         <jetbrains-annotations.version>24.0.1</jetbrains-annotations.version>
@@ -224,6 +225,13 @@
             <optional>true</optional>
         </dependency>
 
+        <dependency>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+            <version>${zstd-jni.version}</version>
+            <optional>true</optional>
+        </dependency>
+
         <dependency>
             <groupId>com.aayushatharva.brotli4j</groupId>
             <artifactId>brotli4j</artifactId>