Add gzip support for bidder requests (#1518)

src/main/java/org/prebid/server/bidder/HttpBidderRequester.java

@@ -2,9 +2,11 @@
 
 import com.iab.openrtb.request.BidRequest;
 import io.netty.channel.ConnectTimeoutException;
+import io.netty.handler.codec.http.HttpHeaderValues;
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.vertx.core.CompositeFuture;
 import io.vertx.core.Future;
+import io.vertx.core.MultiMap;
 import io.vertx.core.logging.Logger;
 import io.vertx.core.logging.LoggerFactory;
 import org.apache.commons.collections4.CollectionUtils;
@@ -18,6 +20,7 @@
 import org.prebid.server.bidder.model.HttpRequest;
 import org.prebid.server.bidder.model.HttpResponse;
 import org.prebid.server.bidder.model.Result;
+import org.prebid.server.exception.PreBidException;
 import org.prebid.server.execution.Timeout;
 import org.prebid.server.json.JacksonMapper;
 import org.prebid.server.model.CaseInsensitiveMultiMap;
@@ -26,6 +29,8 @@
 import org.prebid.server.vertx.http.HttpClient;
 import org.prebid.server.vertx.http.model.HttpClientResponse;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -35,6 +40,7 @@
 import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.zip.GZIPOutputStream;
 
 /**
  * Implements HTTP communication functionality common for {@link Bidder}'s.
@@ -169,12 +175,45 @@ private <T> Future<HttpCall<T>> doRequest(HttpRequest<T> httpRequest, Timeout ti
             return failResponse(new TimeoutException("Timeout has been exceeded"), httpRequest);
         }
 
-        return httpClient.request(httpRequest.getMethod(), httpRequest.getUri(), httpRequest.getHeaders(),
-                        httpRequest.getBody(), remainingTimeout)
+        return createRequest(httpRequest, remainingTimeout)
                 .compose(response -> processResponse(response, httpRequest))
                 .recover(exception -> failResponse(exception, httpRequest));
     }
 
+    private <T> Future<HttpClientResponse> createRequest(HttpRequest<T> httpRequest, long remainingTimeout) {
+        final byte[] requestBody = httpRequest.getBody();
+        final MultiMap requestHeaders = httpRequest.getHeaders();
+
+        final byte[] preparedBody = compressIfRequired(requestBody, requestHeaders);
+
+        return httpClient.request(httpRequest.getMethod(),
+                httpRequest.getUri(),
+                requestHeaders,
+                preparedBody,
+                remainingTimeout);
+    }
+
+    private static byte[] compressIfRequired(byte[] existingBody, MultiMap headers) {
+        final String contentEncodingHeader = headers.get(HttpUtil.CONTENT_ENCODING_HEADER);
+        if (Objects.equals(contentEncodingHeader, HttpHeaderValues.GZIP.toString())) {
+            return gzip(existingBody);
+        }
+
+        return existingBody;
+    }
+
+    private static byte[] gzip(byte[] value) {
+        try (ByteArrayOutputStream obj = new ByteArrayOutputStream(); GZIPOutputStream gzip = new GZIPOutputStream(
+                obj)) {
+            gzip.write(value);
+            gzip.finish();
+
+            return obj.toByteArray();
+        } catch (IOException e) {
+            throw new PreBidException(String.format("Failed to compress request : %s", e.getMessage()));
+        }
+    }
+
     /**
      * Produces {@link Future} with {@link HttpCall} containing request and error description.
      */

src/main/java/org/prebid/server/bidder/adagio/AdagioBidder.java

@@ -7,6 +7,7 @@
 import com.iab.openrtb.response.Bid;
 import com.iab.openrtb.response.BidResponse;
 import com.iab.openrtb.response.SeatBid;
+import io.netty.handler.codec.http.HttpHeaderValues;
 import io.vertx.core.MultiMap;
 import io.vertx.core.http.HttpMethod;
 import org.apache.commons.collections4.CollectionUtils;
@@ -51,7 +52,8 @@ public Result<List<HttpRequest<BidRequest>>> makeHttpRequests(BidRequest request
     }
 
     private static MultiMap resolveHeaders(Device device) {
-        final MultiMap headers = HttpUtil.headers();
+        final MultiMap headers = HttpUtil.headers()
+                .add(HttpUtil.CONTENT_ENCODING_HEADER, HttpHeaderValues.GZIP);
 
         if (device != null) {
             HttpUtil.addHeaderIfValueIsNotEmpty(headers, HttpUtil.X_FORWARDED_FOR_HEADER, device.getIpv6());

src/main/java/org/prebid/server/bidder/gamma/GammaBidder.java

@@ -181,8 +181,7 @@ private MultiMap makeHeaders(Device device) {
                 .set(HttpUtil.X_OPENRTB_VERSION_HEADER, "2.5")
                 .set(HttpUtil.ACCEPT_HEADER, "*/*")
                 .set(HttpUtil.CACHE_CONTROL_HEADER, "no-cache")
-                .set(HttpUtil.CONNECTION_HEADER, "keep-alive")
-                .set(HttpUtil.ACCEPT_ENCODING_HEADER, "gzip, deflate");
+                .set(HttpUtil.CONNECTION_HEADER, "keep-alive");
 
         if (device != null) {
             HttpUtil.addHeaderIfValueIsNotEmpty(headers, HttpUtil.USER_AGENT_HEADER, device.getUa());

src/main/java/org/prebid/server/util/HttpUtil.java

@@ -67,6 +67,7 @@ public final class HttpUtil {
     public static final CharSequence LOCATION_HEADER = HttpHeaders.createOptimized("Location");
     public static final CharSequence CONNECTION_HEADER = HttpHeaders.createOptimized("Connection");
     public static final CharSequence ACCEPT_ENCODING_HEADER = HttpHeaders.createOptimized("Accept-Encoding");
+    public static final CharSequence CONTENT_ENCODING_HEADER = HttpHeaders.createOptimized("Content-Encoding");
     public static final CharSequence X_OPENRTB_VERSION_HEADER = HttpHeaders.createOptimized("x-openrtb-version");
     public static final CharSequence X_PREBID_HEADER = HttpHeaders.createOptimized("x-prebid");
     private static final Set<String> SENSITIVE_HEADERS = new HashSet<>(Arrays.asList(AUTHORIZATION_HEADER.toString()));

src/main/resources/application.yaml

@@ -86,7 +86,7 @@ http-client:
   idle-timeout-ms: 0
   pool-cleaner-period-ms: 1000
   connect-timeout-ms: 2500
-  use-compression: false
+  use-compression: true
   max-redirects: 0
   ssl: false
   jks-path:

src/test/java/org/prebid/server/bidder/HttpBidderRequesterTest.java

@@ -6,6 +6,7 @@
 import com.iab.openrtb.request.Imp;
 import com.iab.openrtb.request.Pmp;
 import com.iab.openrtb.response.Bid;
+import io.netty.handler.codec.http.HttpHeaderValues;
 import io.vertx.core.Future;
 import io.vertx.core.MultiMap;
 import io.vertx.core.Promise;
@@ -36,6 +37,7 @@
 import org.prebid.server.execution.TimeoutFactory;
 import org.prebid.server.model.CaseInsensitiveMultiMap;
 import org.prebid.server.proto.openrtb.ext.response.ExtHttpCall;
+import org.prebid.server.util.HttpUtil;
 import org.prebid.server.vertx.http.HttpClient;
 import org.prebid.server.vertx.http.model.HttpClientResponse;
 
@@ -187,7 +189,7 @@ public void shouldPassStoredResponseToBidderMakeBidsMethodAndReturnSeatBids() {
     @Test
     public void shouldMakeRequestToBidderWhenStoredResponseDefinedButBidderCreatesMoreThanOneRequest() {
         // given
-        givenHttpClientReturnsResponse(200, null);
+        givenHttpClientResponse(200, null);
         final MultiMap headers = new CaseInsensitiveHeaders();
         headers.add("header1", "value1");
         headers.add("header2", "value2");
@@ -218,7 +220,7 @@ public void shouldMakeRequestToBidderWhenStoredResponseDefinedButBidderCreatesMo
     @Test
     public void shouldSendPopulatedGetRequestWithoutBody() {
         // given
-        givenHttpClientReturnsResponse(200, null);
+        givenHttpClientResponse(200, null);
 
         given(bidder.makeHttpRequests(any())).willReturn(Result.of(singletonList(
                         HttpRequest.<BidRequest>builder()
@@ -239,7 +241,7 @@ public void shouldSendPopulatedGetRequestWithoutBody() {
     @Test
     public void shouldSendMultipleRequests() throws JsonProcessingException {
         // given
-        givenHttpClientReturnsResponse(200, null);
+        givenHttpClientResponse(200, null);
         final BidRequest bidRequest = givenBidRequest(identity());
 
         given(bidder.makeHttpRequests(any())).willReturn(Result.of(asList(
@@ -278,7 +280,7 @@ public void shouldReturnBidsCreatedByBidder() {
                                 .build()),
                 emptyList()));
 
-        givenHttpClientReturnsResponse(200, "responseBody");
+        givenHttpClientResponse(200, "responseBody");
 
         final List<BidderBid> bids = asList(BidderBid.of(null, null, null), BidderBid.of(null, null, null));
         given(bidder.makeBids(any(), any())).willReturn(Result.of(bids, emptyList()));
@@ -294,6 +296,34 @@ public void shouldReturnBidsCreatedByBidder() {
         assertThat(bidderSeatBid.getBids()).containsOnlyElementsOf(bids);
     }
 
+    @Test
+    public void shouldCompressRequestBodyIfContentEncodingHeaderIsGzip() {
+        // given
+        final MultiMap headers = new CaseInsensitiveHeaders()
+                .add(HttpUtil.CONTENT_ENCODING_HEADER, HttpHeaderValues.GZIP);
+        given(bidder.makeHttpRequests(any())).willReturn(Result.of(singletonList(
+                        HttpRequest.<BidRequest>builder()
+                                .method(HttpMethod.POST)
+                                .uri(EMPTY)
+                                .body(EMPTY_BYTE_BODY)
+                                .headers(new CaseInsensitiveHeaders())
+                                .build()),
+                emptyList()));
+
+        given(requestEnricher.enrichHeaders(any(), any(), any())).willReturn(headers);
+        givenHttpClientResponse(200, "responseBody");
+        final BidderRequest bidderRequest = BidderRequest.of("bidder", null, BidRequest.builder().build());
+
+        // when
+        httpBidderRequester.requestBids(bidder, bidderRequest, timeout, CaseInsensitiveMultiMap.empty(), false)
+                .result();
+
+        // then
+        final ArgumentCaptor<byte[]> actualRequestBody = ArgumentCaptor.forClass(byte[].class);
+        verify(httpClient).request(any(), anyString(), any(), actualRequestBody.capture(), anyLong());
+        assertThat(actualRequestBody.getValue()).isNotSameAs(EMPTY_BYTE_BODY);
+    }
+
     @Test
     public void shouldNotWaitForResponsesWhenAllDealsIsGathered() throws JsonProcessingException {
         // given
@@ -413,7 +443,7 @@ public void shouldFinishWhenAllDealRequestsAreFinishedAndNoDealsProvided() {
                                 .build()),
                 emptyList()));
 
-        givenHttpClientReturnsResponse(200, "responseBody");
+        givenHttpClientResponse(200, "responseBody");
 
         final BidderBid bidderBid = BidderBid.of(Bid.builder().dealid("deal2").build(), null, null);
         given(bidder.makeBids(any(), any())).willReturn(Result.of(singletonList(bidderBid), emptyList()));
@@ -766,7 +796,7 @@ public void shouldNotMakeBidsIfResponseStatusIs204() {
                                 .build()),
                 emptyList()));
 
-        givenHttpClientReturnsResponse(204, EMPTY);
+        givenHttpClientResponse(204, EMPTY);
 
         final BidderRequest bidderRequest = BidderRequest.of("bidder", null, BidRequest.builder().test(1).build());
 
@@ -803,7 +833,7 @@ private static Imp impWithDeal(String dealId) {
                 .build();
     }
 
-    private void givenHttpClientReturnsResponse(int statusCode, String response) {
+    private void givenHttpClientResponse(int statusCode, String response) {
         given(httpClient.request(any(), anyString(), any(), (byte[]) any(), anyLong()))
                 .willReturn(Future.succeededFuture(HttpClientResponse.of(statusCode, null, response)));
     }

src/test/java/org/prebid/server/bidder/adagio/AdagioBidderTest.java

@@ -71,6 +71,7 @@ public void makeHttpRequestsShouldCorrectlyAddAllHeaders() {
                 .containsExactlyInAnyOrder(
                         tuple(HttpUtil.CONTENT_TYPE_HEADER.toString(), HttpUtil.APPLICATION_JSON_CONTENT_TYPE),
                         tuple(HttpUtil.ACCEPT_HEADER.toString(), HttpHeaderValues.APPLICATION_JSON.toString()),
+                        tuple(HttpUtil.CONTENT_ENCODING_HEADER.toString(), HttpHeaderValues.GZIP.toString()),
                         tuple(HttpUtil.X_FORWARDED_FOR_HEADER.toString(), "someIp"));
     }
 

src/test/java/org/prebid/server/bidder/gamma/GammaBidderTest.java

@@ -152,7 +152,6 @@ public void makeHttpRequestsShouldFillMethodAndUrlAndExpectedHeaders() {
                         tuple("Accept", "*/*"),
                         tuple("Connection", "keep-alive"),
                         tuple("Cache-Control", "no-cache"),
-                        tuple("Accept-Encoding", "gzip, deflate"),
                         tuple("x-openrtb-version", "2.5"));
     }
 
@@ -197,7 +196,6 @@ public void makeHttpRequestsShouldFillMethodAndUrlAndExpectedHeadersWhenDeviceAn
                         tuple("Accept", "*/*"),
                         tuple("Connection", "keep-alive"),
                         tuple("Cache-Control", "no-cache"),
-                        tuple("Accept-Encoding", "gzip, deflate"),
                         tuple("User-Agent", "userAgent"),
                         tuple("x-openrtb-version", "2.5"),
                         tuple("X-Forwarded-For", "123.123.123.12"),