From 6bda093988e500eca3b9f29b925a7cce756c459d Mon Sep 17 00:00:00 2001 From: Serhii Nahornyi Date: Thu, 11 Nov 2021 18:36:34 +0200 Subject: [PATCH] Add gzip support for bidder requests (#1518) --- .../server/bidder/HttpBidderRequester.java | 43 +++++++++++++++++- .../server/bidder/adagio/AdagioBidder.java | 4 +- .../server/bidder/gamma/GammaBidder.java | 3 +- .../java/org/prebid/server/util/HttpUtil.java | 1 + src/main/resources/application.yaml | 2 +- .../bidder/HttpBidderRequesterTest.java | 44 ++++++++++++++++--- .../bidder/adagio/AdagioBidderTest.java | 1 + .../server/bidder/gamma/GammaBidderTest.java | 2 - 8 files changed, 85 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/prebid/server/bidder/HttpBidderRequester.java b/src/main/java/org/prebid/server/bidder/HttpBidderRequester.java index 5faed59173d..cf999aac0dc 100644 --- a/src/main/java/org/prebid/server/bidder/HttpBidderRequester.java +++ b/src/main/java/org/prebid/server/bidder/HttpBidderRequester.java @@ -2,9 +2,11 @@ import com.iab.openrtb.request.BidRequest; import io.netty.channel.ConnectTimeoutException; +import io.netty.handler.codec.http.HttpHeaderValues; import io.netty.handler.codec.http.HttpResponseStatus; import io.vertx.core.CompositeFuture; import io.vertx.core.Future; +import io.vertx.core.MultiMap; import io.vertx.core.logging.Logger; import io.vertx.core.logging.LoggerFactory; import org.apache.commons.collections4.CollectionUtils; @@ -18,6 +20,7 @@ import org.prebid.server.bidder.model.HttpRequest; import org.prebid.server.bidder.model.HttpResponse; import org.prebid.server.bidder.model.Result; +import org.prebid.server.exception.PreBidException; import org.prebid.server.execution.Timeout; import org.prebid.server.json.JacksonMapper; import org.prebid.server.model.CaseInsensitiveMultiMap; @@ -26,6 +29,8 @@ import org.prebid.server.vertx.http.HttpClient; import org.prebid.server.vertx.http.model.HttpClientResponse; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -35,6 +40,7 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.zip.GZIPOutputStream; /** * Implements HTTP communication functionality common for {@link Bidder}'s. @@ -169,12 +175,45 @@ private Future> doRequest(HttpRequest httpRequest, Timeout ti return failResponse(new TimeoutException("Timeout has been exceeded"), httpRequest); } - return httpClient.request(httpRequest.getMethod(), httpRequest.getUri(), httpRequest.getHeaders(), - httpRequest.getBody(), remainingTimeout) + return createRequest(httpRequest, remainingTimeout) .compose(response -> processResponse(response, httpRequest)) .recover(exception -> failResponse(exception, httpRequest)); } + private Future createRequest(HttpRequest httpRequest, long remainingTimeout) { + final byte[] requestBody = httpRequest.getBody(); + final MultiMap requestHeaders = httpRequest.getHeaders(); + + final byte[] preparedBody = compressIfRequired(requestBody, requestHeaders); + + return httpClient.request(httpRequest.getMethod(), + httpRequest.getUri(), + requestHeaders, + preparedBody, + remainingTimeout); + } + + private static byte[] compressIfRequired(byte[] existingBody, MultiMap headers) { + final String contentEncodingHeader = headers.get(HttpUtil.CONTENT_ENCODING_HEADER); + if (Objects.equals(contentEncodingHeader, HttpHeaderValues.GZIP.toString())) { + return gzip(existingBody); + } + + return existingBody; + } + + private static byte[] gzip(byte[] value) { + try (ByteArrayOutputStream obj = new ByteArrayOutputStream(); GZIPOutputStream gzip = new GZIPOutputStream( + obj)) { + gzip.write(value); + gzip.finish(); + + return obj.toByteArray(); + } catch (IOException e) { + throw new PreBidException(String.format("Failed to compress request : %s", e.getMessage())); + } + } + /** * Produces {@link Future} with {@link HttpCall} containing request and error description. */ diff --git a/src/main/java/org/prebid/server/bidder/adagio/AdagioBidder.java b/src/main/java/org/prebid/server/bidder/adagio/AdagioBidder.java index 7f11c4aa7a0..795e90aa1d0 100644 --- a/src/main/java/org/prebid/server/bidder/adagio/AdagioBidder.java +++ b/src/main/java/org/prebid/server/bidder/adagio/AdagioBidder.java @@ -7,6 +7,7 @@ import com.iab.openrtb.response.Bid; import com.iab.openrtb.response.BidResponse; import com.iab.openrtb.response.SeatBid; +import io.netty.handler.codec.http.HttpHeaderValues; import io.vertx.core.MultiMap; import io.vertx.core.http.HttpMethod; import org.apache.commons.collections4.CollectionUtils; @@ -51,7 +52,8 @@ public Result>> makeHttpRequests(BidRequest request } private static MultiMap resolveHeaders(Device device) { - final MultiMap headers = HttpUtil.headers(); + final MultiMap headers = HttpUtil.headers() + .add(HttpUtil.CONTENT_ENCODING_HEADER, HttpHeaderValues.GZIP); if (device != null) { HttpUtil.addHeaderIfValueIsNotEmpty(headers, HttpUtil.X_FORWARDED_FOR_HEADER, device.getIpv6()); diff --git a/src/main/java/org/prebid/server/bidder/gamma/GammaBidder.java b/src/main/java/org/prebid/server/bidder/gamma/GammaBidder.java index 9148599dcd5..d6e9dcf20b3 100644 --- a/src/main/java/org/prebid/server/bidder/gamma/GammaBidder.java +++ b/src/main/java/org/prebid/server/bidder/gamma/GammaBidder.java @@ -181,8 +181,7 @@ private MultiMap makeHeaders(Device device) { .set(HttpUtil.X_OPENRTB_VERSION_HEADER, "2.5") .set(HttpUtil.ACCEPT_HEADER, "*/*") .set(HttpUtil.CACHE_CONTROL_HEADER, "no-cache") - .set(HttpUtil.CONNECTION_HEADER, "keep-alive") - .set(HttpUtil.ACCEPT_ENCODING_HEADER, "gzip, deflate"); + .set(HttpUtil.CONNECTION_HEADER, "keep-alive"); if (device != null) { HttpUtil.addHeaderIfValueIsNotEmpty(headers, HttpUtil.USER_AGENT_HEADER, device.getUa()); diff --git a/src/main/java/org/prebid/server/util/HttpUtil.java b/src/main/java/org/prebid/server/util/HttpUtil.java index 21bbfdc6842..4f9a8f5081f 100644 --- a/src/main/java/org/prebid/server/util/HttpUtil.java +++ b/src/main/java/org/prebid/server/util/HttpUtil.java @@ -67,6 +67,7 @@ public final class HttpUtil { public static final CharSequence LOCATION_HEADER = HttpHeaders.createOptimized("Location"); public static final CharSequence CONNECTION_HEADER = HttpHeaders.createOptimized("Connection"); public static final CharSequence ACCEPT_ENCODING_HEADER = HttpHeaders.createOptimized("Accept-Encoding"); + public static final CharSequence CONTENT_ENCODING_HEADER = HttpHeaders.createOptimized("Content-Encoding"); public static final CharSequence X_OPENRTB_VERSION_HEADER = HttpHeaders.createOptimized("x-openrtb-version"); public static final CharSequence X_PREBID_HEADER = HttpHeaders.createOptimized("x-prebid"); private static final Set SENSITIVE_HEADERS = new HashSet<>(Arrays.asList(AUTHORIZATION_HEADER.toString())); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 7ce457621f1..b66c0973ecc 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -86,7 +86,7 @@ http-client: idle-timeout-ms: 0 pool-cleaner-period-ms: 1000 connect-timeout-ms: 2500 - use-compression: false + use-compression: true max-redirects: 0 ssl: false jks-path: diff --git a/src/test/java/org/prebid/server/bidder/HttpBidderRequesterTest.java b/src/test/java/org/prebid/server/bidder/HttpBidderRequesterTest.java index 85572959879..b57f65c3dc1 100644 --- a/src/test/java/org/prebid/server/bidder/HttpBidderRequesterTest.java +++ b/src/test/java/org/prebid/server/bidder/HttpBidderRequesterTest.java @@ -6,6 +6,7 @@ import com.iab.openrtb.request.Imp; import com.iab.openrtb.request.Pmp; import com.iab.openrtb.response.Bid; +import io.netty.handler.codec.http.HttpHeaderValues; import io.vertx.core.Future; import io.vertx.core.MultiMap; import io.vertx.core.Promise; @@ -36,6 +37,7 @@ import org.prebid.server.execution.TimeoutFactory; import org.prebid.server.model.CaseInsensitiveMultiMap; import org.prebid.server.proto.openrtb.ext.response.ExtHttpCall; +import org.prebid.server.util.HttpUtil; import org.prebid.server.vertx.http.HttpClient; import org.prebid.server.vertx.http.model.HttpClientResponse; @@ -187,7 +189,7 @@ public void shouldPassStoredResponseToBidderMakeBidsMethodAndReturnSeatBids() { @Test public void shouldMakeRequestToBidderWhenStoredResponseDefinedButBidderCreatesMoreThanOneRequest() { // given - givenHttpClientReturnsResponse(200, null); + givenHttpClientResponse(200, null); final MultiMap headers = new CaseInsensitiveHeaders(); headers.add("header1", "value1"); headers.add("header2", "value2"); @@ -218,7 +220,7 @@ public void shouldMakeRequestToBidderWhenStoredResponseDefinedButBidderCreatesMo @Test public void shouldSendPopulatedGetRequestWithoutBody() { // given - givenHttpClientReturnsResponse(200, null); + givenHttpClientResponse(200, null); given(bidder.makeHttpRequests(any())).willReturn(Result.of(singletonList( HttpRequest.builder() @@ -239,7 +241,7 @@ public void shouldSendPopulatedGetRequestWithoutBody() { @Test public void shouldSendMultipleRequests() throws JsonProcessingException { // given - givenHttpClientReturnsResponse(200, null); + givenHttpClientResponse(200, null); final BidRequest bidRequest = givenBidRequest(identity()); given(bidder.makeHttpRequests(any())).willReturn(Result.of(asList( @@ -278,7 +280,7 @@ public void shouldReturnBidsCreatedByBidder() { .build()), emptyList())); - givenHttpClientReturnsResponse(200, "responseBody"); + givenHttpClientResponse(200, "responseBody"); final List bids = asList(BidderBid.of(null, null, null), BidderBid.of(null, null, null)); given(bidder.makeBids(any(), any())).willReturn(Result.of(bids, emptyList())); @@ -294,6 +296,34 @@ public void shouldReturnBidsCreatedByBidder() { assertThat(bidderSeatBid.getBids()).containsOnlyElementsOf(bids); } + @Test + public void shouldCompressRequestBodyIfContentEncodingHeaderIsGzip() { + // given + final MultiMap headers = new CaseInsensitiveHeaders() + .add(HttpUtil.CONTENT_ENCODING_HEADER, HttpHeaderValues.GZIP); + given(bidder.makeHttpRequests(any())).willReturn(Result.of(singletonList( + HttpRequest.builder() + .method(HttpMethod.POST) + .uri(EMPTY) + .body(EMPTY_BYTE_BODY) + .headers(new CaseInsensitiveHeaders()) + .build()), + emptyList())); + + given(requestEnricher.enrichHeaders(any(), any(), any())).willReturn(headers); + givenHttpClientResponse(200, "responseBody"); + final BidderRequest bidderRequest = BidderRequest.of("bidder", null, BidRequest.builder().build()); + + // when + httpBidderRequester.requestBids(bidder, bidderRequest, timeout, CaseInsensitiveMultiMap.empty(), false) + .result(); + + // then + final ArgumentCaptor actualRequestBody = ArgumentCaptor.forClass(byte[].class); + verify(httpClient).request(any(), anyString(), any(), actualRequestBody.capture(), anyLong()); + assertThat(actualRequestBody.getValue()).isNotSameAs(EMPTY_BYTE_BODY); + } + @Test public void shouldNotWaitForResponsesWhenAllDealsIsGathered() throws JsonProcessingException { // given @@ -413,7 +443,7 @@ public void shouldFinishWhenAllDealRequestsAreFinishedAndNoDealsProvided() { .build()), emptyList())); - givenHttpClientReturnsResponse(200, "responseBody"); + givenHttpClientResponse(200, "responseBody"); final BidderBid bidderBid = BidderBid.of(Bid.builder().dealid("deal2").build(), null, null); given(bidder.makeBids(any(), any())).willReturn(Result.of(singletonList(bidderBid), emptyList())); @@ -766,7 +796,7 @@ public void shouldNotMakeBidsIfResponseStatusIs204() { .build()), emptyList())); - givenHttpClientReturnsResponse(204, EMPTY); + givenHttpClientResponse(204, EMPTY); final BidderRequest bidderRequest = BidderRequest.of("bidder", null, BidRequest.builder().test(1).build()); @@ -803,7 +833,7 @@ private static Imp impWithDeal(String dealId) { .build(); } - private void givenHttpClientReturnsResponse(int statusCode, String response) { + private void givenHttpClientResponse(int statusCode, String response) { given(httpClient.request(any(), anyString(), any(), (byte[]) any(), anyLong())) .willReturn(Future.succeededFuture(HttpClientResponse.of(statusCode, null, response))); } diff --git a/src/test/java/org/prebid/server/bidder/adagio/AdagioBidderTest.java b/src/test/java/org/prebid/server/bidder/adagio/AdagioBidderTest.java index 35b45107fef..c249927594a 100644 --- a/src/test/java/org/prebid/server/bidder/adagio/AdagioBidderTest.java +++ b/src/test/java/org/prebid/server/bidder/adagio/AdagioBidderTest.java @@ -71,6 +71,7 @@ public void makeHttpRequestsShouldCorrectlyAddAllHeaders() { .containsExactlyInAnyOrder( tuple(HttpUtil.CONTENT_TYPE_HEADER.toString(), HttpUtil.APPLICATION_JSON_CONTENT_TYPE), tuple(HttpUtil.ACCEPT_HEADER.toString(), HttpHeaderValues.APPLICATION_JSON.toString()), + tuple(HttpUtil.CONTENT_ENCODING_HEADER.toString(), HttpHeaderValues.GZIP.toString()), tuple(HttpUtil.X_FORWARDED_FOR_HEADER.toString(), "someIp")); } diff --git a/src/test/java/org/prebid/server/bidder/gamma/GammaBidderTest.java b/src/test/java/org/prebid/server/bidder/gamma/GammaBidderTest.java index d6069fcd6fe..f04c3d89e1b 100644 --- a/src/test/java/org/prebid/server/bidder/gamma/GammaBidderTest.java +++ b/src/test/java/org/prebid/server/bidder/gamma/GammaBidderTest.java @@ -152,7 +152,6 @@ public void makeHttpRequestsShouldFillMethodAndUrlAndExpectedHeaders() { tuple("Accept", "*/*"), tuple("Connection", "keep-alive"), tuple("Cache-Control", "no-cache"), - tuple("Accept-Encoding", "gzip, deflate"), tuple("x-openrtb-version", "2.5")); } @@ -197,7 +196,6 @@ public void makeHttpRequestsShouldFillMethodAndUrlAndExpectedHeadersWhenDeviceAn tuple("Accept", "*/*"), tuple("Connection", "keep-alive"), tuple("Cache-Control", "no-cache"), - tuple("Accept-Encoding", "gzip, deflate"), tuple("User-Agent", "userAgent"), tuple("x-openrtb-version", "2.5"), tuple("X-Forwarded-For", "123.123.123.12"),