From 6eae64464cfbc75c47ac518fa2e1cc9c7388445d Mon Sep 17 00:00:00 2001 From: Serhii Nahornyi Date: Mon, 2 Aug 2021 14:01:53 +0300 Subject: [PATCH] Support additional AMP params (#1379) --- docs/endpoints/openrtb2/amp.md | 20 +- .../server/auction/model/ConsentType.java | 9 + .../requestfactory/AmpRequestFactory.java | 77 +++++- .../request/ConsentedProvidersSettings.java | 11 + .../proto/openrtb/ext/request/ExtUser.java | 7 + .../requestfactory/AmpRequestFactoryTest.java | 226 +++++++++++++++++- .../org/prebid/server/it/ApplicationTest.java | 3 + .../it/amp/test-appnexus-bid-request.json | 10 + .../it/amp/test-rubicon-bid-request.json | 7 +- 9 files changed, 350 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/prebid/server/auction/model/ConsentType.java create mode 100644 src/main/java/org/prebid/server/proto/openrtb/ext/request/ConsentedProvidersSettings.java diff --git a/docs/endpoints/openrtb2/amp.md b/docs/endpoints/openrtb2/amp.md index 7e7037c4790..1072c52ee4b 100644 --- a/docs/endpoints/openrtb2/amp.md +++ b/docs/endpoints/openrtb2/amp.md @@ -116,11 +116,14 @@ This endpoint supports the following query parameters: 6. `curl` - the canonical URL of the page 7. `timeout` - the publisher-specified timeout for the RTC callout - A configuration option `amp.default-timeout-ms` may be set to account for estimated latency so that Prebid Server can handle timeouts from adapters and respond to the AMP RTC request before it times out. -8. `us_privacy` - CCPA value for request. -9. `gdpr_consent` - GDPR value for request. +8. `gdpr_consent` - consent string for request. +9. `consent_string` - consent string for request. 10. `debug` - When set to `1`, the responses will contain extra info for debugging. 11. `account` - accountId parameter for Site object. 12. `slot` - tagId parameter for Imp object. +13. `gdpr_applies` - GDPR param for Regs Ext object. +14. `consent_type` - param to define what type of consent_string passed. +15. `attl_consent` - consentedProviders param for User Ext ConsentedProvidersSettings object. For information on how these get from AMP into this endpoint, see [this pull request adding the query params to the Prebid callout](https://github.com/ampproject/amphtml/pull/14155) and [this issue adding support for network-level RTC macros](https://github.com/ampproject/amphtml/issues/12374). @@ -130,11 +133,14 @@ If present, these will override parts of your Stored Request. 1. `ow`, `oh`, `w`, `h`, and/or `ms` will be used to set `request.imp[0].banner.format` if `request.imp[0].banner` is present. 2. `curl` will be used to set `request.site.page` 3. `timeout` will generally be used to set `request.tmax`. However, the Prebid Server host can [configure](../../config.md) their deploy to reduce this timeout for technical reasons. -4. `us_privacy` will be used to set `request.regs.ext.us_privacy` -5. `debug` will be used to set `request.test`, causing the `response.debug` to have extra debugging info in it. -6. `account` - will be used to set `site.publisher.id` parameter for Site object. -7. `slot` - will be used to set `tagId` parameter to overwrite Imp object. -8. `gdpr_consent` will be used to set `user.ext.consent`. +4. `debug` will be used to set `request.test`, causing the `response.debug` to have extra debugging info in it. +5. `account` - will be used to set `site.publisher.id` parameter for Site object. +6. `slot` - will be used to set `tagId` parameter to overwrite Imp object. +7. `gdpr_applies` will be used to set `request.regs.ext.gdpr` +8. `consent_type` will be used to check what should be done with consent string +9. `attl_consent` will be used to set `user.ext.ConsentedProvidersSettings.consented_providers`. +10. `gdpr_consent` will be used to set `request.regs.ext.us_privacy` or `user.ext.consent` +11. `consent_string` will be used to set `request.regs.ext.us_privacy` or `user.ext.consent`. This param has bigger priority then `gdpr_consent`. diff --git a/src/main/java/org/prebid/server/auction/model/ConsentType.java b/src/main/java/org/prebid/server/auction/model/ConsentType.java new file mode 100644 index 00000000000..a35297fb3f7 --- /dev/null +++ b/src/main/java/org/prebid/server/auction/model/ConsentType.java @@ -0,0 +1,9 @@ +package org.prebid.server.auction.model; + +/** + * Describes consent types that can be present in `consent_type` amp query param + */ +public enum ConsentType { + + tcfV1, tcfV2, usPrivacy, unknown; +} diff --git a/src/main/java/org/prebid/server/auction/requestfactory/AmpRequestFactory.java b/src/main/java/org/prebid/server/auction/requestfactory/AmpRequestFactory.java index fc3604ad2fb..038bb8d1f15 100644 --- a/src/main/java/org/prebid/server/auction/requestfactory/AmpRequestFactory.java +++ b/src/main/java/org/prebid/server/auction/requestfactory/AmpRequestFactory.java @@ -26,6 +26,7 @@ import org.prebid.server.auction.StoredRequestProcessor; import org.prebid.server.auction.TimeoutResolver; import org.prebid.server.auction.model.AuctionContext; +import org.prebid.server.auction.model.ConsentType; import org.prebid.server.exception.InvalidRequestException; import org.prebid.server.json.JacksonMapper; import org.prebid.server.metric.MetricName; @@ -33,6 +34,7 @@ import org.prebid.server.model.HttpRequestContext; import org.prebid.server.privacy.ccpa.Ccpa; import org.prebid.server.privacy.gdpr.TcfDefinerService; +import org.prebid.server.proto.openrtb.ext.request.ConsentedProvidersSettings; import org.prebid.server.proto.openrtb.ext.request.ExtMediaTypePriceGranularity; import org.prebid.server.proto.openrtb.ext.request.ExtPriceGranularity; import org.prebid.server.proto.openrtb.ext.request.ExtRegs; @@ -76,6 +78,9 @@ public class AmpRequestFactory { private static final String TIMEOUT_REQUEST_PARAM = "timeout"; private static final String GDPR_CONSENT_PARAM = "gdpr_consent"; private static final String CONSENT_PARAM = "consent_string"; + private static final String GDPR_APPLIES_PARAM = "gdpr_applies"; + private static final String CONSENT_TYPE_PARAM = "consent_type"; + private static final String ATTL_CONSENT_PARAM = "attl_consent"; private static final int NO_LIMIT_SPLIT_MODE = -1; private static final String AMP_CHANNEL = "amp"; @@ -154,14 +159,17 @@ private Future parseBidRequest(HttpRequestContext httpRequest, Aucti return Future.failedFuture(new InvalidRequestException("AMP requests require an AMP tag_id")); } + final ConsentType consentType = consentTypeFromQueryStringParams(httpRequest); final String consentString = consentStringFromQueryStringParams(httpRequest); + final String attlConsent = attlConsentFromQueryStringParams(httpRequest); + final Integer gdpr = gdprFromQueryStringParams(httpRequest); final Integer debug = debugFromQueryStringParam(httpRequest); final Long timeout = timeoutFromQueryString(httpRequest); final BidRequest bidRequest = BidRequest.builder() .site(createSite(httpRequest)) - .user(createUser(consentString)) - .regs(createRegs(consentString)) + .user(createUser(consentType, consentString, attlConsent)) + .regs(createRegs(consentString, consentType, gdpr)) .test(debug) .tmax(timeout) .ext(createExt(httpRequest, tagId, debug)) @@ -186,16 +194,33 @@ private static Site createSite(HttpRequestContext httpRequest) { : null; } - private static User createUser(String consentString) { - return StringUtils.isNotBlank(consentString) && TcfDefinerService.isConsentStringValid(consentString) - ? User.builder().ext(ExtUser.builder().consent(consentString).build()).build() - : null; + private static User createUser(ConsentType consentType, String consentString, String attlConsent) { + final boolean tcfV2ConsentProvided = (StringUtils.isNotBlank(consentString) + && TcfDefinerService.isConsentStringValid(consentString)) + && (consentType == null || consentType == ConsentType.tcfV2); + + if (StringUtils.isNotBlank(attlConsent) || tcfV2ConsentProvided) { + final ExtUser.ExtUserBuilder userExtBuilder = ExtUser.builder(); + if (tcfV2ConsentProvided) { + userExtBuilder.consent(consentString); + } + if (StringUtils.isNotBlank(attlConsent)) { + userExtBuilder.consentedProvidersSettings(ConsentedProvidersSettings.of(attlConsent)); + } + return User.builder().ext(userExtBuilder.build()).build(); + } + + return null; } - private static Regs createRegs(String consentString) { - return StringUtils.isNotBlank(consentString) && Ccpa.isValid(consentString) - ? Regs.of(null, ExtRegs.of(null, consentString)) - : null; + private static Regs createRegs(String consentString, ConsentType consentType, Integer gdpr) { + final boolean ccpaProvided = Ccpa.isValid(consentString) + && (consentType == null || consentType == ConsentType.usPrivacy); + if (ccpaProvided || gdpr != null) { + return Regs.of(null, ExtRegs.of(gdpr, ccpaProvided ? consentString : null)); + } + + return null; } private static ExtRequest createExt(HttpRequestContext httpRequest, String tagId, Integer debug) { @@ -222,6 +247,23 @@ private static String canonicalUrl(HttpRequestContext httpRequest) { } } + private static ConsentType consentTypeFromQueryStringParams(HttpRequestContext httpRequest) { + final String consentTypeParam = httpRequest.getQueryParams().get(CONSENT_TYPE_PARAM); + if (consentTypeParam == null) { + return null; + } + switch (consentTypeParam) { + case "1": + return ConsentType.tcfV1; + case "2": + return ConsentType.tcfV2; + case "3": + return ConsentType.usPrivacy; + default: + return ConsentType.unknown; + } + } + private static String consentStringFromQueryStringParams(HttpRequestContext httpRequest) { final String requestConsentParam = httpRequest.getQueryParams().get(CONSENT_PARAM); final String requestGdprConsentParam = httpRequest.getQueryParams().get(GDPR_CONSENT_PARAM); @@ -229,6 +271,21 @@ private static String consentStringFromQueryStringParams(HttpRequestContext http return ObjectUtils.firstNonNull(requestConsentParam, requestGdprConsentParam); } + private static String attlConsentFromQueryStringParams(HttpRequestContext httpRequest) { + return httpRequest.getQueryParams().get(ATTL_CONSENT_PARAM); + } + + private static Integer gdprFromQueryStringParams(HttpRequestContext httpRequest) { + final String gdprAppliesParam = httpRequest.getQueryParams().get(GDPR_APPLIES_PARAM); + if (StringUtils.equals(gdprAppliesParam, "true")) { + return 1; + } else if (StringUtils.equals(gdprAppliesParam, "false")) { + return 0; + } + + return null; + } + private static Long timeoutFromQueryString(HttpRequestContext httpRequest) { final String timeoutQueryParam = httpRequest.getQueryParams().get(TIMEOUT_REQUEST_PARAM); if (timeoutQueryParam == null) { diff --git a/src/main/java/org/prebid/server/proto/openrtb/ext/request/ConsentedProvidersSettings.java b/src/main/java/org/prebid/server/proto/openrtb/ext/request/ConsentedProvidersSettings.java new file mode 100644 index 00000000000..dc739edc03b --- /dev/null +++ b/src/main/java/org/prebid/server/proto/openrtb/ext/request/ConsentedProvidersSettings.java @@ -0,0 +1,11 @@ +package org.prebid.server.proto.openrtb.ext.request; + +import lombok.AllArgsConstructor; +import lombok.Value; + +@AllArgsConstructor(staticName = "of") +@Value +public class ConsentedProvidersSettings { + + String consentedProviders; +} diff --git a/src/main/java/org/prebid/server/proto/openrtb/ext/request/ExtUser.java b/src/main/java/org/prebid/server/proto/openrtb/ext/request/ExtUser.java index b49e24904bb..6f54f3febf1 100644 --- a/src/main/java/org/prebid/server/proto/openrtb/ext/request/ExtUser.java +++ b/src/main/java/org/prebid/server/proto/openrtb/ext/request/ExtUser.java @@ -1,6 +1,7 @@ package org.prebid.server.proto.openrtb.ext.request; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Builder; @@ -46,6 +47,12 @@ public class ExtUser extends FlexibleExtension { */ JsonNode digitrust; + /** + * Defines the contract for bidrequest.user.ext.ConsentedProvidersSettings + */ + @JsonProperty("ConsentedProvidersSettings") + ConsentedProvidersSettings consentedProvidersSettings; + @JsonIgnore public boolean isEmpty() { return Objects.equals(this, EMPTY); diff --git a/src/test/java/org/prebid/server/auction/requestfactory/AmpRequestFactoryTest.java b/src/test/java/org/prebid/server/auction/requestfactory/AmpRequestFactoryTest.java index 1200188c803..6421e47e1c1 100644 --- a/src/test/java/org/prebid/server/auction/requestfactory/AmpRequestFactoryTest.java +++ b/src/test/java/org/prebid/server/auction/requestfactory/AmpRequestFactoryTest.java @@ -41,6 +41,7 @@ import org.prebid.server.privacy.model.Privacy; import org.prebid.server.privacy.model.PrivacyContext; import org.prebid.server.proto.openrtb.ext.ExtIncludeBrandCategory; +import org.prebid.server.proto.openrtb.ext.request.ConsentedProvidersSettings; import org.prebid.server.proto.openrtb.ext.request.ExtGranularityRange; import org.prebid.server.proto.openrtb.ext.request.ExtPriceGranularity; import org.prebid.server.proto.openrtb.ext.request.ExtRegs; @@ -1142,7 +1143,7 @@ public void shouldReturnBidRequestWithoutUserWhenGdprConsentQueryParamIsBlank() } @Test - public void shouldReturnBidRequestWithUserExtConsentWhenGdprConsentQueryParamIsValid() { + public void shouldReturnBidRequestWithUserExtConsentWhenGdprConsentIsValidAndConsentTypeIsNotPresent() { // given routingContext.queryParams().add("gdpr_consent", "BONV8oqONXwgmADACHENAO7pqzAAppY"); @@ -1158,6 +1159,120 @@ public void shouldReturnBidRequestWithUserExtConsentWhenGdprConsentQueryParamIsV .build()); } + @Test + public void shouldReturnBidRequestWithUserExtConsentWhenGdprConsentIsValidAndConsentTypeIsTCFV2() { + // given + routingContext.queryParams() + .add("gdpr_consent", "BONV8oqONXwgmADACHENAO7pqzAAppY") + .add("consent_type", "2"); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getUser()) + .isEqualTo(User.builder() + .ext(ExtUser.builder().consent("BONV8oqONXwgmADACHENAO7pqzAAppY").build()) + .build()); + } + + @Test + public void shouldReturnBidRequestWithoutUserExtConsentWhenGdprConsentIsValidAndConsentTypeIsTCFV1() { + // given + routingContext.queryParams() + .add("gdpr_consent", "BONV8oqONXwgmADACHENAO7pqzAAppY") + .add("consent_type", "1"); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getUser()).isNull(); + } + + @Test + public void shouldReturnBidRequestWithoutUserExtConsentWhenGdprConsentIsValidAndConsentTypeIsUsPrivacy() { + // given + routingContext.queryParams() + .add("gdpr_consent", "BONV8oqONXwgmADACHENAO7pqzAAppY") + .add("consent_type", "3"); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getUser()).isNull(); + } + + @Test + public void shouldReturnBidRequestWithoutUserExtConsentWhenGdprConsentIsValidAndConsentTypeIsUnknown() { + // given + routingContext.queryParams() + .add("gdpr_consent", "BONV8oqONXwgmADACHENAO7pqzAAppY") + .add("consent_type", "23"); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getUser()).isNull(); + } + + @Test + public void shouldReturnBidRequestWithProvidersSettingsContainsAttlConsentIfParamIsPresent() { + // given + routingContext.queryParams() + .add("attl_consent", "someConsent"); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getUser()) + .isEqualTo(User.builder() + .ext(ExtUser.builder() + .consentedProvidersSettings(ConsentedProvidersSettings.of("someConsent")) + .build()) + .build()); + } + + @Test + public void shouldReturnBidRequestWithoutProvidersSettingsIfAttlConsentIsMissed() { + // given + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getUser()).isNull(); + } + + @Test + public void shouldReturnBidRequestWithoutProvidersSettingsIfAttlConsentIsBlank() { + // given + routingContext.queryParams() + .add("attl_consent", " "); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getUser()).isNull(); + } + @Test public void shouldReturnBidRequestWithoutUserWhenGdprConsentQueryParamIsInvalid() { // given @@ -1265,6 +1380,48 @@ public void shouldReturnBidRequestWithoutRegsExtWhenNoPrivacyPolicyIsExist() { assertThat(result.getRegs()).isNull(); } + @Test + public void shouldReturnBidRequestWithRegsContainsGdprEqualOneIfGdprAppliesIsTrue() { + // given + routingContext.queryParams().add("gdpr_applies", "true"); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getRegs()) + .isEqualTo(Regs.of(null, ExtRegs.of(1, null))); + } + + @Test + public void shouldReturnBidRequestWithRegsContainsGdprEqualZeroIfGdprAppliesIsFalse() { + // given + routingContext.queryParams().add("gdpr_applies", "false"); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getRegs()) + .isEqualTo(Regs.of(null, ExtRegs.of(0, null))); + } + + @Test + public void shouldReturnBidRequestWithoutRegsIfGdprAppliesIsNotPresent() { + // given + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getRegs()).isNull(); + } + @Test public void shouldReturnBidRequestWithRegsExtUsPrivacyWhenGdprConsentQueryParamIsValidUsPrivacyString() { // given @@ -1281,7 +1438,7 @@ public void shouldReturnBidRequestWithRegsExtUsPrivacyWhenGdprConsentQueryParamI } @Test - public void shouldReturnBidRequestWithRegsExtUsPrivacyWhenConsentStringQueryParamIsValid() { + public void shouldReturnBidRequestWithRegsExtUsPrivacyWhenConsentStringIsValidAndConsentTypeIsNotPresent() { // given routingContext.queryParams().add("consent_string", "1Y-N"); @@ -1295,6 +1452,71 @@ public void shouldReturnBidRequestWithRegsExtUsPrivacyWhenConsentStringQueryPara .isEqualTo(Regs.of(null, ExtRegs.of(null, "1Y-N"))); } + @Test + public void shouldReturnBidRequestWithRegsExtUsPrivacyWhenConsentStringIsValidAndConsentTypeIsUsPrivacy() { + // given + routingContext.queryParams() + .add("consent_string", "1Y-N") + .add("consent_type", "3"); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getRegs()) + .isEqualTo(Regs.of(null, ExtRegs.of(null, "1Y-N"))); + } + + @Test + public void shouldReturnBidRequestWithoutRegsExtUsPrivacyWhenConsentStringIsValidAndConsentTypeIsTcfV1() { + // given + routingContext.queryParams() + .add("consent_string", "1Y-N") + .add("consent_type", "1"); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getRegs()).isNull(); + } + + @Test + public void shouldReturnBidRequestWithoutRegsExtUsPrivacyWhenConsentStringIsValidAndConsentTypeIsTcfV2() { + // given + routingContext.queryParams() + .add("consent_string", "1Y-N") + .add("consent_type", "2"); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getRegs()).isNull(); + } + + @Test + public void shouldReturnBidRequestWithoutRegsExtUsPrivacyWhenConsentStringIsValidAndConsentTypeIsUnknown() { + // given + routingContext.queryParams() + .add("consent_string", "1Y-N") + .add("consent_type", "23"); + + givenBidRequest(); + + // when + final BidRequest result = target.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(result.getRegs()).isNull(); + } + @SuppressWarnings("unchecked") @Test public void shouldReturnBidRequestWithCreatedExtPrebidAmpData() { diff --git a/src/test/java/org/prebid/server/it/ApplicationTest.java b/src/test/java/org/prebid/server/it/ApplicationTest.java index 6c2ec7a931c..2322b7f6494 100644 --- a/src/test/java/org/prebid/server/it/ApplicationTest.java +++ b/src/test/java/org/prebid/server/it/ApplicationTest.java @@ -278,6 +278,9 @@ public void ampShouldReturnTargeting() throws IOException, JSONException { + "&targeting=%7B%22gam-key1%22%3A%22val1%22%2C%22gam-key2%22%3A%22val2%22%7D" + "&curl=https%3A%2F%2Fgoogle.com" + "&account=accountId" + + "&attl_consent=someConsent" + + "&gdpr_applies=false" + + "&consent_type=3" + "&consent_string=1YNN"); // then diff --git a/src/test/resources/org/prebid/server/it/amp/test-appnexus-bid-request.json b/src/test/resources/org/prebid/server/it/amp/test-appnexus-bid-request.json index 733f5de352c..9ed6f515e51 100644 --- a/src/test/resources/org/prebid/server/it/amp/test-appnexus-bid-request.json +++ b/src/test/resources/org/prebid/server/it/amp/test-appnexus-bid-request.json @@ -40,6 +40,13 @@ "ua": "userAgent", "ip": "193.168.244.1" }, + "user": { + "ext": { + "ConsentedProvidersSettings": { + "consented_providers": "someConsent" + } + } + }, "at": 1, "tmax": 5000, "cur": [ @@ -94,7 +101,10 @@ "amp": { "data": { "curl": "https%3A%2F%2Fgoogle.com", + "consent_type": "3", "consent_string": "1YNN", + "gdpr_applies": "false", + "attl_consent": "someConsent", "ow": "980", "oh": "120", "tag_id": "test-amp-stored-request", diff --git a/src/test/resources/org/prebid/server/it/amp/test-rubicon-bid-request.json b/src/test/resources/org/prebid/server/it/amp/test-rubicon-bid-request.json index cda0735236f..d12c3db99ce 100644 --- a/src/test/resources/org/prebid/server/it/amp/test-rubicon-bid-request.json +++ b/src/test/resources/org/prebid/server/it/amp/test-rubicon-bid-request.json @@ -69,7 +69,12 @@ } }, "user": { - "buyeruid": "J5VLCWQP-26-CWFT" + "buyeruid": "J5VLCWQP-26-CWFT", + "ext": { + "ConsentedProvidersSettings": { + "consented_providers": "someConsent" + } + } }, "at": 1, "tmax": 5000,