diff --git a/docs/application-settings.md b/docs/application-settings.md index a12b46d3ad4..d15b8ad5c88 100644 --- a/docs/application-settings.md +++ b/docs/application-settings.md @@ -99,7 +99,13 @@ Here's an example YAML file containing account-specific settings: events: enabled: true privacy: - enforce-ccpa: true + ccpa: + enabled: true + integration-enabled: + video: true + web: true + app: true + amp: true gdpr: enabled: true integration-enabled: @@ -252,7 +258,15 @@ example: } }, "privacy": { - "enforce-ccpa": true, + "ccpa": { + "enabled": true, + "integration-enabled": { + "web": true, + "amp": false, + "app": true, + "video": false + } + }, "gdpr": { "enabled": true, "integration-enabled": { @@ -393,7 +407,7 @@ several tables. MySQL and Postgres provides necessary functions allowing to proj expected JSON format. Let's assume following table schema for example: -```sql +```mysql-sql 'CREATE TABLE `accounts_account` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `uuid` varchar(40) NOT NULL, @@ -419,7 +433,7 @@ ENGINE=InnoDB DEFAULT CHARSET=utf8' The following Mysql SQL query could be used to construct a JSON document of required shape on the fly: -```sql +```mysql-sql SELECT JSON_MERGE_PATCH(config, JSON_OBJECT( 'id', uuid, diff --git a/src/main/java/org/prebid/server/auction/PrivacyEnforcementService.java b/src/main/java/org/prebid/server/auction/PrivacyEnforcementService.java index 215f554693b..22b07524e66 100644 --- a/src/main/java/org/prebid/server/auction/PrivacyEnforcementService.java +++ b/src/main/java/org/prebid/server/auction/PrivacyEnforcementService.java @@ -53,7 +53,7 @@ import java.util.stream.Collectors; /** - * Service provides masking for OpenRTB user sensitive information. + * Service provides masking for OpenRTB client sensitive information. */ public class PrivacyEnforcementService { @@ -135,8 +135,9 @@ private String resolveIpAddress(Device device, Privacy privacy) { return null; } - public Future contextFromSetuidRequest( - HttpServerRequest httpRequest, Account account, Timeout timeout) { + public Future contextFromSetuidRequest(HttpServerRequest httpRequest, + Account account, + Timeout timeout) { final Privacy privacy = privacyExtractor.validPrivacyFromSetuidRequest(httpRequest); final String ipAddress = resolveIpFromRequest(httpRequest); @@ -149,8 +150,10 @@ public Future contextFromSetuidRequest( .map(tcfContext -> PrivacyContext.of(privacy, tcfContext)); } - public Future contextFromCookieSyncRequest( - CookieSyncRequest cookieSyncRequest, HttpServerRequest httpRequest, Account account, Timeout timeout) { + public Future contextFromCookieSyncRequest(CookieSyncRequest cookieSyncRequest, + HttpServerRequest httpRequest, + Account account, + Timeout timeout) { final Privacy privacy = privacyExtractor.validPrivacyFrom(cookieSyncRequest); final String ipAddress = resolveIpFromRequest(httpRequest); @@ -220,6 +223,7 @@ Future> mask(AuctionContext auctionContext, public Future> resultForVendorIds(Set vendorIds, TcfContext tcfContext) { + return tcfDefinerService.resultForVendorIds(vendorIds, tcfContext) .map(TcfResponse::getActions); } @@ -233,16 +237,13 @@ private Map ccpaResult(BidRequest bidRequest, Privacy privacy, MetricName requestType) { - if (isCcpaEnforced(privacy.getCcpa(), account, requestType)) { - return maskCcpa(extractCcpaEnforcedBidders(bidders, bidRequest, aliases), device, bidderToUser); - } - - return Collections.emptyMap(); + return isCcpaEnforced(privacy.getCcpa(), account, requestType) + ? maskCcpa(extractCcpaEnforcedBidders(bidders, bidRequest, aliases), device, bidderToUser) + : Collections.emptyMap(); } public boolean isCcpaEnforced(Ccpa ccpa, Account account) { final boolean shouldEnforceCcpa = isCcpaEnabled(account); - return shouldEnforceCcpa && ccpa.isEnforced(); } @@ -255,20 +256,19 @@ private Boolean isCcpaEnabled(Account account) { final AccountPrivacyConfig accountPrivacyConfig = account.getPrivacy(); final AccountCcpaConfig accountCcpaConfig = accountPrivacyConfig != null ? accountPrivacyConfig.getCcpa() : null; - final Boolean accountEnforceCcpa = accountPrivacyConfig != null ? accountPrivacyConfig.getEnforceCcpa() : null; final Boolean accountCcpaEnabled = accountCcpaConfig != null ? accountCcpaConfig.getEnabled() : null; - return ObjectUtils.firstNonNull(accountCcpaEnabled, accountEnforceCcpa, ccpaEnforce); + return ObjectUtils.defaultIfNull(accountCcpaEnabled, ccpaEnforce); } private boolean isCcpaEnabled(Account account, MetricName requestType) { final AccountPrivacyConfig accountPrivacyConfig = account.getPrivacy(); final AccountCcpaConfig accountCcpaConfig = accountPrivacyConfig != null ? accountPrivacyConfig.getCcpa() : null; - final Boolean accountEnforceCcpa = accountPrivacyConfig != null ? accountPrivacyConfig.getEnforceCcpa() : null; + final Boolean accountCcpaEnabled = accountCcpaConfig != null ? accountCcpaConfig.getEnabled() : null; if (requestType == null) { - return ObjectUtils.firstNonNull(accountCcpaEnabled, accountEnforceCcpa, ccpaEnforce); + return ObjectUtils.defaultIfNull(accountCcpaEnabled, ccpaEnforce); } final EnabledForRequestType enabledForRequestType = accountCcpaConfig != null @@ -278,11 +278,12 @@ private boolean isCcpaEnabled(Account account, MetricName requestType) { final Boolean enabledForType = enabledForRequestType != null ? enabledForRequestType.isEnabledFor(requestType) : null; - return ObjectUtils.firstNonNull(enabledForType, accountCcpaEnabled, accountEnforceCcpa, ccpaEnforce); + return ObjectUtils.firstNonNull(enabledForType, accountCcpaEnabled, ccpaEnforce); } - private Map maskCcpa( - Set biddersToMask, Device device, Map bidderToUser) { + private Map maskCcpa(Set biddersToMask, + Device device, + Map bidderToUser) { return biddersToMask.stream() .collect(Collectors.toMap(Function.identity(), @@ -306,17 +307,18 @@ private User maskCcpaUser(User user) { } private Device maskCcpaDevice(Device device) { - return device != null - ? device.toBuilder() - .ip(ipAddressHelper.maskIpv4(device.getIp())) - .ipv6(ipAddressHelper.anonymizeIpv6(device.getIpv6())) - .geo(maskGeoDefault(device.getGeo())) - .ifa(null) - .macsha1(null).macmd5(null) - .dpidsha1(null).dpidmd5(null) - .didsha1(null).didmd5(null) - .build() - : null; + if (device != null) { + return device.toBuilder() + .ip(ipAddressHelper.maskIpv4(device.getIp())) + .ipv6(ipAddressHelper.anonymizeIpv6(device.getIpv6())) + .geo(maskGeoDefault(device.getGeo())) + .ifa(null) + .macsha1(null).macmd5(null) + .dpidsha1(null).dpidmd5(null) + .didsha1(null).didmd5(null) + .build(); + } + return null; } private static boolean isCoppaMaskingRequired(Privacy privacy) { @@ -350,17 +352,18 @@ private User maskCoppaUser(User user) { } private Device maskCoppaDevice(Device device) { - return device != null - ? device.toBuilder() - .ip(ipAddressHelper.maskIpv4(device.getIp())) - .ipv6(ipAddressHelper.anonymizeIpv6(device.getIpv6())) - .geo(maskGeoForCoppa(device.getGeo())) - .ifa(null) - .macsha1(null).macmd5(null) - .dpidsha1(null).dpidmd5(null) - .didsha1(null).didmd5(null) - .build() - : null; + if (device != null) { + return device.toBuilder() + .ip(ipAddressHelper.maskIpv4(device.getIp())) + .ipv6(ipAddressHelper.anonymizeIpv6(device.getIpv6())) + .geo(maskGeoForCoppa(device.getGeo())) + .ifa(null) + .macsha1(null).macmd5(null) + .dpidsha1(null).dpidmd5(null) + .didsha1(null).didmd5(null) + .build(); + } + return null; } /** @@ -377,8 +380,10 @@ private static Geo maskGeoForCoppa(Geo geo) { * Returns {@link Future <{@link Map}<{@link String}, {@link PrivacyEnforcementAction}>>}, * where bidder names mapped to actions for GDPR masking for pbs server. */ - private Future> getBidderToEnforcementAction( - TcfContext tcfContext, Set bidders, BidderAliases aliases, Account account) { + private Future> getBidderToEnforcementAction(TcfContext tcfContext, + Set bidders, + BidderAliases aliases, + Account account) { return tcfDefinerService.resultForBidderNames( Collections.unmodifiableSet(bidders), @@ -388,8 +393,10 @@ private Future> getBidderToEnforcementActi .map(tcfResponse -> mapTcfResponseToEachBidder(tcfResponse, bidders)); } - private Set extractCcpaEnforcedBidders(List bidders, BidRequest bidRequest, BidderAliases - aliases) { + private Set extractCcpaEnforcedBidders(List bidders, + BidRequest bidRequest, + BidderAliases aliases) { + final Set ccpaEnforcedBidders = new HashSet<>(bidders); final ExtRequest extBidRequest = bidRequest.getExt(); @@ -414,7 +421,8 @@ private static Map mapTcfResponseToEachBidder( Set bidders) { final Map bidderNameToAction = tcfResponse.getActions(); - return bidders.stream().collect(Collectors.toMap(Function.identity(), bidderNameToAction::get)); + return bidders.stream() + .collect(Collectors.toMap(Function.identity(), bidderNameToAction::get)); } private void updateCcpaMetrics(Ccpa ccpa) { @@ -476,7 +484,7 @@ private static boolean shouldMaskUser(User user) { return true; } final ExtUser extUser = user.getExt(); - return extUser != null && (CollectionUtils.isNotEmpty(extUser.getEids())); + return extUser != null && CollectionUtils.isNotEmpty(extUser.getEids()); } /** @@ -497,6 +505,7 @@ private List getBidderToPrivacyResult( Device device) { final boolean isLmtEnabled = lmtEnforce && isLmtEnabled(device); + return bidderToUser.entrySet().stream() .filter(entry -> bidders.contains(entry.getKey())) .map(bidderUserEntry -> createBidderPrivacyResult( @@ -599,12 +608,13 @@ private Device maskTcfDevice(Device device, boolean maskIp, boolean maskGeo, boo * Returns masked for GDPR {@link Geo} by rounding lon and lat properties. */ private static Geo maskGeoDefault(Geo geo) { - return geo != null - ? geo.toBuilder() - .lat(maskGeoCoordinate(geo.getLat())) - .lon(maskGeoCoordinate(geo.getLon())) - .build() - : null; + if (geo != null) { + return geo.toBuilder() + .lat(maskGeoCoordinate(geo.getLat())) + .lon(maskGeoCoordinate(geo.getLon())) + .build(); + } + return null; } /** @@ -651,7 +661,6 @@ private static List merge( private static AccountGdprConfig accountGdprConfig(Account account) { final AccountPrivacyConfig privacyConfig = account.getPrivacy(); - return privacyConfig != null ? privacyConfig.getGdpr() : null; } } diff --git a/src/main/java/org/prebid/server/settings/model/AccountPrivacyConfig.java b/src/main/java/org/prebid/server/settings/model/AccountPrivacyConfig.java index 3b41cdf53b6..9e745512647 100644 --- a/src/main/java/org/prebid/server/settings/model/AccountPrivacyConfig.java +++ b/src/main/java/org/prebid/server/settings/model/AccountPrivacyConfig.java @@ -1,14 +1,10 @@ package org.prebid.server.settings.model; -import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Value; @Value(staticConstructor = "of") public class AccountPrivacyConfig { - @JsonProperty("enforce-ccpa") - Boolean enforceCcpa; - AccountGdprConfig gdpr; AccountCcpaConfig ccpa; diff --git a/src/test/java/org/prebid/server/auction/PrivacyEnforcementServiceTest.java b/src/test/java/org/prebid/server/auction/PrivacyEnforcementServiceTest.java index 5d25145cde2..f0bc5e84fc9 100644 --- a/src/test/java/org/prebid/server/auction/PrivacyEnforcementServiceTest.java +++ b/src/test/java/org/prebid/server/auction/PrivacyEnforcementServiceTest.java @@ -23,6 +23,7 @@ import org.prebid.server.auction.model.BidderPrivacyResult; import org.prebid.server.auction.model.IpAddress; import org.prebid.server.bidder.BidderCatalog; +import org.prebid.server.bidder.BidderInfo; import org.prebid.server.exception.InvalidRequestException; import org.prebid.server.execution.Timeout; import org.prebid.server.execution.TimeoutFactory; @@ -47,7 +48,6 @@ import org.prebid.server.proto.openrtb.ext.request.ExtUserEid; import org.prebid.server.proto.openrtb.ext.request.ExtUserPrebid; import org.prebid.server.proto.request.CookieSyncRequest; -import org.prebid.server.bidder.BidderInfo; import org.prebid.server.settings.model.Account; import org.prebid.server.settings.model.AccountCcpaConfig; import org.prebid.server.settings.model.AccountPrivacyConfig; @@ -156,8 +156,8 @@ public void contextFromBidRequestShouldReturnTcfContextForCoppa() { final Future privacyContext = privacyEnforcementService.contextFromBidRequest(auctionContext); // then - FutureAssertion.assertThat(privacyContext).succeededWith( - PrivacyContext.of(Privacy.of(EMPTY, EMPTY, Ccpa.EMPTY, 1), tcfContext)); + FutureAssertion.assertThat(privacyContext) + .succeededWith(PrivacyContext.of(Privacy.of(EMPTY, EMPTY, Ccpa.EMPTY, 1), tcfContext)); } @Test @@ -477,7 +477,6 @@ public void shouldMaskForCcpaWhenAccountHasCppaConfigEnabledForRequestType() { final AuctionContext context = AuctionContext.builder() .account(Account.builder() .privacy(AccountPrivacyConfig.of( - null, null, AccountCcpaConfig.builder() .enabledForRequestType(EnabledForRequestType.of(false, false, true, false)) @@ -529,7 +528,7 @@ public void shouldMaskForCcpaWhenAccountHasCppaEnforcedTrue() { final AuctionContext context = AuctionContext.builder() .account(Account.builder() - .privacy(AccountPrivacyConfig.of(true, null, null)) + .privacy(AccountPrivacyConfig.of(null, AccountCcpaConfig.builder().enabled(true).build())) .build()) .requestTypeMetric(MetricName.openrtb2app) .bidRequest(bidRequest) @@ -578,7 +577,6 @@ public void shouldMaskForCcpaWhenAccountHasCcpaConfigEnabled() { final AuctionContext context = AuctionContext.builder() .account(Account.builder() .privacy(AccountPrivacyConfig.of( - null, null, AccountCcpaConfig.builder().enabled(true).build())) .build()) @@ -777,8 +775,7 @@ public void shouldNotMaskForTcfWhenTcfServiceAllowAllAndDeviceLmtIsOneAndLmtIsNo .build(); assertThat(result).containsOnly(expectedBidderPrivacy); - verify(tcfDefinerService) - .resultForBidderNames(eq(singleton(BIDDER_NAME)), any(), any(), any()); + verify(tcfDefinerService).resultForBidderNames(eq(singleton(BIDDER_NAME)), any(), any(), any()); } @Test @@ -1466,7 +1463,7 @@ public void isCcpaEnforcedShouldReturnFalseWhenEnforcedPropertyIsTrueInConfigura final Ccpa ccpa = Ccpa.of("1YYY"); final Account account = Account.builder() - .privacy(AccountPrivacyConfig.of(false, null, null)) + .privacy(AccountPrivacyConfig.of(null, AccountCcpaConfig.builder().enabled(false).build())) .build(); // when and then @@ -1497,7 +1494,6 @@ public void isCcpaEnforcedShouldReturnFalseWhenAccountCcpaConfigHasEnabledTrue() final Ccpa ccpa = Ccpa.of("1YYY"); final Account account = Account.builder() .privacy(AccountPrivacyConfig.of( - null, null, AccountCcpaConfig.builder().enabled(true).build())) .build(); diff --git a/src/test/java/org/prebid/server/handler/CookieSyncHandlerTest.java b/src/test/java/org/prebid/server/handler/CookieSyncHandlerTest.java index 3bc380ce940..108f7ae96e7 100644 --- a/src/test/java/org/prebid/server/handler/CookieSyncHandlerTest.java +++ b/src/test/java/org/prebid/server/handler/CookieSyncHandlerTest.java @@ -19,6 +19,7 @@ import org.prebid.server.analytics.model.CookieSyncEvent; import org.prebid.server.auction.PrivacyEnforcementService; import org.prebid.server.bidder.BidderCatalog; +import org.prebid.server.bidder.BidderInfo; import org.prebid.server.bidder.Usersyncer; import org.prebid.server.cookie.UidsCookie; import org.prebid.server.cookie.UidsCookieService; @@ -34,7 +35,6 @@ import org.prebid.server.privacy.model.Privacy; import org.prebid.server.privacy.model.PrivacyContext; import org.prebid.server.proto.request.CookieSyncRequest; -import org.prebid.server.bidder.BidderInfo; import org.prebid.server.proto.response.BidderUsersyncStatus; import org.prebid.server.proto.response.CookieSyncResponse; import org.prebid.server.proto.response.UsersyncInfo; @@ -295,7 +295,7 @@ public void shouldPassAccountToPrivacyEnforcementServiceWhenAccountIsFound() { final AccountGdprConfig accountGdprConfig = AccountGdprConfig.builder() .enabledForRequestType(EnabledForRequestType.of(true, true, true, true)).build(); final Account account = Account.builder() - .privacy(AccountPrivacyConfig.of(null, accountGdprConfig, null)) + .privacy(AccountPrivacyConfig.of(accountGdprConfig, null)) .build(); given(applicationSettings.getAccountById(any(), any())).willReturn(Future.succeededFuture(account)); @@ -1040,8 +1040,7 @@ public void shouldRespondWithOriginalUsersyncInfoIfNoHostCookieInRequest() throw final CookieSyncResponse cookieSyncResponse = captureCookieSyncResponse(); assertThat(cookieSyncResponse.getBidderStatus()) .extracting(BidderUsersyncStatus::getUsersync) - .containsOnly( - UsersyncInfo.of("http://rubiconexample.com", "redirect", false)); + .containsOnly(UsersyncInfo.of("http://rubiconexample.com", "redirect", false)); } @Test diff --git a/src/test/java/org/prebid/server/handler/SetuidHandlerTest.java b/src/test/java/org/prebid/server/handler/SetuidHandlerTest.java index 054e66df774..31b280608b0 100644 --- a/src/test/java/org/prebid/server/handler/SetuidHandlerTest.java +++ b/src/test/java/org/prebid/server/handler/SetuidHandlerTest.java @@ -318,7 +318,7 @@ public void shouldPassAccountToPrivacyEnforcementServiceWhenAccountIsFound() { .enabledForRequestType(EnabledForRequestType.of(true, true, true, true)) .build(); final Account account = Account.builder() - .privacy(AccountPrivacyConfig.of(null, accountGdprConfig, null)) + .privacy(AccountPrivacyConfig.of(accountGdprConfig, null)) .build(); final Future accountFuture = Future.succeededFuture(account); given(applicationSettings.getAccountById(any(), any())).willReturn(accountFuture); diff --git a/src/test/java/org/prebid/server/settings/EnrichingApplicationSettingsTest.java b/src/test/java/org/prebid/server/settings/EnrichingApplicationSettingsTest.java index 00b8938378e..9931dfc5220 100644 --- a/src/test/java/org/prebid/server/settings/EnrichingApplicationSettingsTest.java +++ b/src/test/java/org/prebid/server/settings/EnrichingApplicationSettingsTest.java @@ -91,7 +91,6 @@ public void getAccountByIdShouldMergeAccountWithDefaultAccount() { .videoCacheTtl(200) .build()) .privacy(AccountPrivacyConfig.of( - true, AccountGdprConfig.builder() .enabledForRequestType(EnabledForRequestType.of(true, null, null, null)) .build(), @@ -109,7 +108,6 @@ public void getAccountByIdShouldMergeAccountWithDefaultAccount() { .videoCacheTtl(200) .build()) .privacy(AccountPrivacyConfig.of( - true, AccountGdprConfig.builder() .enabled(true) .enabledForRequestType(EnabledForRequestType.of(true, null, null, null)) diff --git a/src/test/java/org/prebid/server/settings/FileApplicationSettingsTest.java b/src/test/java/org/prebid/server/settings/FileApplicationSettingsTest.java index 05377eecfe4..f7e5d564c48 100644 --- a/src/test/java/org/prebid/server/settings/FileApplicationSettingsTest.java +++ b/src/test/java/org/prebid/server/settings/FileApplicationSettingsTest.java @@ -99,7 +99,6 @@ public void getAccountByIdShouldReturnPresentAccount() { + "}" + "}," + "privacy: {" - + "enforce-ccpa: true," + "gdpr: {" + "enabled: true," + "integration-enabled: {" @@ -148,7 +147,6 @@ public void getAccountByIdShouldReturnPresentAccount() { .events(AccountEventsConfig.of(true)) .build()) .privacy(AccountPrivacyConfig.of( - true, AccountGdprConfig.builder() .enabled(true) .enabledForRequestType(EnabledForRequestType.of(true, true, true, true)) diff --git a/src/test/java/org/prebid/server/settings/HttpApplicationSettingsTest.java b/src/test/java/org/prebid/server/settings/HttpApplicationSettingsTest.java index f77f302f781..ca10fd0beb5 100644 --- a/src/test/java/org/prebid/server/settings/HttpApplicationSettingsTest.java +++ b/src/test/java/org/prebid/server/settings/HttpApplicationSettingsTest.java @@ -105,7 +105,7 @@ public void getAccountByIdShouldReturnFetchedAccount() throws JsonProcessingExce .auction(AccountAuctionConfig.builder() .priceGranularity("testPriceGranularity") .build()) - .privacy(AccountPrivacyConfig.of(true, null, null)) + .privacy(AccountPrivacyConfig.of(null, null)) .build(); HttpAccountsResponse response = HttpAccountsResponse.of(Collections.singletonMap("someId", account)); givenHttpClientReturnsResponse(200, mapper.writeValueAsString(response)); @@ -116,7 +116,6 @@ public void getAccountByIdShouldReturnFetchedAccount() throws JsonProcessingExce // then assertThat(future.succeeded()).isTrue(); assertThat(future.result().getId()).isEqualTo("someId"); - assertThat(future.result().getPrivacy().getEnforceCcpa()).isEqualTo(true); assertThat(future.result().getAuction().getPriceGranularity()).isEqualTo("testPriceGranularity"); verify(httpClient).get(eq("http://stored-requests?account-ids=[\"someId\"]"), any(), diff --git a/src/test/java/org/prebid/server/settings/JdbcApplicationSettingsTest.java b/src/test/java/org/prebid/server/settings/JdbcApplicationSettingsTest.java index 9e955bdc811..98b3544de62 100644 --- a/src/test/java/org/prebid/server/settings/JdbcApplicationSettingsTest.java +++ b/src/test/java/org/prebid/server/settings/JdbcApplicationSettingsTest.java @@ -234,7 +234,6 @@ public void getAccountByIdShouldReturnAccountWithAllFieldsPopulated(TestContext .events(AccountEventsConfig.of(true)) .build()) .privacy(AccountPrivacyConfig.of( - true, AccountGdprConfig.builder() .enabled(true) .enabledForRequestType(EnabledForRequestType.of(true, true, true, true))