diff --git a/docs/config-app.md b/docs/config-app.md index 42a1be3e90f..69d0b760303 100644 --- a/docs/config-app.md +++ b/docs/config-app.md @@ -63,6 +63,9 @@ Removes and downloads file again if depending service cant process probably corr - `max-timeout-ms` - this setting controls maximum timeout for /auction endpoint. - `timeout-adjustment-ms` - reduces timeout value passed in legacy Auction request so that Prebid Server can handle timeouts from adapters and respond to the request before it times out. +## Default bid request +- `default-request.file.path` - path to a JSON file containing the default request + ## Auction (OpenRTB) - `auction.blacklisted-accounts` - comma separated list of blacklisted account IDs. - `auction.blacklisted-apps` - comma separated list of blacklisted applications IDs, requests from which should not be processed. diff --git a/docs/developers/default-request.md b/docs/developers/default-request.md new file mode 100644 index 00000000000..08bf1041009 --- /dev/null +++ b/docs/developers/default-request.md @@ -0,0 +1,29 @@ +# Server Based Global Default Request + +This allows a default request to be defined that allows the server to set up some defaults for all incoming +requests. A stored request(s) referenced by a bid request override default request, and of course any options specified +directly in the bid request override both. The default request is only read on server startup, it is meant as an +installation static default rather than a dynamic tuning option. + +## Config Options + +Following config options are exposed to support this feature. +```yaml +default-request: + file: + path : path/to/default/request.json +``` + +The `path` option is the path to a JSON file containing the default request JSON. +```json +{ + "tmax": "2000", + "regs": { + "ext": { + "gdpr": 1 + } + } +} +``` +This will be JSON merged into the incoming requests at the top level. These will be used as fallbacks which can be +overridden by both Stored Requests _and_ the incoming HTTP request payload. diff --git a/docs/endpoints/openrtb2/auction.md b/docs/endpoints/openrtb2/auction.md index 4c1356a1ba5..a48a9a99bd0 100644 --- a/docs/endpoints/openrtb2/auction.md +++ b/docs/endpoints/openrtb2/auction.md @@ -164,6 +164,7 @@ to set these params on the response at `response.seatbid[i].bid[j].ext.prebid.ta }, "includewinners": false, // Optional param defaulting to true "includebidderkeys": false // Optional param defaulting to true + "includeformat": false // Optional param defaulting to false } } } @@ -178,6 +179,8 @@ For backwards compatibility the following strings will also be allowed as price One of "includewinners" or "includebidderkeys" must be true (both default to true if unset). If both were false, then no targeting keys would be set, which is better configured by omitting targeting altogether. +The parameter "includeformat" indicates the type of the bid (banner, video, etc) for multiformat requests. It will add the key `hb_format` and/or `hb_format_{bidderName}` as per "includewinners" and "includebidderkeys" above. + MediaType PriceGranularity - when a single OpenRTB request contains multiple impressions with different mediatypes, or a single impression supports multiple formats, the different mediatypes may need different price granularities. If `mediatypepricegranularity` is present, `pricegranularity` would only be used for any mediatypes not specified. ``` diff --git a/docs/metrics.md b/docs/metrics.md index 8930e4204e7..7360cdb0d04 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -38,8 +38,6 @@ where `[DATASOURCE]` is a data source name, `DEFAULT_DS` by defaul. ## General auction metrics - `app_requests` - number of requests received from applications - `no_cookie_requests` - number of requests without `uids` cookie or with one that didn't contain at least one live UID -- `safari_requests` - number of requests received from Safari browser -- `safari_no_cookie_requests` - number of requests received from Safari browser without `uids` cookie or with one that didn't contain at least one live UID - `request_time` - timer tracking how long did it take for Prebid Server to serve a request - `imps_requested` - number if impressions requested - `imps_banner` - number of banner impressions diff --git a/src/main/java/org/prebid/server/auction/AmpRequestFactory.java b/src/main/java/org/prebid/server/auction/AmpRequestFactory.java index 4c8a45ade5a..4b7a3130c41 100644 --- a/src/main/java/org/prebid/server/auction/AmpRequestFactory.java +++ b/src/main/java/org/prebid/server/auction/AmpRequestFactory.java @@ -634,11 +634,14 @@ private ExtRequestTargeting createTargetingWithDefaults(ExtRequestPrebid prebid) final boolean includeBidderKeys = isTargetingNull || targeting.getIncludebidderkeys() == null || targeting.getIncludebidderkeys(); + final Boolean includeFormat = !isTargetingNull ? targeting.getIncludeformat() : null; + return ExtRequestTargeting.builder() .pricegranularity(outgoingPriceGranularityNode) .mediatypepricegranularity(mediaTypePriceGranularity) .includewinners(includeWinners) .includebidderkeys(includeBidderKeys) + .includeformat(includeFormat) .build(); } } diff --git a/src/main/java/org/prebid/server/auction/AuctionRequestFactory.java b/src/main/java/org/prebid/server/auction/AuctionRequestFactory.java index 4564a844a34..c2e254738f0 100644 --- a/src/main/java/org/prebid/server/auction/AuctionRequestFactory.java +++ b/src/main/java/org/prebid/server/auction/AuctionRequestFactory.java @@ -570,6 +570,7 @@ private ExtRequestTargeting targetingOrNull(ExtRequestPrebid prebid, Set processStoredRequests(String accountId, BidRequest bidRequest applicationSettings.getStoredData(accountId, requestIds, impIds, timeout(bidRequest)) .compose(storedDataResult -> updateMetrics(storedDataResult, requestIds, impIds)); - return storedRequestsToBidRequest(storedDataFuture, bidRequest, - bidRequestToStoredRequestId.get(bidRequest), impToStoredRequestId); - } - - private Future updateMetrics(StoredDataResult storedDataResult, Set requestIds, - Set impIds) { - requestIds.forEach( - id -> metrics.updateStoredRequestMetric(storedDataResult.getStoredIdToRequest().containsKey(id))); - impIds.forEach( - id -> metrics.updateStoredImpsMetric(storedDataResult.getStoredIdToImp().containsKey(id))); - - return Future.succeededFuture(storedDataResult); + return storedRequestsToBidRequest( + storedDataFuture, bidRequest, bidRequestToStoredRequestId.get(bidRequest), impToStoredRequestId); } /** * Fetches AMP request from the source. */ Future processAmpRequest(String accountId, String ampRequestId) { - final BidRequest bidRequest = BidRequest.builder().build(); + final BidRequest bidRequest = defaultBidRequest != null ? defaultBidRequest : BidRequest.builder().build(); final Future ampStoredDataFuture = applicationSettings.getAmpStoredData( accountId, Collections.singleton(ampRequestId), Collections.emptySet(), timeout(bidRequest)) @@ -132,9 +146,27 @@ Future videoStoredDataResult(String accountId, List .map(storedDataResult -> makeVideoStoredDataResult(storedDataResult, storedIdToImpId, errors)); } + private Future updateMetrics(StoredDataResult storedDataResult, Set requestIds, + Set impIds) { + requestIds.forEach( + id -> metrics.updateStoredRequestMetric(storedDataResult.getStoredIdToRequest().containsKey(id))); + impIds.forEach(id -> metrics.updateStoredImpsMetric(storedDataResult.getStoredIdToImp().containsKey(id))); + + return Future.succeededFuture(storedDataResult); + } + + private static BidRequest readBidRequest( + String defaultBidRequestPath, FileSystem fileSystem, JacksonMapper mapper) { + + return StringUtils.isNotBlank(defaultBidRequestPath) + ? mapper.decodeValue(fileSystem.readFileBlocking(defaultBidRequestPath), BidRequest.class) + : null; + } + private VideoStoredDataResult makeVideoStoredDataResult(StoredDataResult storedDataResult, Map storedIdToImpId, List errors) { + final Map storedIdToStoredImp = storedDataResult.getStoredIdToImp(); final Map impIdToStoredVideo = new HashMap<>(); @@ -172,8 +204,10 @@ private Video parseVideoFromImp(String storedJson) { } private Future storedRequestsToBidRequest(Future storedDataFuture, - BidRequest bidRequest, String storedBidRequestId, + BidRequest bidRequest, + String storedBidRequestId, Map impsToStoredRequestId) { + return storedDataFuture .recover(exception -> Future.failedFuture(new InvalidRequestException( String.format("Stored request fetching failed: %s", exception.getMessage())))) @@ -187,21 +221,31 @@ private Future storedRequestsToBidRequest(Future s /** * Runs {@link BidRequest} and {@link Imp}s merge processes. */ - private BidRequest mergeBidRequestAndImps(BidRequest bidRequest, String storedRequestId, - Map impToStoredId, StoredDataResult storedDataResult) { - return mergeBidRequestImps(mergeBidRequest(bidRequest, storedRequestId, storedDataResult), - impToStoredId, storedDataResult); + private BidRequest mergeBidRequestAndImps(BidRequest bidRequest, + String storedRequestId, + Map impToStoredId, + StoredDataResult storedDataResult) { + + return mergeBidRequestImps( + mergeBidRequest(mergeDefaultRequest(bidRequest), storedRequestId, storedDataResult), + impToStoredId, + storedDataResult); + } + + private BidRequest mergeDefaultRequest(BidRequest bidRequest) { + return jsonMerger.merge(bidRequest, defaultBidRequest, BidRequest.class); } /** * Merges original request with request from stored request source. Values from original request * has higher priority than stored request values. */ - private BidRequest mergeBidRequest(BidRequest originalRequest, String storedRequestId, - StoredDataResult storedDataResult) { + private BidRequest mergeBidRequest( + BidRequest originalRequest, String storedRequestId, StoredDataResult storedDataResult) { + final String storedRequest = storedDataResult.getStoredIdToRequest().get(storedRequestId); return StringUtils.isNotBlank(storedRequestId) - ? jsonMergeUtil.merge(originalRequest, storedRequest, storedRequestId, BidRequest.class) + ? jsonMerger.merge(originalRequest, storedRequest, storedRequestId, BidRequest.class) : originalRequest; } @@ -209,8 +253,9 @@ private BidRequest mergeBidRequest(BidRequest originalRequest, String storedRequ * Merges {@link Imp}s from original request with Imps from stored request source. Values from original request * has higher priority than stored request values. */ - private BidRequest mergeBidRequestImps(BidRequest bidRequest, Map impToStoredId, - StoredDataResult storedDataResult) { + private BidRequest mergeBidRequestImps( + BidRequest bidRequest, Map impToStoredId, StoredDataResult storedDataResult) { + if (impToStoredId.isEmpty()) { return bidRequest; } @@ -220,7 +265,7 @@ private BidRequest mergeBidRequestImps(BidRequest bidRequest, Map i final String storedRequestId = impToStoredId.get(imp); if (storedRequestId != null) { final String storedImp = storedDataResult.getStoredIdToImp().get(storedRequestId); - final Imp mergedImp = jsonMergeUtil.merge(imp, storedImp, storedRequestId, Imp.class); + final Imp mergedImp = jsonMerger.merge(imp, storedImp, storedRequestId, Imp.class); mergedImps.set(i, mergedImp); } } @@ -228,10 +273,10 @@ private BidRequest mergeBidRequestImps(BidRequest bidRequest, Map i } /** - * Maps object to its StoredRequestId if exists. If object's extension contains storedRequest field, expected that - * it includes id too, in another case error about missed id in stored request will be added to error list. - * Gathers all errors into list, and in case if it is not empty, throws {@link InvalidRequestException} with - * list of errors. + * Maps object to its StoredRequestId if exists. If object's extension contains storedRequest field, expected + * that it includes id too, in another case error about missed id in stored request will be added to error list. + * Gathers all errors into list, and in case if it is not empty, throws {@link InvalidRequestException} with list + * of errors. */ private Map mapStoredRequestHolderToStoredRequestId( List storedRequestHolders, Function storedRequestExtractor) { diff --git a/src/main/java/org/prebid/server/auction/TargetingKeywordsCreator.java b/src/main/java/org/prebid/server/auction/TargetingKeywordsCreator.java index 8ea9ea42d92..1ddc9b9221a 100644 --- a/src/main/java/org/prebid/server/auction/TargetingKeywordsCreator.java +++ b/src/main/java/org/prebid/server/auction/TargetingKeywordsCreator.java @@ -6,6 +6,7 @@ import org.prebid.server.exception.PreBidException; import org.prebid.server.proto.openrtb.ext.request.ExtPriceGranularity; import org.prebid.server.proto.response.Bid; +import org.prebid.server.proto.response.MediaType; import java.math.BigDecimal; import java.util.ArrayList; @@ -76,9 +77,15 @@ public class TargetingKeywordsCreator { */ private static final String HB_CACHE_PATH_KEY = "hb_cache_path"; + /** + * Stores bid's format. For example "video" or "banner". + */ + private static final String HB_FORMAT_KEY = "hb_format"; + private final PriceGranularity priceGranularity; private final boolean includeWinners; private final boolean includeBidderKeys; + private final boolean includeFormat; private final boolean isApp; private final int truncateAttrChars; private final String cacheHost; @@ -88,6 +95,7 @@ public class TargetingKeywordsCreator { private TargetingKeywordsCreator(PriceGranularity priceGranularity, boolean includeWinners, boolean includeBidderKeys, + boolean includeFormat, boolean isApp, int truncateAttrChars, String cacheHost, @@ -97,6 +105,7 @@ private TargetingKeywordsCreator(PriceGranularity priceGranularity, this.priceGranularity = priceGranularity; this.includeWinners = includeWinners; this.includeBidderKeys = includeBidderKeys; + this.includeFormat = includeFormat; this.isApp = isApp; this.truncateAttrChars = truncateAttrChars; this.cacheHost = cacheHost; @@ -110,6 +119,7 @@ private TargetingKeywordsCreator(PriceGranularity priceGranularity, public static TargetingKeywordsCreator create(ExtPriceGranularity extPriceGranularity, boolean includeWinners, boolean includeBidderKeys, + boolean includeFormat, boolean isApp, int truncateAttrChars, String cacheHost, @@ -120,6 +130,7 @@ public static TargetingKeywordsCreator create(ExtPriceGranularity extPriceGranul PriceGranularity.createFromExtPriceGranularity(extPriceGranularity), includeWinners, includeBidderKeys, + includeFormat, isApp, truncateAttrChars, cacheHost, @@ -133,6 +144,7 @@ public static TargetingKeywordsCreator create(ExtPriceGranularity extPriceGranul public static TargetingKeywordsCreator create(String stringPriceGranularity, boolean includeWinners, boolean includeBidderKeys, + boolean includeFormat, boolean isApp, int truncateAttrChars) { @@ -140,6 +152,7 @@ public static TargetingKeywordsCreator create(String stringPriceGranularity, convertToCustomPriceGranularity(stringPriceGranularity), includeWinners, includeBidderKeys, + includeFormat, isApp, truncateAttrChars, null, @@ -169,6 +182,7 @@ private static PriceGranularity convertToCustomPriceGranularity(String stringPri * Creates map of keywords for the given {@link Bid}. */ public Map makeFor(Bid bid, boolean winningBid) { + final MediaType mediaType = bid.getMediaType(); return truncateKeys(makeFor( bid.getBidder(), winningBid, @@ -178,6 +192,7 @@ public Map makeFor(Bid bid, boolean winningBid) { bid.getHeight(), bid.getCacheId(), null, + mediaType != null ? mediaType.name() : null, bid.getDealId())); } @@ -188,6 +203,7 @@ Map makeFor(com.iab.openrtb.response.Bid bid, String bidder, boolean winningBid, String cacheId, + String format, String vastCacheId) { final Map keywords = makeFor( @@ -199,6 +215,7 @@ Map makeFor(com.iab.openrtb.response.Bid bid, bid.getH(), cacheId, vastCacheId, + format, bid.getDealid()); if (resolver == null) { @@ -222,6 +239,7 @@ private Map makeFor(String bidder, Integer height, String cacheId, String vastCacheId, + String format, String dealId) { final KeywordMap keywordMap = new KeywordMap(bidder, winningBid, includeWinners, includeBidderKeys, @@ -247,6 +265,9 @@ private Map makeFor(String bidder, keywordMap.put(HB_CACHE_HOST_KEY, cacheHost); keywordMap.put(HB_CACHE_PATH_KEY, cachePath); } + if (StringUtils.isNotBlank(format) && includeFormat) { + keywordMap.put(HB_FORMAT_KEY, format); + } if (StringUtils.isNotBlank(dealId)) { keywordMap.put(HB_DEAL_KEY, dealId); } diff --git a/src/main/java/org/prebid/server/auction/VideoRequestFactory.java b/src/main/java/org/prebid/server/auction/VideoRequestFactory.java index f815cf27da7..556eedcb9c8 100644 --- a/src/main/java/org/prebid/server/auction/VideoRequestFactory.java +++ b/src/main/java/org/prebid/server/auction/VideoRequestFactory.java @@ -42,10 +42,13 @@ public class VideoRequestFactory { private final TimeoutResolver timeoutResolver; private final JacksonMapper mapper; - public VideoRequestFactory(int maxRequestSize, boolean enforceStoredRequest, + public VideoRequestFactory(int maxRequestSize, + boolean enforceStoredRequest, VideoStoredRequestProcessor storedRequestProcessor, - AuctionRequestFactory auctionRequestFactory, TimeoutResolver timeoutResolver, + AuctionRequestFactory auctionRequestFactory, + TimeoutResolver timeoutResolver, JacksonMapper mapper) { + this.enforceStoredRequest = enforceStoredRequest; this.maxRequestSize = maxRequestSize; this.storedRequestProcessor = Objects.requireNonNull(storedRequestProcessor); @@ -175,8 +178,9 @@ private Future> createBidRequest(RoutingContext routin BidRequestVideo bidRequestVideo, String storedVideoId, Set podConfigIds) { - return storedRequestProcessor.processVideoRequest(accountIdFrom(bidRequestVideo), storedVideoId, podConfigIds, - bidRequestVideo) + + return storedRequestProcessor.processVideoRequest( + accountIdFrom(bidRequestVideo), storedVideoId, podConfigIds, bidRequestVideo) .map(bidRequestToErrors -> fillImplicitParameters(routingContext, bidRequestToErrors)) .map(this::validateRequest); } diff --git a/src/main/java/org/prebid/server/auction/VideoResponseFactory.java b/src/main/java/org/prebid/server/auction/VideoResponseFactory.java index 2a9083d2385..752b632c2a1 100644 --- a/src/main/java/org/prebid/server/auction/VideoResponseFactory.java +++ b/src/main/java/org/prebid/server/auction/VideoResponseFactory.java @@ -88,7 +88,7 @@ private List adPodsWithTargetingFrom(List bids) { final List adPods = new ArrayList<>(); for (Bid bid : bids) { final Map targeting = targeting(bid); - if (targeting.get("hb_uuid") == null) { + if (findByPrefix(targeting, "hb_uuid") == null) { continue; } final String impId = bid.getImpid(); @@ -99,9 +99,9 @@ private List adPodsWithTargetingFrom(List bids) { final Integer podId = Integer.parseInt(podIdString); final ExtResponseVideoTargeting videoTargeting = ExtResponseVideoTargeting.of( - targeting.get("hb_pb"), - targeting.get("hb_pb_cat_dur"), - targeting.get("hb_uuid")); + findByPrefix(targeting, "hb_pb"), + findByPrefix(targeting, "hb_pb_cat_dur"), + findByPrefix(targeting, "hb_uuid")); ExtAdPod adPod = adPods.stream() .filter(extAdPod -> extAdPod.getPodid().equals(podId)) @@ -130,6 +130,14 @@ private Map targeting(Bid bid) { return targeting != null ? targeting : Collections.emptyMap(); } + private static String findByPrefix(Map keyToValue, String prefix) { + return keyToValue.entrySet().stream() + .filter(keyAndValue -> keyAndValue.getKey().startsWith(prefix)) + .map(Map.Entry::getValue) + .findFirst() + .orElse(null); + } + private static List adPodsWithErrors(List podErrors) { return podErrors.stream() .map(podError -> ExtAdPod.of(podError.getPodId(), null, podError.getPodErrors())) diff --git a/src/main/java/org/prebid/server/auction/VideoStoredRequestProcessor.java b/src/main/java/org/prebid/server/auction/VideoStoredRequestProcessor.java index 0b87f2a49d3..65f95ba330f 100644 --- a/src/main/java/org/prebid/server/auction/VideoStoredRequestProcessor.java +++ b/src/main/java/org/prebid/server/auction/VideoStoredRequestProcessor.java @@ -16,6 +16,7 @@ import com.iab.openrtb.request.video.PodError; import com.iab.openrtb.request.video.Podconfig; import io.vertx.core.Future; +import io.vertx.core.file.FileSystem; import io.vertx.core.logging.Logger; import io.vertx.core.logging.LoggerFactory; import org.apache.commons.collections4.CollectionUtils; @@ -27,6 +28,7 @@ import org.prebid.server.exception.InvalidRequestException; import org.prebid.server.execution.TimeoutFactory; import org.prebid.server.json.JacksonMapper; +import org.prebid.server.json.JsonMerger; import org.prebid.server.metric.Metrics; import org.prebid.server.proto.openrtb.ext.ExtIncludeBrandCategory; import org.prebid.server.proto.openrtb.ext.request.ExtRequest; @@ -36,7 +38,6 @@ import org.prebid.server.proto.openrtb.ext.request.ExtRequestTargeting; import org.prebid.server.settings.ApplicationSettings; import org.prebid.server.settings.model.StoredDataResult; -import org.prebid.server.util.JsonMergeUtil; import org.prebid.server.validation.VideoRequestValidator; import java.util.ArrayList; @@ -63,39 +64,69 @@ public class VideoStoredRequestProcessor { private final long defaultTimeout; private final String currency; private final BidRequest defaultBidRequest; - private final VideoRequestValidator validator; private final ApplicationSettings applicationSettings; + private final VideoRequestValidator validator; private final Metrics metrics; - private final TimeoutFactory timeoutFactory; private final TimeoutResolver timeoutResolver; + private final TimeoutFactory timeoutFactory; private final JacksonMapper mapper; - private final JsonMergeUtil jsonMergeUtil; - - public VideoStoredRequestProcessor(boolean enforceStoredRequest, - List blacklistedAccounts, - long defaultTimeout, - String adServerCurrency, - BidRequest defaultBidRequest, - VideoRequestValidator validator, - ApplicationSettings applicationSettings, - Metrics metrics, - TimeoutFactory timeoutFactory, - TimeoutResolver timeoutResolver, - JacksonMapper mapper) { + private final JsonMerger jsonMerger; + + private VideoStoredRequestProcessor(boolean enforceStoredRequest, + List blacklistedAccounts, + long defaultTimeout, + String adServerCurrency, + BidRequest defaultBidRequest, + ApplicationSettings applicationSettings, + VideoRequestValidator validator, + Metrics metrics, + TimeoutFactory timeoutFactory, + TimeoutResolver timeoutResolver, + JacksonMapper mapper, + JsonMerger jsonMerger) { this.enforceStoredRequest = enforceStoredRequest; this.blacklistedAccounts = blacklistedAccounts; this.defaultTimeout = defaultTimeout; this.currency = StringUtils.isBlank(adServerCurrency) ? DEFAULT_CURRENCY : adServerCurrency; - this.defaultBidRequest = Objects.requireNonNull(defaultBidRequest); - this.validator = Objects.requireNonNull(validator); - this.applicationSettings = Objects.requireNonNull(applicationSettings); - this.metrics = Objects.requireNonNull(metrics); - this.timeoutFactory = Objects.requireNonNull(timeoutFactory); - this.timeoutResolver = Objects.requireNonNull(timeoutResolver); - this.mapper = Objects.requireNonNull(mapper); - - jsonMergeUtil = new JsonMergeUtil(mapper); + this.defaultBidRequest = defaultBidRequest; + this.applicationSettings = applicationSettings; + this.validator = validator; + this.metrics = metrics; + this.timeoutFactory = timeoutFactory; + this.timeoutResolver = timeoutResolver; + this.mapper = mapper; + this.jsonMerger = jsonMerger; + } + + public static VideoStoredRequestProcessor create(boolean enforceStoredRequest, + List blacklistedAccounts, + long defaultTimeout, + String adServerCurrency, + String defaultBidRequestPath, + FileSystem fileSystem, + ApplicationSettings applicationSettings, + VideoRequestValidator validator, + Metrics metrics, + TimeoutFactory timeoutFactory, + TimeoutResolver timeoutResolver, + JacksonMapper mapper, + JsonMerger jsonMerger) { + + return new VideoStoredRequestProcessor( + enforceStoredRequest, + Objects.requireNonNull(blacklistedAccounts), + defaultTimeout, + adServerCurrency, + readBidRequest( + defaultBidRequestPath, Objects.requireNonNull(fileSystem), Objects.requireNonNull(mapper)), + Objects.requireNonNull(applicationSettings), + Objects.requireNonNull(validator), + Objects.requireNonNull(metrics), + Objects.requireNonNull(timeoutFactory), + Objects.requireNonNull(timeoutResolver), + Objects.requireNonNull(mapper), + Objects.requireNonNull(jsonMerger)); } /** @@ -116,6 +147,14 @@ Future> processVideoRequest(String accountId, String s String.format("Stored request fetching failed: %s", exception.getMessage())))); } + private static BidRequest readBidRequest( + String defaultBidRequestPath, FileSystem fileSystem, JacksonMapper mapper) { + + return StringUtils.isNotBlank(defaultBidRequestPath) + ? mapper.decodeValue(fileSystem.readFileBlocking(defaultBidRequestPath), BidRequest.class) + : null; + } + private Future updateMetrics(StoredDataResult storedDataResult, Set requestIds, Set impIds) { requestIds.forEach( @@ -150,7 +189,7 @@ private BidRequestVideo mergeBidRequest(BidRequestVideo originalRequest, String } return StringUtils.isNotBlank(storedRequest) - ? jsonMergeUtil.merge(originalRequest, storedRequest, storedRequestId, BidRequestVideo.class) + ? jsonMerger.merge(originalRequest, storedRequest, storedRequestId, BidRequestVideo.class) : originalRequest; } @@ -256,7 +295,9 @@ private static Video updateVideo(Video video, Integer maxDuration, Integer minDu } private BidRequest mergeWithDefaultBidRequest(BidRequestVideo validatedVideoRequest, List imps) { - final BidRequest.BidRequestBuilder bidRequestBuilder = defaultBidRequest.toBuilder(); + final BidRequest.BidRequestBuilder bidRequestBuilder = + defaultBidRequest != null ? defaultBidRequest.toBuilder() : BidRequest.builder(); + final Site site = validatedVideoRequest.getSite(); if (site != null) { final Site.SiteBuilder siteBuilder = site.toBuilder(); diff --git a/src/main/java/org/prebid/server/bidder/rubicon/RubiconBidder.java b/src/main/java/org/prebid/server/bidder/rubicon/RubiconBidder.java index fcd3d1cbaa8..1a1ff24424b 100644 --- a/src/main/java/org/prebid/server/bidder/rubicon/RubiconBidder.java +++ b/src/main/java/org/prebid/server/bidder/rubicon/RubiconBidder.java @@ -170,8 +170,14 @@ public Result>> makeHttpRequests(BidRequest bidRequ final List> httpRequests = new ArrayList<>(); final List errors = new ArrayList<>(); + final List imps = extractValidImps(bidRequest, errors); + if (CollectionUtils.isEmpty(imps)) { + errors.add(BidderError.of("There are no valid impressions to create bid request to rubicon bidder", + BidderError.Type.bad_input)); + return Result.of(Collections.emptyList(), errors); + } final Map> impToImpExt = - parseRubiconImpExts(bidRequest.getImp(), errors); + parseRubiconImpExts(imps, errors); final String impLanguage = firstImpExtLanguage(impToImpExt.values()); for (Map.Entry> impToExt : impToImpExt.entrySet()) { @@ -226,6 +232,44 @@ public Map extractTargeting(ObjectNode extBidBidder) { : Collections.emptyMap(); } + private List extractValidImps(BidRequest bidRequest, List errors) { + final Map> isValidToImps = bidRequest.getImp().stream() + .collect(Collectors.groupingBy(RubiconBidder::isValidType)); + + isValidToImps.getOrDefault(false, Collections.emptyList()).stream() + .map(this::impTypeErrorMessage) + .forEach(errors::add); + + return isValidToImps.getOrDefault(true, Collections.emptyList()); + } + + private static boolean isValidType(Imp imp) { + return imp.getVideo() != null || imp.getBanner() != null; + } + + private BidderError impTypeErrorMessage(Imp imp) { + final BidType type = resolveExpectedBidType(imp); + return BidderError.of( + String.format("Impression with id %s rejected with invalid type `%s`." + " Allowed types are banner and" + + " video.", imp.getId(), type != null ? type.name() : "unknown"), BidderError.Type.bad_input); + } + + private static BidType resolveExpectedBidType(Imp imp) { + if (imp.getBanner() != null) { + return BidType.banner; + } + if (imp.getVideo() != null) { + return BidType.video; + } + if (imp.getAudio() != null) { + return BidType.audio; + } + if (imp.getXNative() != null) { + return BidType.xNative; + } + return null; + } + private static MultiMap headers(String xapiUsername, String xapiPassword) { return MultiMap.caseInsensitiveMultiMap() .add(HttpUtil.AUTHORIZATION_HEADER, authHeader(xapiUsername, xapiPassword)) @@ -903,14 +947,11 @@ private void mergeFirstPartyDataFromUser(User user, ObjectNode result) { private static String extractLiverampId(Map> sourceToUserEidExt) { final List liverampEids = MapUtils.emptyIfNull(sourceToUserEidExt).get(LIVERAMP_EID); for (ExtUserEid extUserEid : CollectionUtils.emptyIfNull(liverampEids)) { - final ExtUserEidUid eidUid = extUserEid != null - ? CollectionUtils.emptyIfNull(extUserEid.getUids()).stream().findFirst().orElse(null) - : null; - - final String id = eidUid != null ? eidUid.getId() : null; - if (StringUtils.isNotEmpty(id)) { - return id; - } + return extUserEid.getUids().stream() + .map(ExtUserEidUid::getId) + .filter(StringUtils::isNotEmpty) + .findFirst() + .orElse(null); } return null; } diff --git a/src/main/java/org/prebid/server/handler/AuctionHandler.java b/src/main/java/org/prebid/server/handler/AuctionHandler.java index c79c8d28545..0aeadc8d5a2 100644 --- a/src/main/java/org/prebid/server/handler/AuctionHandler.java +++ b/src/main/java/org/prebid/server/handler/AuctionHandler.java @@ -107,16 +107,11 @@ public AuctionHandler(ApplicationSettings applicationSettings, public void handle(RoutingContext context) { final long startTime = clock.millis(); - final boolean isSafari = HttpUtil.isSafari(context.request().headers().get(HttpUtil.USER_AGENT_HEADER)); - - metrics.updateSafariRequestsMetric(isSafari); - preBidRequestContextFactory.fromRequest(context) .recover(exception -> failWithInvalidRequest( String.format("Error parsing request: %s", exception.getMessage()), exception)) - .map(preBidRequestContext -> - updateAppAndNoCookieAndImpsMetrics(preBidRequestContext, isSafari)) + .map(this::updateAppAndNoCookieAndImpsMetrics) .compose(preBidRequestContext -> accountFrom(preBidRequestContext) .map(account -> Tuple2.of(preBidRequestContext, account))) @@ -147,13 +142,12 @@ public void handle(RoutingContext context) { respondWith(bidResponseOrError(preBidResponseResult), context, startTime)); } - private PreBidRequestContext updateAppAndNoCookieAndImpsMetrics(PreBidRequestContext preBidRequestContext, - boolean isSafari) { + private PreBidRequestContext updateAppAndNoCookieAndImpsMetrics(PreBidRequestContext preBidRequestContext) { final PreBidRequest preBidRequest = preBidRequestContext.getPreBidRequest(); final List adUnits = preBidRequest.getAdUnits(); metrics.updateAppAndNoCookieAndImpsRequestedMetrics(preBidRequest.getApp() != null, - !preBidRequestContext.isNoLiveUids(), isSafari, adUnits.size()); + !preBidRequestContext.isNoLiveUids(), adUnits.size()); final Map mediaTypeToCount = adUnits.stream() .map(AdUnit::getMediaTypes) @@ -400,7 +394,7 @@ private static PreBidResponse addTargetingKeywords(PreBidRequest preBidRequest, final Integer sortBids = preBidRequest.getSortBids(); if (sortBids != null && sortBids == 1) { final TargetingKeywordsCreator keywordsCreator = - TargetingKeywordsCreator.create(account.getPriceGranularity(), true, true, false, 0); + TargetingKeywordsCreator.create(account.getPriceGranularity(), true, true, false, false, 0); final Map> adUnitCodeToBids = preBidResponse.getBids().stream() .collect(Collectors.groupingBy(Bid::getCode)); diff --git a/src/main/java/org/prebid/server/handler/openrtb2/AmpHandler.java b/src/main/java/org/prebid/server/handler/openrtb2/AmpHandler.java index ad5b50fb092..9be3171ee9b 100644 --- a/src/main/java/org/prebid/server/handler/openrtb2/AmpHandler.java +++ b/src/main/java/org/prebid/server/handler/openrtb2/AmpHandler.java @@ -115,16 +115,13 @@ public void handle(RoutingContext routingContext) { // more accurately if we note the real start time, and use it to compute the auction timeout. final long startTime = clock.millis(); - final boolean isSafari = HttpUtil.isSafari(routingContext.request().headers().get(HttpUtil.USER_AGENT_HEADER)); - metrics.updateSafariRequestsMetric(isSafari); - final AmpEvent.AmpEventBuilder ampEventBuilder = AmpEvent.builder() .httpContext(HttpContext.from(routingContext)); ampRequestFactory.fromRequest(routingContext, startTime) .map(context -> addToEvent(context, ampEventBuilder::auctionContext, context)) - .map(context -> updateAppAndNoCookieAndImpsMetrics(context, isSafari)) + .map(this::updateAppAndNoCookieAndImpsMetrics) .compose(context -> exchangeService.holdAuction(context) .map(bidResponse -> Tuple2.of(bidResponse, context))) @@ -155,13 +152,13 @@ private static R addToEvent(T field, Consumer consumer, R result) { return result; } - private AuctionContext updateAppAndNoCookieAndImpsMetrics(AuctionContext context, boolean isSafari) { + private AuctionContext updateAppAndNoCookieAndImpsMetrics(AuctionContext context) { final BidRequest bidRequest = context.getBidRequest(); final UidsCookie uidsCookie = context.getUidsCookie(); final List imps = bidRequest.getImp(); metrics.updateAppAndNoCookieAndImpsRequestedMetrics(bidRequest.getApp() != null, uidsCookie.hasLiveUids(), - isSafari, imps.size()); + imps.size()); metrics.updateImpTypesMetrics(imps); diff --git a/src/main/java/org/prebid/server/handler/openrtb2/AuctionHandler.java b/src/main/java/org/prebid/server/handler/openrtb2/AuctionHandler.java index 6e0b6e553ee..86651f0bb07 100644 --- a/src/main/java/org/prebid/server/handler/openrtb2/AuctionHandler.java +++ b/src/main/java/org/prebid/server/handler/openrtb2/AuctionHandler.java @@ -74,16 +74,13 @@ public void handle(RoutingContext routingContext) { // more accurately if we note the real start time, and use it to compute the auction timeout. final long startTime = clock.millis(); - final boolean isSafari = HttpUtil.isSafari(routingContext.request().headers().get(HttpUtil.USER_AGENT_HEADER)); - metrics.updateSafariRequestsMetric(isSafari); - final AuctionEvent.AuctionEventBuilder auctionEventBuilder = AuctionEvent.builder() .httpContext(HttpContext.from(routingContext)); auctionRequestFactory.fromRequest(routingContext, startTime) .map(context -> addToEvent(context, auctionEventBuilder::auctionContext, context)) - .map(context -> updateAppAndNoCookieAndImpsMetrics(context, isSafari)) + .map(context -> updateAppAndNoCookieAndImpsMetrics(context)) .compose(context -> exchangeService.holdAuction(context) .map(bidResponse -> Tuple2.of(bidResponse, context))) @@ -97,13 +94,13 @@ private static R addToEvent(T field, Consumer consumer, R result) { return result; } - private AuctionContext updateAppAndNoCookieAndImpsMetrics(AuctionContext context, boolean isSafari) { + private AuctionContext updateAppAndNoCookieAndImpsMetrics(AuctionContext context) { final BidRequest bidRequest = context.getBidRequest(); final UidsCookie uidsCookie = context.getUidsCookie(); final List imps = bidRequest.getImp(); metrics.updateAppAndNoCookieAndImpsRequestedMetrics(bidRequest.getApp() != null, uidsCookie.hasLiveUids(), - isSafari, imps.size()); + imps.size()); metrics.updateImpTypesMetrics(imps); diff --git a/src/main/java/org/prebid/server/handler/openrtb2/VideoHandler.java b/src/main/java/org/prebid/server/handler/openrtb2/VideoHandler.java index 36218834993..d021c76df0f 100644 --- a/src/main/java/org/prebid/server/handler/openrtb2/VideoHandler.java +++ b/src/main/java/org/prebid/server/handler/openrtb2/VideoHandler.java @@ -63,8 +63,6 @@ public void handle(RoutingContext routingContext) { // more accurately if we note the real start time, and use it to compute the auction timeout. final long startTime = clock.millis(); - final boolean isSafari = HttpUtil.isSafari(routingContext.request().headers().get(HttpUtil.USER_AGENT_HEADER)); - metrics.updateSafariRequestsMetric(isSafari); final VideoEvent.VideoEventBuilder videoEventBuilder = VideoEvent.builder() .httpContext(HttpContext.from(routingContext)); diff --git a/src/main/java/org/prebid/server/util/JsonMergeUtil.java b/src/main/java/org/prebid/server/json/JsonMerger.java similarity index 94% rename from src/main/java/org/prebid/server/util/JsonMergeUtil.java rename to src/main/java/org/prebid/server/json/JsonMerger.java index de5b4371095..ee21248df27 100644 --- a/src/main/java/org/prebid/server/util/JsonMergeUtil.java +++ b/src/main/java/org/prebid/server/json/JsonMerger.java @@ -1,4 +1,4 @@ -package org.prebid.server.util; +package org.prebid.server.json; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -6,17 +6,15 @@ import com.github.fge.jsonpatch.mergepatch.JsonMergePatch; import org.apache.commons.lang3.ObjectUtils; import org.prebid.server.exception.InvalidRequestException; -import org.prebid.server.json.JacksonMapper; import java.io.IOException; import java.util.Objects; -// TODO: refactor to be instance instead of util -public class JsonMergeUtil { +public class JsonMerger { private final JacksonMapper mapper; - public JsonMergeUtil(JacksonMapper mapper) { + public JsonMerger(JacksonMapper mapper) { this.mapper = Objects.requireNonNull(mapper); } diff --git a/src/main/java/org/prebid/server/metric/MetricName.java b/src/main/java/org/prebid/server/metric/MetricName.java index 428daed2af1..e1580af8958 100644 --- a/src/main/java/org/prebid/server/metric/MetricName.java +++ b/src/main/java/org/prebid/server/metric/MetricName.java @@ -24,8 +24,6 @@ public enum MetricName { requests, app_requests, no_cookie_requests, - safari_requests, - safari_no_cookie_requests, request_time, prices, imps_requested, diff --git a/src/main/java/org/prebid/server/metric/Metrics.java b/src/main/java/org/prebid/server/metric/Metrics.java index 7029aa98ffc..5980206f0a4 100644 --- a/src/main/java/org/prebid/server/metric/Metrics.java +++ b/src/main/java/org/prebid/server/metric/Metrics.java @@ -121,21 +121,11 @@ SettingsCacheMetrics forSettingsCacheType(MetricName type) { return settingsCacheMetrics.computeIfAbsent(type, settingsCacheMetricsCreator); } - public void updateSafariRequestsMetric(boolean isSafari) { - if (isSafari) { - incCounter(MetricName.safari_requests); - } - } - - public void updateAppAndNoCookieAndImpsRequestedMetrics(boolean isApp, boolean liveUidsPresent, boolean isSafari, - int numImps) { + public void updateAppAndNoCookieAndImpsRequestedMetrics(boolean isApp, boolean liveUidsPresent, int numImps) { if (isApp) { incCounter(MetricName.app_requests); } else if (!liveUidsPresent) { incCounter(MetricName.no_cookie_requests); - if (isSafari) { - incCounter(MetricName.safari_no_cookie_requests); - } } incCounter(MetricName.imps_requested, numImps); } diff --git a/src/main/java/org/prebid/server/proto/openrtb/ext/request/ExtRequestTargeting.java b/src/main/java/org/prebid/server/proto/openrtb/ext/request/ExtRequestTargeting.java index 0099e1d9d94..b2eac627e62 100644 --- a/src/main/java/org/prebid/server/proto/openrtb/ext/request/ExtRequestTargeting.java +++ b/src/main/java/org/prebid/server/proto/openrtb/ext/request/ExtRequestTargeting.java @@ -44,6 +44,11 @@ public class ExtRequestTargeting { */ Boolean includebidderkeys; + /** + * Defines the contract for bidrequest.ext.prebid.targeting.includeformat + */ + Boolean includeformat; + /** * Defines the contract for bidrequest.ext.prebid.targeting.truncateattrchars */ diff --git a/src/main/java/org/prebid/server/proto/openrtb/ext/response/BidType.java b/src/main/java/org/prebid/server/proto/openrtb/ext/response/BidType.java index 161ec51a73a..f1143141538 100644 --- a/src/main/java/org/prebid/server/proto/openrtb/ext/response/BidType.java +++ b/src/main/java/org/prebid/server/proto/openrtb/ext/response/BidType.java @@ -8,5 +8,9 @@ public enum BidType { video, audio, @JsonProperty("native") - xNative + xNative; + + public String getName() { + return this == xNative ? "native" : this.name(); + } } diff --git a/src/main/java/org/prebid/server/spring/config/JacksonConfiguration.java b/src/main/java/org/prebid/server/spring/config/JsonConfiguration.java similarity index 68% rename from src/main/java/org/prebid/server/spring/config/JacksonConfiguration.java rename to src/main/java/org/prebid/server/spring/config/JsonConfiguration.java index 52239591ca5..eda4e2fb32d 100644 --- a/src/main/java/org/prebid/server/spring/config/JacksonConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/JsonConfiguration.java @@ -1,15 +1,21 @@ package org.prebid.server.spring.config; import org.prebid.server.json.JacksonMapper; +import org.prebid.server.json.JsonMerger; import org.prebid.server.json.ObjectMapperProvider; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration -public class JacksonConfiguration { +public class JsonConfiguration { @Bean JacksonMapper jacksonMapper() { return new JacksonMapper(ObjectMapperProvider.mapper()); } + + @Bean + JsonMerger jsonMerger(JacksonMapper mapper) { + return new JsonMerger(mapper); + } } diff --git a/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java b/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java index 60576ce785a..ba7957c8d80 100644 --- a/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java +++ b/src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java @@ -1,9 +1,9 @@ package org.prebid.server.spring.config; -import com.iab.openrtb.request.BidRequest; import de.malkusch.whoisServerList.publicSuffixList.PublicSuffixList; import de.malkusch.whoisServerList.publicSuffixList.PublicSuffixListFactory; import io.vertx.core.Vertx; +import io.vertx.core.file.FileSystem; import io.vertx.core.http.HttpClientOptions; import io.vertx.core.net.JksOptions; import org.prebid.server.auction.AmpRequestFactory; @@ -42,6 +42,7 @@ import org.prebid.server.identity.NoneIdGenerator; import org.prebid.server.identity.UUIDIdGenerator; import org.prebid.server.json.JacksonMapper; +import org.prebid.server.json.JsonMerger; import org.prebid.server.log.HttpInteractionLogger; import org.prebid.server.log.LoggerControlKnob; import org.prebid.server.metric.Metrics; @@ -125,13 +126,13 @@ IpAddressHelper ipAddressHelper(@Value("${ipv6.always-mask-right}") int ipv6Alwa } @Bean - FpdResolver fpdResolver(JacksonMapper mapper) { - return new FpdResolver(mapper); + FpdResolver fpdResolver(JacksonMapper mapper, JsonMerger jsonMerger) { + return new FpdResolver(mapper, jsonMerger); } @Bean - OrtbTypesResolver ortbTypesResolver(JacksonMapper jacksonMapper) { - return new OrtbTypesResolver(jacksonMapper); + OrtbTypesResolver ortbTypesResolver(JacksonMapper jacksonMapper, JsonMerger jsonMerger) { + return new OrtbTypesResolver(jacksonMapper, jsonMerger); } @Bean @@ -284,32 +285,35 @@ VideoStoredRequestProcessor videoStoredRequestProcessor( @Value("${auction.blacklisted-accounts}") String blacklistedAccountsString, @Value("${video.stored-requests-timeout-ms}") long defaultTimeoutMs, @Value("${auction.ad-server-currency:#{null}}") String adServerCurrency, - BidRequest defaultVideoBidRequest, + @Value("${default-request.file.path:#{null}}") String defaultBidRequestPath, + FileSystem fileSystem, ApplicationSettings applicationSettings, + VideoRequestValidator videoRequestValidator, Metrics metrics, TimeoutFactory timeoutFactory, TimeoutResolver timeoutResolver, - JacksonMapper mapper) { - - final List blacklistedAccounts = splitToList(blacklistedAccountsString); + JacksonMapper mapper, + JsonMerger jsonMerger) { - return new VideoStoredRequestProcessor( + return VideoStoredRequestProcessor.create( enforceStoredRequest, - blacklistedAccounts, + splitToList(blacklistedAccountsString), defaultTimeoutMs, adServerCurrency, - defaultVideoBidRequest, - new VideoRequestValidator(), + defaultBidRequestPath, + fileSystem, applicationSettings, + videoRequestValidator, metrics, timeoutFactory, timeoutResolver, - mapper); + mapper, + jsonMerger); } @Bean - BidRequest defaultVideoBidRequest() { - return BidRequest.builder().build(); + VideoRequestValidator videoRequestValidator() { + return new VideoRequestValidator(); } @Bean @@ -501,12 +505,23 @@ ExchangeService exchangeService( @Bean StoredRequestProcessor storedRequestProcessor( @Value("${auction.stored-requests-timeout-ms}") long defaultTimeoutMs, + @Value("${default-request.file.path:#{null}}") String defaultBidRequestPath, + FileSystem fileSystem, ApplicationSettings applicationSettings, Metrics metrics, TimeoutFactory timeoutFactory, - JacksonMapper mapper) { + JacksonMapper mapper, + JsonMerger jsonMerger) { - return new StoredRequestProcessor(defaultTimeoutMs, applicationSettings, metrics, timeoutFactory, mapper); + return StoredRequestProcessor.create( + defaultTimeoutMs, + defaultBidRequestPath, + fileSystem, + applicationSettings, + metrics, + timeoutFactory, + mapper, + jsonMerger); } @Bean diff --git a/src/main/java/org/prebid/server/util/HttpUtil.java b/src/main/java/org/prebid/server/util/HttpUtil.java index 1f12b937722..3344f8d0e04 100644 --- a/src/main/java/org/prebid/server/util/HttpUtil.java +++ b/src/main/java/org/prebid/server/util/HttpUtil.java @@ -53,20 +53,6 @@ public final class HttpUtil { private HttpUtil() { } - /** - * Detects whether browser is safari or not by user agent analysis. - */ - public static boolean isSafari(String userAgent) { - // this is a simple heuristic based on this article: - // https://developer.mozilla.org/en-US/docs/Web/HTTP/Browser_detection_using_the_user_agent - // - // there are libraries available doing different kinds of User-Agent analysis but they impose performance - // implications as well, example: https://github.com/nielsbasjes/yauaa - return StringUtils.isNotBlank(userAgent) - && userAgent.contains("AppleWebKit") && userAgent.contains("Safari") - && !userAgent.contains("Chrome") && !userAgent.contains("Chromium"); - } - /** * Checks the input string for using as URL. */ diff --git a/src/main/java/org/prebid/server/validation/RequestValidator.java b/src/main/java/org/prebid/server/validation/RequestValidator.java index 28d93c9f807..526c7114a41 100644 --- a/src/main/java/org/prebid/server/validation/RequestValidator.java +++ b/src/main/java/org/prebid/server/validation/RequestValidator.java @@ -59,13 +59,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; /** @@ -397,7 +397,6 @@ private void validateUser(User user, Map aliases) throws Validat throw new ValidationException( "request.user.ext.eids must contain at least one element or be undefined"); } - final Set uniqueSources = new HashSet<>(eids.size()); for (int index = 0; index < eids.size(); index++) { final ExtUserEid eid = eids.get(index); if (StringUtils.isBlank(eid.getSource())) { @@ -425,9 +424,10 @@ private void validateUser(User user, Map aliases) throws Validat } } } - uniqueSources.add(eid.getSource()); } - + final Set uniqueSources = eids.stream() + .map(ExtUserEid::getSource) + .collect(Collectors.toSet()); if (eids.size() != uniqueSources.size()) { throw new ValidationException("request.user.ext.eids must contain unique sources"); } diff --git a/src/test/java/org/prebid/server/auction/AmpRequestFactoryTest.java b/src/test/java/org/prebid/server/auction/AmpRequestFactoryTest.java index 8f3e7bb64b6..864a85ce91b 100644 --- a/src/test/java/org/prebid/server/auction/AmpRequestFactoryTest.java +++ b/src/test/java/org/prebid/server/auction/AmpRequestFactoryTest.java @@ -278,10 +278,8 @@ public void shouldReturnBidRequestWithDefaultIncludeWinnersIfStoredBidRequestExt .extracting(BidRequest::getExt).isNotNull() .extracting(ExtRequest::getPrebid) .extracting(ExtRequestPrebid::getTargeting) - .extracting(ExtRequestTargeting::getIncludewinners, ExtRequestTargeting::getPricegranularity) - // assert that includeWinners was set with default value and priceGranularity remained unchanged - .containsExactly( - tuple(true, mapper.createObjectNode().put("foo", "bar"))); + .extracting(ExtRequestTargeting::getIncludewinners) + .containsExactly(true); } @Test @@ -308,6 +306,30 @@ public void shouldReturnBidRequestWithIncludeWinnersFromStoredBidRequest() { .containsExactly(false); } + @Test + public void shouldReturnBidRequestWithIncludeFormatFromStoredBidRequest() { + // given + givenBidRequest( + builder -> builder + .ext(givenRequestExt( + ExtRequestTargeting.builder() + .pricegranularity(mapper.createObjectNode().put("foo", "bar")) + .includeformat(true) + .build())), + Imp.builder().build()); + + // when + final BidRequest request = factory.fromRequest(routingContext, 0L).result().getBidRequest(); + + // then + assertThat(singletonList(request)) + .extracting(BidRequest::getExt).isNotNull() + .extracting(ExtRequest::getPrebid) + .extracting(ExtRequestPrebid::getTargeting) + .extracting(ExtRequestTargeting::getIncludeformat) + .containsExactly(true); + } + @Test public void shouldReturnBidRequestWithDefaultIncludeBidderKeysIfStoredRequestExtTargetingHasNoIncludeBidderKeys() { // given diff --git a/src/test/java/org/prebid/server/auction/BidResponseCreatorTest.java b/src/test/java/org/prebid/server/auction/BidResponseCreatorTest.java index 424a3a8e578..f9f536ca465 100644 --- a/src/test/java/org/prebid/server/auction/BidResponseCreatorTest.java +++ b/src/test/java/org/prebid/server/auction/BidResponseCreatorTest.java @@ -980,7 +980,8 @@ public void shouldTruncateTargetingKeywordsByRequestPassedValue() { BigDecimal.valueOf(0.5)))))) .includewinners(true) .includebidderkeys(true) - .truncateattrchars(20) + .includeformat(false) + .truncateattrchars(20) .build())); final AuctionContext auctionContext = givenAuctionContext( bidRequest, @@ -1061,7 +1062,8 @@ public void shouldPopulateTargetingKeywordsFromMediaTypePriceGranularities() { null)) .includewinners(true) .includebidderkeys(true) - .build()))); + .includeformat(false) + .build()))); final Bid bid = Bid.builder().id("bidId").price(BigDecimal.valueOf(5.67)).impid("i1").build(); final List bidderResponses = singletonList(BidderResponse.of("bidder1", @@ -1455,7 +1457,8 @@ public void shouldNotPopulateWinningBidTargetingIfIncludeWinnersFlagIsFalse() { BigDecimal.valueOf(0.5)))))) .includewinners(false) .includebidderkeys(true) - .build()))); + .includeformat(false) + .build()))); final Bid bid = Bid.builder().id("bidId").price(BigDecimal.valueOf(5.67)).impid("i1").build(); final List bidderResponses = singletonList(BidderResponse.of("bidder1", @@ -1496,7 +1499,8 @@ public void shouldNotPopulateBidderKeysTargetingIfIncludeBidderKeysFlagIsFalse() BigDecimal.valueOf(0.5)))))) .includewinners(true) .includebidderkeys(false) - .build()))); + .includeformat(false) + .build()))); final Bid bid = Bid.builder().id("bidId").price(BigDecimal.valueOf(5.67)).impid("i1").build(); final List bidderResponses = singletonList(BidderResponse.of("bidder1", @@ -1956,6 +1960,7 @@ private static ExtRequestTargeting givenTargeting() { BigDecimal.valueOf(0.5)))))) .includewinners(true) .includebidderkeys(true) + .includeformat(false) .build(); } diff --git a/src/test/java/org/prebid/server/auction/FpdResolverTest.java b/src/test/java/org/prebid/server/auction/FpdResolverTest.java index bb0c6def595..46e9bec4fda 100644 --- a/src/test/java/org/prebid/server/auction/FpdResolverTest.java +++ b/src/test/java/org/prebid/server/auction/FpdResolverTest.java @@ -14,6 +14,7 @@ import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; import org.prebid.server.VertxTest; +import org.prebid.server.json.JsonMerger; import org.prebid.server.proto.openrtb.ext.request.ExtApp; import org.prebid.server.proto.openrtb.ext.request.ExtAppPrebid; import org.prebid.server.proto.openrtb.ext.request.ExtBidderConfig; @@ -40,7 +41,7 @@ public class FpdResolverTest extends VertxTest { @Before public void setUp() { - fpdResolver = new FpdResolver(jacksonMapper); + fpdResolver = new FpdResolver(jacksonMapper, new JsonMerger(jacksonMapper)); } @Test diff --git a/src/test/java/org/prebid/server/auction/OrtbTypesResolverTest.java b/src/test/java/org/prebid/server/auction/OrtbTypesResolverTest.java index 3071553c82b..89ef6ec9751 100644 --- a/src/test/java/org/prebid/server/auction/OrtbTypesResolverTest.java +++ b/src/test/java/org/prebid/server/auction/OrtbTypesResolverTest.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Test; import org.prebid.server.VertxTest; +import org.prebid.server.json.JsonMerger; import java.util.ArrayList; import java.util.List; @@ -13,7 +14,8 @@ public class OrtbTypesResolverTest extends VertxTest { - private final OrtbTypesResolver ortbTypesResolver = new OrtbTypesResolver(jacksonMapper); + private final OrtbTypesResolver ortbTypesResolver = + new OrtbTypesResolver(jacksonMapper, new JsonMerger(jacksonMapper)); @Test public void normalizeTargetingShouldNotChangeNodeIfItsTypeIsNotObject() { diff --git a/src/test/java/org/prebid/server/auction/StoredRequestProcessorTest.java b/src/test/java/org/prebid/server/auction/StoredRequestProcessorTest.java index 4eae1ad5459..fe1911813ec 100644 --- a/src/test/java/org/prebid/server/auction/StoredRequestProcessorTest.java +++ b/src/test/java/org/prebid/server/auction/StoredRequestProcessorTest.java @@ -9,6 +9,8 @@ import com.iab.openrtb.request.Imp.ImpBuilder; import com.iab.openrtb.request.Video; import io.vertx.core.Future; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.file.FileSystem; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -20,6 +22,7 @@ import org.prebid.server.exception.InvalidRequestException; import org.prebid.server.execution.Timeout; import org.prebid.server.execution.TimeoutFactory; +import org.prebid.server.json.JsonMerger; import org.prebid.server.metric.Metrics; import org.prebid.server.proto.openrtb.ext.request.ExtImp; import org.prebid.server.proto.openrtb.ext.request.ExtImpPrebid; @@ -50,6 +53,7 @@ import static org.assertj.core.api.Assertions.entry; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.verify; @@ -57,9 +61,13 @@ public class StoredRequestProcessorTest extends VertxTest { + private static final int DEFAULT_TIMEOUT = 500; + @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); + @Mock + private FileSystem fileSystem; @Mock private ApplicationSettings applicationSettings; @Mock @@ -70,12 +78,15 @@ public class StoredRequestProcessorTest extends VertxTest { @Before public void setUp() { final TimeoutFactory timeoutFactory = new TimeoutFactory(Clock.fixed(Instant.now(), ZoneId.systemDefault())); - storedRequestProcessor = new StoredRequestProcessor( - 500, + storedRequestProcessor = StoredRequestProcessor.create( + DEFAULT_TIMEOUT, + null, + fileSystem, applicationSettings, metrics, timeoutFactory, - jacksonMapper); + jacksonMapper, + new JsonMerger(jacksonMapper)); } @Test @@ -156,6 +167,52 @@ public void shouldReturnMergedBidRequest() throws IOException { .build()); } + @Test + public void shouldReturnMergedDefaultAndBidRequest() throws IOException { + // given + given(fileSystem.readFileBlocking(anyString())) + .willReturn(Buffer.buffer(mapper.writeValueAsString(BidRequest.builder().at(1).build()))); + + final TimeoutFactory timeoutFactory = new TimeoutFactory(Clock.fixed(Instant.now(), ZoneId.systemDefault())); + storedRequestProcessor = StoredRequestProcessor.create( + DEFAULT_TIMEOUT, + "path/to/default/request.json", + fileSystem, + applicationSettings, + metrics, + timeoutFactory, + jacksonMapper, + new JsonMerger(jacksonMapper)); + + final BidRequest bidRequest = givenBidRequest(builder -> builder + .ext(ExtRequest.of(ExtRequestPrebid.builder() + .storedrequest(ExtStoredRequest.of("123")) + .build()))); + + final String storedRequestBidRequestJson = mapper.writeValueAsString(BidRequest.builder() + .id("test-request-id") + .tmax(1000L) + .imp(singletonList(Imp.builder().build())) + .build()); + given(applicationSettings.getStoredData(any(), anySet(), anySet(), any())) + .willReturn(Future.succeededFuture( + StoredDataResult.of(singletonMap("123", storedRequestBidRequestJson), emptyMap(), + emptyList()))); + + // when + final Future bidRequestFuture = storedRequestProcessor.processStoredRequests(null, bidRequest); + + // then + assertThat(bidRequestFuture.succeeded()).isTrue(); + assertThat(bidRequestFuture.result()).isEqualTo(BidRequest.builder() + .id("test-request-id") + .at(1) + .tmax(1000L) + .imp(singletonList(Imp.builder().build())) + .ext(ExtRequest.of(ExtRequestPrebid.builder().storedrequest(ExtStoredRequest.of("123")).build())) + .build()); + } + @Test public void shouldReturnAmpRequest() throws IOException { // given @@ -174,6 +231,39 @@ public void shouldReturnAmpRequest() throws IOException { .build()); } + @Test + public void shouldReturnMergedDefaultAndAmpRequest() throws IOException { + // given + given(fileSystem.readFileBlocking(anyString())) + .willReturn(Buffer.buffer(mapper.writeValueAsString(BidRequest.builder().at(1).build()))); + + final TimeoutFactory timeoutFactory = new TimeoutFactory(Clock.fixed(Instant.now(), ZoneId.systemDefault())); + storedRequestProcessor = StoredRequestProcessor.create( + DEFAULT_TIMEOUT, + "path/to/default/request.json", + fileSystem, + applicationSettings, + metrics, + timeoutFactory, + jacksonMapper, + new JsonMerger(jacksonMapper)); + + given(applicationSettings.getAmpStoredData(any(), anySet(), anySet(), any())) + .willReturn(Future.succeededFuture(StoredDataResult.of( + singletonMap("123", mapper.writeValueAsString( + BidRequest.builder().id("test-request-id").build())), emptyMap(), emptyList()))); + + // when + final Future bidRequestFuture = storedRequestProcessor.processAmpRequest(null, "123"); + + // then + assertThat(bidRequestFuture.succeeded()).isTrue(); + assertThat(bidRequestFuture.result()).isEqualTo(BidRequest.builder() + .id("test-request-id") + .at(1) + .build()); + } + @Test public void shouldReturnFailedFutureWhenStoredBidRequestJsonIsNotValid() { // given diff --git a/src/test/java/org/prebid/server/auction/TargetingKeywordsCreatorTest.java b/src/test/java/org/prebid/server/auction/TargetingKeywordsCreatorTest.java index 48860242eb9..773006fd3f8 100644 --- a/src/test/java/org/prebid/server/auction/TargetingKeywordsCreatorTest.java +++ b/src/test/java/org/prebid/server/auction/TargetingKeywordsCreatorTest.java @@ -8,6 +8,7 @@ import org.prebid.server.proto.openrtb.ext.request.ExtGranularityRange; import org.prebid.server.proto.openrtb.ext.request.ExtPriceGranularity; import org.prebid.server.proto.response.Bid; +import org.prebid.server.proto.response.MediaType; import java.math.BigDecimal; import java.util.Map; @@ -29,7 +30,8 @@ public class TargetingKeywordsCreatorTest { @Test public void shouldReturnTargetingKeywordsForOrdinaryBid() { // given - final Bid bid = Bid.builder().bidder("bidder1").price(BigDecimal.ONE).dealId("dealId1").cacheId("cacheId1") + final Bid bid = Bid.builder().bidder("bidder1").price(BigDecimal.ONE).dealId("dealId1") + .mediaType(MediaType.banner).cacheId("cacheId1") .width(50).height(100).build(); // when @@ -40,6 +42,7 @@ public void shouldReturnTargetingKeywordsForOrdinaryBid() { true, true, false, + false, 0, null, null, @@ -69,11 +72,12 @@ public void shouldReturnTargetingKeywordsForOrdinaryBidOpenrtb() { true, true, false, + false, 0, null, null, null) - .makeFor(bid, "bidder1", false, null, null); + .makeFor(bid, "bidder1", false, null, null, null); // then assertThat(keywords).containsOnly( @@ -87,7 +91,7 @@ public void shouldReturnTargetingKeywordsForOrdinaryBidOpenrtb() { public void shouldReturnTargetingKeywordsWithEntireKeys() { // given final Bid bid = Bid.builder().bidder("veryververyverylongbidder1").price(BigDecimal.ONE).dealId("dealId1") - .cacheId("cacheId1").width(50).height(100).build(); + .cacheId("cacheId1").mediaType(MediaType.banner).width(50).height(100).build(); // when final Map keywords = TargetingKeywordsCreator.create( @@ -97,6 +101,7 @@ public void shouldReturnTargetingKeywordsWithEntireKeys() { true, true, false, + false, 0, null, null, @@ -126,11 +131,12 @@ public void shouldReturnTargetingKeywordsWithEntireKeysOpenrtb() { true, true, false, + false, 0, null, null, null) - .makeFor(bid, "veryververyverylongbidder1", false, null, null); + .makeFor(bid, "veryververyverylongbidder1", false, null, null, null); // then assertThat(keywords).containsOnly( @@ -148,7 +154,7 @@ public void shouldReturnTargetingKeywordsForWinningBid() { .price(BigDecimal.ONE) .dealId("dealId1") .cacheId("cacheId1") - .width(50) + .mediaType(MediaType.banner).width(50) .height(100) .build(); @@ -159,6 +165,7 @@ public void shouldReturnTargetingKeywordsForWinningBid() { singletonList(ExtGranularityRange.of(BigDecimal.valueOf(5), BigDecimal.valueOf(0.5)))), true, true, + true, false, 0, null, @@ -177,7 +184,9 @@ public void shouldReturnTargetingKeywordsForWinningBid() { entry("hb_bidder", "bidder1"), entry("hb_cache_id", "cacheId1"), entry("hb_size", "50x100"), - entry("hb_deal", "dealId1")); + entry("hb_deal", "dealId1"), + entry("hb_format", "banner"), + entry("hb_format_bidder1", "banner")); } @Test @@ -197,12 +206,13 @@ public void shouldReturnTargetingKeywordsForWinningBidOpenrtb() { singletonList(ExtGranularityRange.of(BigDecimal.valueOf(5), BigDecimal.valueOf(0.5)))), true, true, + true, false, 0, null, null, null) - .makeFor(bid, "bidder1", true, "cacheId1", "videoCacheId1"); + .makeFor(bid, "bidder1", true, "cacheId1", "banner", "videoCacheId1"); // then assertThat(keywords).containsOnly( @@ -217,30 +227,46 @@ public void shouldReturnTargetingKeywordsForWinningBidOpenrtb() { entry("hb_cache_id", "cacheId1"), entry("hb_cache_id_bidder1", "cacheId1"), entry("hb_uuid", "videoCacheId1"), - entry("hb_uuid_bidder1", "videoCacheId1")); + entry("hb_uuid_bidder1", "videoCacheId1"), + entry("hb_format", "banner"), + entry("hb_format_bidder1", "banner")); } @Test public void shouldFallbackToDefaultPriceIfInvalidPriceGranularity() { // given - final Bid bid = Bid.builder().bidder("").price(BigDecimal.valueOf(3.87)).build(); + final Bid bid = Bid.builder().bidder("").price(BigDecimal.valueOf(3.87)).mediaType(MediaType.banner).build(); // when final Map keywords = TargetingKeywordsCreator.create( - "invalid", true, true, false, 0) + "invalid", true, true, false, false, 0) .makeFor(bid, true); // then assertThat(keywords).contains(entry("hb_pb", StringUtils.EMPTY)); } + @Test + public void shouldIncludeFormat() { + // given + final Bid bid = Bid.builder().bidder("").price(BigDecimal.valueOf(3.87)).mediaType(MediaType.banner).build(); + + // when + final Map keywords = TargetingKeywordsCreator.create( + "invalid", true, true, true, false, 0) + .makeFor(bid, true); + + // then + assertThat(keywords).contains(entry("hb_format", "banner")); + } + @Test public void shouldUseDefaultPriceGranularity() { // given - final Bid bid = Bid.builder().bidder("").price(BigDecimal.valueOf(3.87)).build(); + final Bid bid = Bid.builder().bidder("").price(BigDecimal.valueOf(3.87)).mediaType(MediaType.banner).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, 0) + final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, false, 0) .makeFor(bid, true); // then @@ -254,20 +280,34 @@ public void shouldUseDefaultPriceGranularityOpenrtb() { .price(BigDecimal.valueOf(3.87)).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, 0) - .makeFor(bid, "", true, null, null); + final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, false, 0) + .makeFor(bid, "", true, null, null, null); // then assertThat(keywords).contains(entry("hb_pb", "3.80")); } + @Test + public void shouldIncludeFormatOpenrtb() { + // given + final com.iab.openrtb.response.Bid bid = com.iab.openrtb.response.Bid.builder() + .price(BigDecimal.valueOf(3.87)).build(); + + // when + final Map keywords = TargetingKeywordsCreator.create(null, true, true, true, false, 0) + .makeFor(bid, "", true, null, "banner", null); + + // then + assertThat(keywords).contains(entry("hb_format", "banner")); + } + @Test public void shouldNotIncludeCacheIdAndDealIdAndSize() { // given - final Bid bid = Bid.builder().bidder("bidder").price(BigDecimal.ONE).build(); + final Bid bid = Bid.builder().bidder("bidder").mediaType(MediaType.banner).price(BigDecimal.ONE).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, 0) + final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, false, 0) .makeFor(bid, true); // then @@ -281,8 +321,8 @@ public void shouldNotIncludeCacheIdAndDealIdAndSizeOpenrtb() { final com.iab.openrtb.response.Bid bid = com.iab.openrtb.response.Bid.builder().price(BigDecimal.ONE).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, 0) - .makeFor(bid, "bidder", true, null, null); + final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, false, 0) + .makeFor(bid, "bidder", true, null, null, null); // then assertThat(keywords).doesNotContainKeys("hb_cache_id_bidder", "hb_deal_bidder", "hb_size_bidder", @@ -292,10 +332,10 @@ public void shouldNotIncludeCacheIdAndDealIdAndSizeOpenrtb() { @Test public void shouldReturnEnvKeyForAppRequest() { // given - final Bid bid = Bid.builder().bidder("bidder").price(BigDecimal.ONE).build(); + final Bid bid = Bid.builder().bidder("bidder").mediaType(MediaType.banner).price(BigDecimal.ONE).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, true, true, true, 0) + final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, true, 0) .makeFor(bid, true); // then @@ -310,8 +350,8 @@ public void shouldReturnEnvKeyForAppRequestOpenrtb() { final com.iab.openrtb.response.Bid bid = com.iab.openrtb.response.Bid.builder().price(BigDecimal.ONE).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, true, true, true, 0) - .makeFor(bid, "bidder", true, null, null); + final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, true, 0) + .makeFor(bid, "bidder", true, null, null, null); // then assertThat(keywords).contains( @@ -325,8 +365,8 @@ public void shouldNotIncludeWinningBidTargetingIfIncludeWinnersFlagIsFalse() { final com.iab.openrtb.response.Bid bid = com.iab.openrtb.response.Bid.builder().price(BigDecimal.ONE).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, false, true, false, 0) - .makeFor(bid, "bidder1", true, null, null); + final Map keywords = TargetingKeywordsCreator.create(null, false, true, false, false, 0) + .makeFor(bid, "bidder1", true, null, null, null); // then assertThat(keywords).doesNotContainKeys("hb_bidder", "hb_pb"); @@ -338,8 +378,8 @@ public void shouldIncludeWinningBidTargetingIfIncludeWinnersFlagIsTrue() { final com.iab.openrtb.response.Bid bid = com.iab.openrtb.response.Bid.builder().price(BigDecimal.ONE).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, 0) - .makeFor(bid, "bidder1", true, null, null); + final Map keywords = TargetingKeywordsCreator.create(null, true, true, false, false, 0) + .makeFor(bid, "bidder1", true, null, null, null); // then assertThat(keywords).containsKeys("hb_bidder", "hb_pb"); @@ -351,8 +391,8 @@ public void shouldNotIncludeBidderKeysTargetingIfIncludeBidderKeysFlagIsFalse() final com.iab.openrtb.response.Bid bid = com.iab.openrtb.response.Bid.builder().price(BigDecimal.ONE).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, false, false, false, 0) - .makeFor(bid, "bidder1", true, null, null); + final Map keywords = TargetingKeywordsCreator.create(null, false, false, false, false, 0) + .makeFor(bid, "bidder1", true, null, null, null); // then assertThat(keywords).doesNotContainKeys("hb_bidder_bidder1", "hb_pb_bidder1"); @@ -364,8 +404,8 @@ public void shouldIncludeBidderKeysTargetingIfIncludeBidderKeysFlagIsTrue() { final com.iab.openrtb.response.Bid bid = com.iab.openrtb.response.Bid.builder().price(BigDecimal.ONE).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, false, true, false, 0) - .makeFor(bid, "bidder1", true, null, null); + final Map keywords = TargetingKeywordsCreator.create(null, false, true, false, false, 0) + .makeFor(bid, "bidder1", true, null, null, null); // then assertThat(keywords).containsKeys("hb_bidder_bidder1", "hb_pb_bidder1"); @@ -377,8 +417,8 @@ public void shouldTruncateTargetingBidderKeywordsIfTruncateAttrCharsIsDefined() final com.iab.openrtb.response.Bid bid = com.iab.openrtb.response.Bid.builder().price(BigDecimal.ONE).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, false, true, false, 20) - .makeFor(bid, "someVeryLongBidderName", true, null, null); + final Map keywords = TargetingKeywordsCreator.create(null, false, true, false, false, 20) + .makeFor(bid, "someVeryLongBidderName", true, null, null, null); // then assertThat(keywords).hasSize(2) @@ -391,8 +431,8 @@ public void shouldTruncateTargetingWithoutBidderSuffixKeywordsIfTruncateAttrChar final com.iab.openrtb.response.Bid bid = com.iab.openrtb.response.Bid.builder().price(BigDecimal.ONE).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, true, false, false, 7) - .makeFor(bid, "bidder", true, null, null); + final Map keywords = TargetingKeywordsCreator.create(null, true, false, false, false, 7) + .makeFor(bid, "bidder", true, null, null, null); // then assertThat(keywords).hasSize(2) @@ -405,8 +445,8 @@ public void shouldNotTruncateTargetingKeywordsIfTruncateAttrCharsIsNotDefined() final com.iab.openrtb.response.Bid bid = com.iab.openrtb.response.Bid.builder().price(BigDecimal.ONE).build(); // when - final Map keywords = TargetingKeywordsCreator.create(null, false, true, false, 0) - .makeFor(bid, "someVeryLongBidderName", true, null, null); + final Map keywords = TargetingKeywordsCreator.create(null, false, true, false, false, 0) + .makeFor(bid, "someVeryLongBidderName", true, null, null, null); // then assertThat(keywords).hasSize(2) @@ -432,11 +472,12 @@ public void shouldTruncateKeysFromResolver() { true, true, false, + false, 20, null, null, resolver) - .makeFor(bid, "bidder1", true, null, null); + .makeFor(bid, "bidder1", true, null, null, null); // then assertThat(keywords).contains(entry("key_longer_than_twen", "value1")); @@ -461,11 +502,12 @@ public void shouldIncludeKeywordsFromResolver() { true, true, false, + false, 0, null, null, resolver) - .makeFor(bid, "bidder1", true, null, null); + .makeFor(bid, "bidder1", true, null, null, null); // then assertThat(keywords).contains(entry("keyword1", "value1")); diff --git a/src/test/java/org/prebid/server/auction/VideoResponseFactoryTest.java b/src/test/java/org/prebid/server/auction/VideoResponseFactoryTest.java index 6916a5b9e10..1b78d21ef6e 100644 --- a/src/test/java/org/prebid/server/auction/VideoResponseFactoryTest.java +++ b/src/test/java/org/prebid/server/auction/VideoResponseFactoryTest.java @@ -35,9 +35,9 @@ public void setUp() { public void shouldReturnExpectedVideoResponse() { // given final Map targeting = new HashMap<>(); - targeting.put("hb_uuid", "value1"); - targeting.put("hb_pb", "hb_pb"); - targeting.put("hb_pb_cat_dur", "hb_pb_cat_dur"); + targeting.put("hb_uuid_appnexus", "hb_uuidVal"); + targeting.put("hb_pb_appnexus", "hb_pbVal"); + targeting.put("hb_pb_cat_dur_appnexus", "hb_pb_cat_durVal"); final Bid bid0 = Bid.builder() .impid("0_0") @@ -73,9 +73,9 @@ public void shouldReturnExpectedVideoResponse() { // then final ExtAdPod expectedExtAdPod0 = ExtAdPod.of(0, - singletonList(ExtResponseVideoTargeting.of("hb_pb", "hb_pb_cat_dur", "value1")), null); + singletonList(ExtResponseVideoTargeting.of("hb_pbVal", "hb_pb_cat_durVal", "hb_uuidVal")), null); final ExtAdPod expectedExtAdPod1 = ExtAdPod.of( - 1, singletonList(ExtResponseVideoTargeting.of("hb_pb", "hb_pb_cat_dur", "value1")), null); + 1, singletonList(ExtResponseVideoTargeting.of("hb_pbVal", "hb_pb_cat_durVal", "hb_uuidVal")), null); final ExtAdPod expectedErrorExtAdPod3 = ExtAdPod.of(3, null, singletonList("Error")); final List expectedAdPodResponse = Arrays.asList(expectedExtAdPod0, expectedExtAdPod1, expectedErrorExtAdPod3); diff --git a/src/test/java/org/prebid/server/auction/VideoStoredRequestProcessorTest.java b/src/test/java/org/prebid/server/auction/VideoStoredRequestProcessorTest.java index c1eb0f71069..ca604d7b3cd 100644 --- a/src/test/java/org/prebid/server/auction/VideoStoredRequestProcessorTest.java +++ b/src/test/java/org/prebid/server/auction/VideoStoredRequestProcessorTest.java @@ -1,5 +1,6 @@ package org.prebid.server.auction; +import com.fasterxml.jackson.core.JsonProcessingException; import com.iab.openrtb.request.BidRequest; import com.iab.openrtb.request.Content; import com.iab.openrtb.request.Imp; @@ -12,6 +13,8 @@ import com.iab.openrtb.request.video.PodError; import com.iab.openrtb.request.video.Podconfig; import io.vertx.core.Future; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.file.FileSystem; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -22,6 +25,7 @@ import org.prebid.server.auction.model.WithPodErrors; import org.prebid.server.exception.InvalidRequestException; import org.prebid.server.execution.TimeoutFactory; +import org.prebid.server.json.JsonMerger; import org.prebid.server.metric.Metrics; import org.prebid.server.proto.openrtb.ext.ExtIncludeBrandCategory; import org.prebid.server.proto.openrtb.ext.request.ExtRequest; @@ -46,6 +50,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.given; import static org.mockito.Mockito.never; @@ -58,6 +63,8 @@ public class VideoStoredRequestProcessorTest extends VertxTest { @Rule public final MockitoRule mockitoRule = MockitoJUnit.rule(); + @Mock + private FileSystem fileSystem; @Mock private ApplicationSettings applicationSettings; @Mock @@ -72,19 +79,24 @@ public class VideoStoredRequestProcessorTest extends VertxTest { private VideoStoredRequestProcessor target; @Before - public void setUp() { - target = new VideoStoredRequestProcessor( + public void setUp() throws JsonProcessingException { + given(fileSystem.readFileBlocking(anyString())) + .willReturn(Buffer.buffer(mapper.writeValueAsString(BidRequest.builder().at(1).build()))); + + target = VideoStoredRequestProcessor.create( false, emptyList(), 2000L, "USD", - BidRequest.builder().build(), - validator, + "path/to/default/request.json", + fileSystem, applicationSettings, + validator, metrics, timeoutFactory, timeoutResolver, - jacksonMapper); + jacksonMapper, + new JsonMerger(jacksonMapper)); } @Test @@ -171,6 +183,7 @@ public void shouldReturnFutureWithMergedStoredAndDefaultRequest() { .build(); final BidRequest expectedMergedRequest = BidRequest.builder() .id("bid_id") + .at(1) .imp(Arrays.asList(expectedImp1, expectedImp2)) .user(User.builder().buyeruid("appnexus").yob(123).gender("gender").keywords("keywords").build()) .site(Site.builder().id("siteId").content(content).build()) diff --git a/src/test/java/org/prebid/server/bidder/rubicon/RubiconBidderTest.java b/src/test/java/org/prebid/server/bidder/rubicon/RubiconBidderTest.java index 466256bc8ae..01e2a6306f6 100644 --- a/src/test/java/org/prebid/server/bidder/rubicon/RubiconBidderTest.java +++ b/src/test/java/org/prebid/server/bidder/rubicon/RubiconBidderTest.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; import com.iab.openrtb.request.App; +import com.iab.openrtb.request.Audio; import com.iab.openrtb.request.Banner; import com.iab.openrtb.request.BidRequest; import com.iab.openrtb.request.BidRequest.BidRequestBuilder; @@ -13,6 +14,7 @@ import com.iab.openrtb.request.Imp; import com.iab.openrtb.request.Imp.ImpBuilder; import com.iab.openrtb.request.Metric; +import com.iab.openrtb.request.Native; import com.iab.openrtb.request.Publisher; import com.iab.openrtb.request.Regs; import com.iab.openrtb.request.Site; @@ -138,6 +140,49 @@ public void makeHttpRequestsShouldFillMethodAndUrlAndExpectedHeaders() { tuple(HttpUtil.USER_AGENT_HEADER.toString(), "prebid-server/1.0")); } + @Test + public void makeHttpRequestsShouldFilterImpressionsWithInvalidTypes() { + // given + final Imp imp1 = givenImp(builder -> builder.video(Video.builder().build())); + final Imp imp2 = givenImp(builder -> builder.id("2").xNative(Native.builder().build())); + final Imp imp3 = givenImp(builder -> builder.id("3").audio(Audio.builder().build())); + final BidRequest bidRequest = BidRequest.builder().imp(asList(imp1, imp2, imp3)).build(); + + // when + final Result>> result = rubiconBidder.makeHttpRequests(bidRequest); + + // then + assertThat(result.getErrors()).hasSize(2) + .containsOnly( + BidderError.of("Impression with id 2 rejected with invalid type `xNative`." + + " Allowed types are banner and video.", BidderError.Type.bad_input), + BidderError.of("Impression with id 3 rejected with invalid type `audio`." + + " Allowed types are banner and video.", BidderError.Type.bad_input)); + assertThat(result.getValue()).hasSize(1); + } + + @Test + public void makeHttpRequestsShouldFilterAllImpressionsAndReturnErrorMeessagesWithoutRequests() { + // given + final Imp imp1 = givenImp(builder -> builder.id("1").xNative(Native.builder().build())); + final Imp imp2 = givenImp(builder -> builder.id("2").audio(Audio.builder().build())); + final BidRequest bidRequest = BidRequest.builder().imp(asList(imp1, imp2)).build(); + + // when + final Result>> result = rubiconBidder.makeHttpRequests(bidRequest); + + // then + assertThat(result.getErrors()).hasSize(3) + .containsOnly( + BidderError.of("Impression with id 1 rejected with invalid type `xNative`." + + " Allowed types are banner and video.", BidderError.Type.bad_input), + BidderError.of("Impression with id 2 rejected with invalid type `audio`." + + " Allowed types are banner and video.", BidderError.Type.bad_input), + BidderError.of("There are no valid impressions to create bid request to rubicon bidder", + BidderError.Type.bad_input)); + assertThat(result.getValue()).isEmpty(); + } + @Test public void makeHttpRequestsShouldReplaceDefaultParametersWithExtPrebidBiddersBidder() { // given @@ -1929,6 +1974,7 @@ public void makeHttpRequestsShouldReturnErrorIfImpExtCouldNotBeParsed() { .imp(asList( givenImp(builder -> builder.video(Video.builder().build())), Imp.builder() + .banner(Banner.builder().build()) .ext(mapper.valueToTree(ExtPrebid.of(null, mapper.createArrayNode()))) .build())) .build(); diff --git a/src/test/java/org/prebid/server/handler/AuctionHandlerTest.java b/src/test/java/org/prebid/server/handler/AuctionHandlerTest.java index f8e71396e93..c53ed71eb3a 100644 --- a/src/test/java/org/prebid/server/handler/AuctionHandlerTest.java +++ b/src/test/java/org/prebid/server/handler/AuctionHandlerTest.java @@ -569,7 +569,7 @@ public void shouldIncrementCommonMetrics() { // then verify(metrics).updateRequestTypeMetric(eq(MetricName.legacy), eq(MetricName.ok)); - verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(true), anyBoolean(), anyBoolean(), eq(1)); + verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(true), anyBoolean(), eq(1)); verify(metrics).updateImpTypesMetrics(singletonMap("banner", 1L)); verify(metrics).updateAccountRequestMetrics(eq("accountId"), eq(MetricName.legacy)); verify(metrics).updateRequestTimeMetric(anyLong()); @@ -594,7 +594,7 @@ public void shouldIncrementNoBidMetrics() { } @Test - public void shouldIncrementSafariAndNoCookieMetrics() { + public void shouldIncrementNoCookieMetrics() { // given givenPreBidRequestContext(identity(), builder -> builder.noLiveUids(true)); @@ -605,7 +605,7 @@ public void shouldIncrementSafariAndNoCookieMetrics() { auctionHandler.handle(routingContext); // then - verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(false), eq(false), eq(true), anyInt()); + verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(false), eq(false), anyInt()); } @Test diff --git a/src/test/java/org/prebid/server/handler/openrtb2/AmpHandlerTest.java b/src/test/java/org/prebid/server/handler/openrtb2/AmpHandlerTest.java index 32513a625f4..f1992b0110b 100644 --- a/src/test/java/org/prebid/server/handler/openrtb2/AmpHandlerTest.java +++ b/src/test/java/org/prebid/server/handler/openrtb2/AmpHandlerTest.java @@ -471,7 +471,7 @@ public void shouldIncrementAppRequestMetrics() { ampHandler.handle(routingContext); // then - verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(true), anyBoolean(), anyBoolean(), anyInt()); + verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(true), anyBoolean(), anyInt()); } @Test @@ -493,7 +493,7 @@ public void shouldIncrementNoCookieMetrics() { ampHandler.handle(routingContext); // then - verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(false), eq(false), eq(true), anyInt()); + verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(false), eq(false), anyInt()); } @Test @@ -511,7 +511,7 @@ public void shouldIncrementImpsRequestedMetrics() { ampHandler.handle(routingContext); // then - verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(anyBoolean(), anyBoolean(), anyBoolean(), eq(1)); + verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(anyBoolean(), anyBoolean(), eq(1)); } @Test diff --git a/src/test/java/org/prebid/server/handler/openrtb2/AuctionHandlerTest.java b/src/test/java/org/prebid/server/handler/openrtb2/AuctionHandlerTest.java index cfca6fb8d69..a355efbd4f0 100644 --- a/src/test/java/org/prebid/server/handler/openrtb2/AuctionHandlerTest.java +++ b/src/test/java/org/prebid/server/handler/openrtb2/AuctionHandlerTest.java @@ -369,7 +369,7 @@ public void shouldIncrementAppRequestMetrics() { auctionHandler.handle(routingContext); // then - verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(true), anyBoolean(), anyBoolean(), anyInt()); + verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(true), anyBoolean(), anyInt()); } @Test @@ -390,7 +390,7 @@ public void shouldIncrementNoCookieMetrics() { auctionHandler.handle(routingContext); // then - verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(false), eq(false), eq(true), anyInt()); + verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(eq(false), eq(false), anyInt()); } @Test @@ -407,7 +407,7 @@ public void shouldIncrementImpsRequestedMetrics() { auctionHandler.handle(routingContext); // then - verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(anyBoolean(), anyBoolean(), anyBoolean(), eq(1)); + verify(metrics).updateAppAndNoCookieAndImpsRequestedMetrics(anyBoolean(), anyBoolean(), eq(1)); } @Test diff --git a/src/test/java/org/prebid/server/util/JsonMergeUtilTest.java b/src/test/java/org/prebid/server/json/JsonMergerTest.java similarity index 94% rename from src/test/java/org/prebid/server/util/JsonMergeUtilTest.java rename to src/test/java/org/prebid/server/json/JsonMergerTest.java index b355617bfad..7ac8e4432f9 100644 --- a/src/test/java/org/prebid/server/util/JsonMergeUtilTest.java +++ b/src/test/java/org/prebid/server/json/JsonMergerTest.java @@ -1,4 +1,4 @@ -package org.prebid.server.util; +package org.prebid.server.json; import com.fasterxml.jackson.databind.node.ObjectNode; import com.iab.openrtb.request.App; @@ -11,13 +11,13 @@ import static org.assertj.core.api.Assertions.assertThat; -public class JsonMergeUtilTest extends VertxTest { +public class JsonMergerTest extends VertxTest { - private JsonMergeUtil target; + private JsonMerger target; @Before public void setUp() { - target = new JsonMergeUtil(jacksonMapper); + target = new JsonMerger(jacksonMapper); } @Test diff --git a/src/test/java/org/prebid/server/metric/MetricsTest.java b/src/test/java/org/prebid/server/metric/MetricsTest.java index 175281c139d..a47b0dea119 100644 --- a/src/test/java/org/prebid/server/metric/MetricsTest.java +++ b/src/test/java/org/prebid/server/metric/MetricsTest.java @@ -341,29 +341,17 @@ public void forCircuitBreakerShouldReturnCircuitBreakerMetricsConfiguredWithId() assertThat(metricRegistry.gauge("circuit-breaker.db.opened.count", () -> null).getValue()).isEqualTo(1L); } - @Test - public void updateSafariRequestsMetricShouldIncrementMetric() { - // when - metrics.updateSafariRequestsMetric(true); - metrics.updateSafariRequestsMetric(false); - - // then - assertThat(metricRegistry.counter("safari_requests").getCount()).isOne(); - } - @Test public void updateAppAndNoCookieAndImpsRequestedMetricsShouldIncrementMetrics() { // when - metrics.updateAppAndNoCookieAndImpsRequestedMetrics(true, false, false, 1); - metrics.updateAppAndNoCookieAndImpsRequestedMetrics(false, false, false, 2); - metrics.updateAppAndNoCookieAndImpsRequestedMetrics(false, false, true, 1); - metrics.updateAppAndNoCookieAndImpsRequestedMetrics(false, true, false, 1); + metrics.updateAppAndNoCookieAndImpsRequestedMetrics(true, false, 1); + metrics.updateAppAndNoCookieAndImpsRequestedMetrics(false, false, 2); + metrics.updateAppAndNoCookieAndImpsRequestedMetrics(false, true, 1); // then assertThat(metricRegistry.counter("app_requests").getCount()).isOne(); - assertThat(metricRegistry.counter("no_cookie_requests").getCount()).isEqualTo(2); - assertThat(metricRegistry.counter("safari_no_cookie_requests").getCount()).isOne(); - assertThat(metricRegistry.counter("imps_requested").getCount()).isEqualTo(5); + assertThat(metricRegistry.counter("no_cookie_requests").getCount()).isOne(); + assertThat(metricRegistry.counter("imps_requested").getCount()).isEqualTo(4); } @Test diff --git a/src/test/java/org/prebid/server/util/HttpUtilTest.java b/src/test/java/org/prebid/server/util/HttpUtilTest.java index 77242c33ca2..889b402b4f6 100644 --- a/src/test/java/org/prebid/server/util/HttpUtilTest.java +++ b/src/test/java/org/prebid/server/util/HttpUtilTest.java @@ -26,16 +26,6 @@ public class HttpUtilTest { @Mock private RoutingContext routingContext; - @Test - public void isSafariShouldReturnTrue() { - assertThat(HttpUtil.isSafari("Useragent with Safari browser and AppleWebKit built-in.")).isTrue(); - } - - @Test - public void isSafariShouldReturnFalse() { - assertThat(HttpUtil.isSafari("Useragent with Safari browser but Chromium forked by.")).isFalse(); - } - @Test public void validateUrlShouldFailOnInvalidUrl() { // when and then diff --git a/src/test/java/org/prebid/server/validation/RequestValidatorTest.java b/src/test/java/org/prebid/server/validation/RequestValidatorTest.java index b51c59e96d5..ee4e722de52 100644 --- a/src/test/java/org/prebid/server/validation/RequestValidatorTest.java +++ b/src/test/java/org/prebid/server/validation/RequestValidatorTest.java @@ -1653,7 +1653,7 @@ public void validateShouldReturnValidationMessageWhenEidUidIdIsMissing() { } @Test - public void validateShouldReturnValidationMessageWhenEidSourceIsNotUnique() { + public void validateShouldReturnErrorWhenEidSourceIsNotUnique() { // given final BidRequest bidRequest = validBidRequestBuilder() .user(User.builder() @@ -1671,8 +1671,7 @@ public void validateShouldReturnValidationMessageWhenEidSourceIsNotUnique() { final ValidationResult result = requestValidator.validate(bidRequest); // then - assertThat(result.getErrors()).hasSize(1) - .containsOnly("request.user.ext.eids must contain unique sources"); + assertThat(result.getErrors()).containsExactly("request.user.ext.eids must contain unique sources"); } @Test