diff --git a/docs/config-app.md b/docs/config-app.md index ffbb821b3b9..c9800108063 100644 --- a/docs/config-app.md +++ b/docs/config-app.md @@ -189,6 +189,26 @@ Also, each bidder could have its own bidder-specific options. - `admin-endpoints.logging-httpinteraction.on-application-port` - when equals to `false` endpoint will be bound to `admin.port`. - `admin-endpoints.logging-httpinteraction.protected` - when equals to `true` endpoint will be protected by basic authentication configured in `admin-endpoints.credentials` +- `admin-endpoints.tracelog.enabled` - if equals to `true` the endpoint will be available. +- `admin-endpoints.tracelog.path` - the server context path where the endpoint will be accessible. +- `admin-endpoints.tracelog.on-application-port` - when equals to `false` endpoint will be bound to `admin.port`. +- `admin-endpoints.tracelog.protected` - when equals to `true` endpoint will be protected by basic authentication configured in `admin-endpoints.credentials` + +- `admin-endpoints.deals-status.enabled` - if equals to `true` the endpoint will be available. +- `admin-endpoints.deals-status.path` - the server context path where the endpoint will be accessible. +- `admin-endpoints.deals-status.on-application-port` - when equals to `false` endpoint will be bound to `admin.port`. +- `admin-endpoints.deals-status.protected` - when equals to `true` endpoint will be protected by basic authentication configured in `admin-endpoints.credentials` + +- `admin-endpoints.lineitem-status.enabled` - if equals to `true` the endpoint will be available. +- `admin-endpoints.lineitem-status.path` - the server context path where the endpoint will be accessible. +- `admin-endpoints.lineitem-status.on-application-port` - when equals to `false` endpoint will be bound to `admin.port`. +- `admin-endpoints.lineitem-status.protected` - when equals to `true` endpoint will be protected by basic authentication configured in `admin-endpoints.credentials` + +- `admin-endpoints.e2eadmin.enabled` - if equals to `true` the endpoint will be available. +- `admin-endpoints.e2eadmin.path` - the server context path where the endpoint will be accessible. +- `admin-endpoints.e2eadmin.on-application-port` - when equals to `false` endpoint will be bound to `admin.port`. +- `admin-endpoints.e2eadmin.protected` - when equals to `true` endpoint will be protected by basic authentication configured in `admin-endpoints.credentials` + - `admin-endpoints.credentials` - user and password for access to admin endpoints if `admin-endpoints.[NAME].protected` is true`. ## Metrics @@ -394,3 +414,38 @@ If not defined in config all other Health Checkers would be disabled and endpoin - `analytics.pubstack.buffers.size-bytes` - threshold in bytes for buffer to send events. - `analytics.pubstack.buffers.count` - threshold in events count for buffer to send events - `analytics.pubstack.buffers.report-ttl-ms` - max period between two reports. + +## Programmatic Guaranteed Delivery +- `deals.planner.plan-endpoint` - planner endpoint to get plans from. +- `deals.planner.update-period` - cron expression to start job for requesting Line Item metadata updates from the Planner. +- `deals.planner.plan-advance-period` - cron expression to start job for advancing Line Items to the next plan. +- `deals.planner.retry-period-sec` - how long (in seconds) to wait before re-sending a request to the Planner that previously failed with 5xx HTTP error code. +- `deals.planner.timeout-ms` - default operation timeout for requests to planner's endpoints. +- `deals.planner.register-endpoint` - register endpoint to get plans from. +- `deals.planner.register-period-sec` - time period (in seconds) to send register request to the Planner. +- `deals.planner.username` - username for planner BasicAuth. +- `deals.planner.password` - password for planner BasicAuth. +- `deals.delivery-stats.delivery-period` - cron expression to start job for sending delivery progress to planner. +- `deals.delivery-stats.cached-reports-number` - how many reports to cache while planner is unresponsive. +- `deals.delivery-stats.timeout-ms` - default operation timeout for requests to delivery progress endpoints. +- `deals.delivery-stats.username` - username for delivery progress BasicAuth. +- `deals.delivery-stats.password` - password for delivery progress BasicAuth. +- `deals.delivery-stats.line-items-per-report` - max number of line items in each report to split for batching. Default is 25. +- `deals.delivery-stats.reports-interval-ms` - interval in ms between consecutive reports. Default is 0. +- `deals.delivery-stats.batches-interval-ms` - interval in ms between consecutive batches. Default is 1000. +- `deals.delivery-stats.request-compression-enabled` - enables request gzip compression when set to true. +- `deals.delivery-progress.line-item-status-ttl-sec` - how long to store line item's metrics after it was expired. +- `deals.delivery-progress.cached-plans-number` - how many plans to store in metrics per line item. +- `deals.delivery-progress.report-reset-period`- cron expression to start job for closing current delivery progress and starting new one. +- `deals.delivery-progress-report.competitors-number`- number of line items top competitors to send in delivery progress report. +- `deals.user-data.user-details-endpoint` - user Data Store endpoint to get user details from. +- `deals.user-data.win-event-endpoint` - user Data Store endpoint to which win events should be sent. +- `deals.user-data.timeout` - time to wait (in milliseconds) for User Data Service response. +- `deals.user-data.user-ids` - list of Rules for determining user identifiers to send to User Data Store. +- `deals.max-deals-per-bidder` - maximum number of deals to send to each bidder. +- `deals.alert-proxy.enabled` - enable alert proxy service if `true`. +- `deals.alert-proxy.url` - alert service endpoint to send alerts to. +- `deals.alert-proxy.timeout-sec` - default operation timeout for requests to alert service endpoint. +- `deals.alert-proxy.username` - username for alert proxy BasicAuth. +- `deals.alert-proxy.password` - password for alert proxy BasicAuth. +- `deals.alert-proxy.alert-types` - key value pair of alert type and sampling factor to send high priority alert. diff --git a/docs/deals.md b/docs/deals.md new file mode 100644 index 00000000000..fca8c585e26 --- /dev/null +++ b/docs/deals.md @@ -0,0 +1,152 @@ +# Deals + +## Planner and Register services + +### Planner service + +Periodically request Line Item metadata from the Planner. Line Item metadata includes: +1. Line Item details +2. Targeting +3. Frequency caps +4. Delivery schedule + +### Register service + +Each Prebid Server instance register itself with the General Planner with a health index +(QoS indicator based on its internal counters like circuit breaker trip counters, timeouts, etc.) +and KPI like ad requests per second. + +Also allows planner send command to PBS admin endpoint to stored request caches and tracelogs. + +### Planner and register service configuration + +```yaml +planner: + register-endpoint: + plan-endpoint: + update-period: "0 */1 * * * *" + register-period-sec: 60 + timeout-ms: 8000 + username: + password: +``` + +## Deals stats service + +Supports sending reports to delivery stats serving with following metrics: + +1. Number of client requests seen since start-up +2. For each Line Item +- Number of tokens spent so far at each token class within active and expired plans +- Number of times the account made requests (this will be the same across all LineItem for the account) +- Number of win notifications +- Number of times the domain part of the target matched +- Number of times impressions matched whole target +- Number of times impressions matched the target but was frequency capped +- Number of times impressions matched the target but the fcap lookup failed +- Number of times LineItem was sent to the bidder +- Number of times LineItem was sent to the bidder as the top match +- Number of times LineItem came back from the bidder +- Number of times the LineItem response was invalidated +- Number of times the LineItem was sent to the client +- Number of times the LineItem was sent to the client as the top match +- Array of top 10 competing LineItems sent to client + +### Deals stats service configuration + +```yaml +delivery-stats: + endpoint: + delivery-period: "0 */1 * * * *" + cached-reports-number: 20 + line-item-status-ttl-sec: 3600 + timeout-ms: 8000 + username: + password: +``` + +## Alert service + +Sends out alerts when PBS cannot talk to general planner and other critical situations. Alerts are simply JSON messages +over HTTP sent to a central proxy server. + +```yaml + alert-proxy: + enabled: truew + timeout-sec: 10 + url: + username: + password: + alert-types: + : + pbs-planner-empty-response-error: 15 +``` + +## GeoLocation service + +This service currently has 1 implementation: +- MaxMind + +In order to support targeting by geographical attributes the service will provide the following information: + +1. `continent` - Continent code +2. `region` - Region code using ISO-3166-2 +3. `metro` - Nielsen DMAs +4. `city` - city using provider specific encoding +5. `lat` - latitude from -90.0 to +90.0, where negative is south +6. `lon` - longitude from -180.0 to +180.0, where negative is west + +### GeoLocation service configuration for MaxMind + +```yaml +geolocation: + enabled: true + type: maxmind + maxmind: + remote-file-syncer: + download-url: + save-filepath: + tmp-filepath: + retry-count: 3 + retry-interval-ms: 3000 + timeout-ms: 300000 + update-interval-ms: 0 + http-client: + connect-timeout-ms: 2500 + max-redirects: 3 +``` + +## User Service + +This service is responsible for: +- Requesting user targeting segments and frequency capping status from the User Data Store +- Reporting to User Data Store when users finally see ads to aid in correctly enforcing frequency caps + +### User service configuration + +```yaml + user-data: + win-event-endpoint: + user-details-endpoint: + timeout: 1000 + user-ids: + - location: rubicon + source: uid + type: khaos +``` +1. khaos, adnxs - types of the ids that will be specified in requests to User Data Store +2. source - source of the id, the only supported value so far is “uids” which stands for uids cookie +3. location - where exactly in the source to look for id + +## Device Info Service + +DeviceInfoService returns device-related attributes based on User-Agent for use in targeting: +- deviceClass: desktop, tablet, phone, ctv +- os: windows, ios, android, osx, unix, chromeos +- osVersion +- browser: chrome, firefox, edge, safari +- browserVersion + +## See also + +- [Configuration](config.md) diff --git a/docs/endpoints/deals-status.md b/docs/endpoints/deals-status.md new file mode 100644 index 00000000000..eb8a234f4eb --- /dev/null +++ b/docs/endpoints/deals-status.md @@ -0,0 +1,10 @@ +# Deals Status +This endpoint is available on admin port called /pbs-admin/deals-status + +## `GET /pbs-admin/deals-status` + +Giving read-only access to current Line Items status, progress and aggregated metrics. + +### Sample request + +`GET http://prebid.site.com/pbs-admin/deals-status` \ No newline at end of file diff --git a/docs/endpoints/lineitem-status.md b/docs/endpoints/lineitem-status.md new file mode 100644 index 00000000000..756161adf71 --- /dev/null +++ b/docs/endpoints/lineitem-status.md @@ -0,0 +1,18 @@ +# Line Item status + +This endpoint is available on admin port called `/pbs-admin/lineitem-status`. + +Giving read-only access to defined in parameters line item. Contains information about active delivery schedule, +ready at timestamp, spent tokens number and pacing frequency in milliseconds. + +## `GET /pbs-admin/lineitem-status?id=` + +### Query parameters: + +This endpoint supports the following query parameters: + +`id` - line item id indicate a the line item about which information is needed. + +### Sample request + +`GET http://prebid.site.com/pbs-admin/lineitem-status?id=lineItemId1` \ No newline at end of file diff --git a/docs/endpoints/tracelog.md b/docs/endpoints/tracelog.md new file mode 100644 index 00000000000..47d94a4c943 --- /dev/null +++ b/docs/endpoints/tracelog.md @@ -0,0 +1,29 @@ +# Tracelog Endpoint + +This endpoint is available on admin port called `/pbs-admin/tracelog`. + +## POST `/pbs-admin/tracelog` + + Allows to configure logging level for specific account, line item and bidder code during some defined time period. + +### Query parameters: + +This endpoint supports the following query parameters: + + 1. `account` - specified an account for which logging level should be changed. (Not required, no default value) + 2. `lineItemId` - specified a lineItemId for which logging level should be changed. (Not required, no default value) + 3. `bidderCode`- specified a bidderCode for which logging level should be changed. (Not required, no default value) + 4. `level` - specified a log level to which logs should be updated. Allowed values are `info`, `warn`, `trace`, + `error`, `fatal`, `debug`. Default value if not defined is `error`. (Not required) + 5. `duration` - time in seconds during which changes will be applied. (Required). + +At least one of `account`, `lineItemId` or `bidderCode` should be specified. If more than one specified, +logic conjuction (and operation) is applied to parameters. + +### Request samples + +`GET http://prebid.site.com/pbs-admin/tracelog?account=1234&duration=100` - updates logging level to `error` level for account 1234 +for 100 seconds. + +`GET http://prebid.site.com/pbs-admin/tracelog?account=1234&bidder=rubicon&lineItemId=lineItemId1&level=debug&duration=100` - updates +logging level to warn, for account = 1234 and bidder = rubicon and lineItemId = lineItemId1 for 100 seconds. \ No newline at end of file diff --git a/docs/metrics.md b/docs/metrics.md index 95cf7a3c584..3354a4f45f3 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -135,3 +135,29 @@ Following metrics are collected and submitted if account is configured with `det - `analytics..(auction|amp|video|cookie_sync|event|setuid).timeout` - number of event requests, failed with timeout cause - `analytics..(auction|amp|video|cookie_sync|event|setuid).err` - number of event requests, failed with errors - `analytics..(auction|amp|video|cookie_sync|event|setuid).badinput` - number of event requests, rejected with bad input cause + +## win notifications +- `win_notifications` - total number of win notifications. +- `win_requests` - total number of requests sent to user service for win notifications. +- `win_request_preparation_failed` - number of request failed validation and were not sent. +- `win_request_time` - latency between request to user service and response for win notifications. +- `win_request_failed` - number of failed request sent to user service for win notifications. +- `win_request_successful` - number of successful request sent to user service for win notifications. + +## user details +- `user_details_requests` - total number of requests sent to user service to get user details. +- `user_details_request_preparation_failed` - number of request failed validation and were not sent. +- `user_details_request_time` - latency between request to user service and response to get user details. +- `user_details_request_failed` - number of failed request sent to user service to get user details. +- `user_details_request_successful` - number of successful request sent to user service to get user details. + +## Programmatic guaranteed metrics +- `pg.planner_lineitems_received` - number of line items received from general planner. +- `pg.planner_requests` - total number of requests sent to general planner. +- `pg.planner_request_failed` - number of failed request sent to general planner. +- `pg.planner_request_successful` - number of successful requests sent to general planner. +- `pg.planner_request_time` - latency between request to general planner and its successful (200 OK) response. +- `pg.delivery_requests` - total number of requests to delivery stats service. +- `pg.delivery_request_failed` - number of failed requests to delivery stats service. +- `pg.delivery_request_successful` - number of successful requests to delivery stats service. +- `pg.delivery_request_time` - latency between request to delivery stats and its successful (200 OK) response. diff --git a/pom.xml b/pom.xml index 661eeefe8dc..b9a8b6b33a3 100644 --- a/pom.xml +++ b/pom.xml @@ -55,6 +55,7 @@ 2.23.4 3.8.0 2.26.3 + 4.0.1 9.4.43.v20210629 3.0.6 1.4.196 @@ -88,6 +89,10 @@ spring-boot-starter ${spring.boot.version} + + org.springframework.boot + spring-boot-starter-aop + javax.annotation javax.annotation-api @@ -304,6 +309,12 @@ ${assertj.version} test + + org.awaitility + awaitility + ${awaitility.version} + test + org.springframework.boot spring-boot-starter-test diff --git a/src/main/java/org/prebid/server/analytics/model/NotificationEvent.java b/src/main/java/org/prebid/server/analytics/model/NotificationEvent.java index 85f5c813a98..49bf7fb7d50 100644 --- a/src/main/java/org/prebid/server/analytics/model/NotificationEvent.java +++ b/src/main/java/org/prebid/server/analytics/model/NotificationEvent.java @@ -17,6 +17,8 @@ public class NotificationEvent { Account account; + String lineItemId; + String bidder; Long timestamp; diff --git a/src/main/java/org/prebid/server/auction/BidResponseCreator.java b/src/main/java/org/prebid/server/auction/BidResponseCreator.java index 68cfff58a2a..0cb1204a81b 100644 --- a/src/main/java/org/prebid/server/auction/BidResponseCreator.java +++ b/src/main/java/org/prebid/server/auction/BidResponseCreator.java @@ -40,6 +40,8 @@ import org.prebid.server.cache.model.CacheInfo; import org.prebid.server.cache.model.CacheServiceResult; import org.prebid.server.cache.model.DebugHttpCall; +import org.prebid.server.deals.model.DeepDebugLog; +import org.prebid.server.deals.model.TxnLog; import org.prebid.server.events.EventsContext; import org.prebid.server.events.EventsService; import org.prebid.server.exception.InvalidRequestException; @@ -52,6 +54,7 @@ import org.prebid.server.identity.IdGeneratorType; import org.prebid.server.json.DecodeException; import org.prebid.server.json.JacksonMapper; +import org.prebid.server.proto.openrtb.ext.request.ExtDealLine; import org.prebid.server.proto.openrtb.ext.request.ExtImp; import org.prebid.server.proto.openrtb.ext.request.ExtImpPrebid; import org.prebid.server.proto.openrtb.ext.request.ExtMediaTypePriceGranularity; @@ -69,14 +72,18 @@ import org.prebid.server.proto.openrtb.ext.response.ExtBidResponse; import org.prebid.server.proto.openrtb.ext.response.ExtBidResponsePrebid; import org.prebid.server.proto.openrtb.ext.response.ExtBidderError; +import org.prebid.server.proto.openrtb.ext.response.ExtDebugPgmetrics; +import org.prebid.server.proto.openrtb.ext.response.ExtDebugTrace; import org.prebid.server.proto.openrtb.ext.response.ExtHttpCall; import org.prebid.server.proto.openrtb.ext.response.ExtResponseCache; import org.prebid.server.proto.openrtb.ext.response.ExtResponseDebug; +import org.prebid.server.proto.openrtb.ext.response.ExtTraceDeal; import org.prebid.server.settings.model.Account; import org.prebid.server.settings.model.AccountAnalyticsConfig; import org.prebid.server.settings.model.AccountAuctionConfig; import org.prebid.server.settings.model.AccountEventsConfig; import org.prebid.server.settings.model.VideoStoredDataResult; +import org.prebid.server.util.LineItemUtil; import org.prebid.server.vast.VastModifier; import java.math.BigDecimal; @@ -170,7 +177,7 @@ Future create(List bidderResponses, return videoStoredDataResult(auctionContext) .compose(videoStoredDataResult -> { final List modifiedBidderResponses = - updateBids(bidderResponses, videoStoredDataResult, auctionContext, eventsContext); + updateBids(bidderResponses, videoStoredDataResult, auctionContext, eventsContext, imps); return invokeProcessedBidderResponseHooks(modifiedBidderResponses, auctionContext) .map(updatedBidderResponses -> toBidderResponseInfos(updatedBidderResponses, imps)) @@ -187,7 +194,8 @@ Future create(List bidderResponses, private List updateBids(List bidderResponses, VideoStoredDataResult videoStoredDataResult, AuctionContext auctionContext, - EventsContext eventsContext) { + EventsContext eventsContext, + List imps) { final List result = new ArrayList<>(); for (final BidderResponse bidderResponse : bidderResponses) { @@ -198,8 +206,13 @@ private List updateBids(List bidderResponses, for (final BidderBid bidderBid : seatBid.getBids()) { final Bid receivedBid = bidderBid.getBid(); final BidType bidType = bidderBid.getType(); + + final Imp correspondingImp = correspondingImp(receivedBid, imps); + final ExtDealLine extDealLine = LineItemUtil.extDealLineFrom(receivedBid, correspondingImp, mapper); + final String lineItemId = extDealLine != null ? extDealLine.getLineItemId() : null; + final Bid modifiedBid = updateBid( - receivedBid, bidType, bidder, videoStoredDataResult, auctionContext, eventsContext); + receivedBid, bidType, bidder, videoStoredDataResult, auctionContext, eventsContext, lineItemId); modifiedBidderBids.add(bidderBid.with(modifiedBid)); } @@ -214,7 +227,8 @@ private Bid updateBid(Bid bid, String bidder, VideoStoredDataResult videoStoredDataResult, AuctionContext auctionContext, - EventsContext eventsContext) { + EventsContext eventsContext, + String lineItemId) { final Account account = auctionContext.getAccount(); final List debugWarnings = auctionContext.getDebugWarnings(); @@ -231,7 +245,8 @@ private Bid updateBid(Bid bid, account, eventsContext, effectiveBidId, - debugWarnings)) + debugWarnings, + lineItemId)) .ext(updateBidExt( bid, bidType, @@ -240,7 +255,8 @@ private Bid updateBid(Bid bid, videoStoredDataResult, eventsContext, generatedBidId, - effectiveBidId)) + effectiveBidId, + lineItemId)) .build(); } @@ -251,7 +267,8 @@ private String updateBidAdm( Account account, EventsContext eventsContext, String effectiveBidId, - List debugWarnings) { + List debugWarnings, + String lineItemId) { final String bidAdm = bid.getAdm(); return BidType.video.equals(bidType) @@ -262,7 +279,8 @@ private String updateBidAdm( effectiveBidId, account.getId(), eventsContext, - debugWarnings) + debugWarnings, + lineItemId) : bidAdm; } @@ -273,7 +291,8 @@ private ObjectNode updateBidExt(Bid bid, VideoStoredDataResult videoStoredDataResult, EventsContext eventsContext, String generatedBidId, - String effectiveBidId) { + String effectiveBidId, + String lineItemId) { final ExtBidPrebid updatedExtBidPrebid = updateBidExtPrebid( bid, @@ -283,7 +302,8 @@ private ObjectNode updateBidExt(Bid bid, videoStoredDataResult, eventsContext, generatedBidId, - effectiveBidId); + effectiveBidId, + lineItemId); final ObjectNode existingBidExt = bid.getExt(); final ObjectNode updatedBidExt = mapper.mapper().createObjectNode(); @@ -305,10 +325,11 @@ private ExtBidPrebid updateBidExtPrebid( VideoStoredDataResult videoStoredDataResult, EventsContext eventsContext, String generatedBidId, - String effectiveBidId) { + String effectiveBidId, + String lineItemId) { final Video storedVideo = videoStoredDataResult.getImpIdToStoredVideo().get(bid.getImpid()); - final Events events = createEvents(bidder, account, effectiveBidId, eventsContext); + final Events events = createEvents(bidder, account, effectiveBidId, eventsContext, lineItemId); final ExtBidPrebidVideo extBidPrebidVideo = getExtBidPrebidVideo(bid.getExt()); final ExtBidPrebid extBidPrebid = getExtPrebid(bid.getExt()); @@ -368,11 +389,15 @@ private List toBidderResponseInfos(List bidd } private BidInfo toBidInfo(Bid bid, BidType type, List imps, String bidder) { + final Imp correspondingImp = correspondingImp(bid, imps); + final ExtDealLine extDealLine = LineItemUtil.extDealLineFrom(bid, correspondingImp, mapper); + final String lineItemId = extDealLine != null ? extDealLine.getLineItemId() : null; return BidInfo.builder() .bid(bid) .bidType(type) .bidder(bidder) - .correspondingImp(correspondingImp(bid, imps)) + .correspondingImp(correspondingImp) + .lineItemId(lineItemId) .build(); } @@ -434,9 +459,11 @@ private Future cacheBidsAndCreateResponse(List } final ExtRequestTargeting targeting = targeting(bidRequest); + final TxnLog txnLog = auctionContext.getTxnLog(); final List bidderResponseInfos = - toBidderResponseWithTargetingBidInfos(bidderResponses, bidderToMultiBids, preferDeals(targeting)); + toBidderResponseWithTargetingBidInfos(bidderResponses, bidderToMultiBids, preferDeals(targeting), + txnLog); final Set bidInfos = bidderResponseInfos.stream() .map(BidderResponseInfo::getSeatBid) @@ -451,6 +478,8 @@ private Future cacheBidsAndCreateResponse(List .filter(bidInfo -> bidInfo.getTargetingInfo().isWinningBid()) .collect(Collectors.toSet()); + updateSentToClientTxnLog(txnLog, bidInfos); + final Set bidsToCache = cacheInfo.isShouldCacheWinningBidsOnly() ? winningBidInfos : bidInfos; return cacheBids(bidsToCache, auctionContext, cacheInfo, eventsContext) @@ -477,7 +506,8 @@ private static boolean preferDeals(ExtRequestTargeting targeting) { private List toBidderResponseWithTargetingBidInfos( List bidderResponses, Map bidderToMultiBids, - boolean preferDeals) { + boolean preferDeals, + TxnLog txnLog) { final Map> bidderResponseToReducedBidInfos = bidderResponses.stream() .collect(Collectors.toMap( @@ -506,6 +536,13 @@ private List toBidderResponseWithTargetingBidInfos( .ifPresent(winningBids::add); } + final Map> impIdToLineItemIds = impIdToBidderToBidInfos.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + impIdToBidderToBidInfoEntry -> toLineItemIds(impIdToBidderToBidInfoEntry.getValue().values()))); + + updateTopMatchAndLostAuctionLineItemsMetric(winningBids, txnLog, impIdToLineItemIds); + return bidderResponseToReducedBidInfos.entrySet().stream() .map(responseToBidInfos -> injectBidInfoWithTargeting( responseToBidInfos.getKey(), @@ -540,6 +577,33 @@ private List sortReducedBidInfo(List bidInfos, int limit, bool .collect(Collectors.toList()); } + private static Set toLineItemIds(Collection> bidInfos) { + return bidInfos.stream() + .flatMap(Collection::stream) + .map(BidInfo::getLineItemId) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + } + + /** + * Updates sent to client as top match and auction lost to line item metric. + */ + private static void updateTopMatchAndLostAuctionLineItemsMetric(Set winningBidInfos, + TxnLog txnLog, + Map> impToLineItemIds) { + for (BidInfo winningBidInfo : winningBidInfos) { + final String winningLineItemId = winningBidInfo.getLineItemId(); + if (winningLineItemId != null) { + txnLog.lineItemSentToClientAsTopMatch().add(winningLineItemId); + + final String impIdOfWinningBid = winningBidInfo.getBid().getImpid(); + impToLineItemIds.get(impIdOfWinningBid).stream() + .filter(lineItemId -> !Objects.equals(lineItemId, winningLineItemId)) + .forEach(lineItemId -> txnLog.lostAuctionToLineItems().get(lineItemId).add(winningLineItemId)); + } + } + } + private BidderResponseInfo injectBidInfoWithTargeting(BidderResponseInfo bidderResponseInfo, List bidderBidInfos, Map bidderToMultiBids, @@ -605,6 +669,16 @@ private List injectTargeting(List bidderImpIdBidInfos, return result; } + /** + * Increments sent to client metrics for each bid with deal. + */ + private void updateSentToClientTxnLog(TxnLog txnLog, Set bidInfos) { + bidInfos.stream() + .map(BidInfo::getLineItemId) + .filter(Objects::nonNull) + .forEach(lineItemId -> txnLog.lineItemsSentToClient().add(lineItemId)); + } + /** * Returns {@link ExtBidResponse} object, populated with response time, errors and debug info (if requested) * from all bidders. @@ -619,10 +693,8 @@ private ExtBidResponse toExtBidResponse(List bidderResponseI final BidRequest bidRequest = auctionContext.getBidRequest(); final boolean debugEnabled = auctionContext.getDebugContext().isDebugEnabled(); - final ExtResponseDebug extResponseDebug = debugEnabled - ? ExtResponseDebug.of(toExtHttpCalls(bidderResponseInfos, cacheResult), bidRequest) - : null; - + final ExtResponseDebug extResponseDebug = + toExtResponseDebug(bidderResponseInfos, auctionContext, cacheResult, debugEnabled); final Map> errors = toExtBidderErrors(bidderResponseInfos, auctionContext, cacheResult, videoStoredDataResult, bidErrors); final Map> warnings = debugEnabled @@ -640,6 +712,25 @@ private ExtBidResponse toExtBidResponse(List bidderResponseI .build(); } + private ExtResponseDebug toExtResponseDebug(List bidderResponseInfos, + AuctionContext auctionContext, + CacheServiceResult cacheResult, + boolean debugEnabled) { + final DeepDebugLog deepDebugLog = auctionContext.getDeepDebugLog(); + + final Map> httpCalls = debugEnabled + ? toExtHttpCalls(bidderResponseInfos, cacheResult, auctionContext.getDebugHttpCalls()) + : null; + final BidRequest bidRequest = debugEnabled ? auctionContext.getBidRequest() : null; + final ExtDebugPgmetrics extDebugPgmetrics = debugEnabled ? toExtDebugPgmetrics( + auctionContext.getTxnLog()) : null; + final ExtDebugTrace extDebugTrace = deepDebugLog.isDeepDebugEnabled() ? toExtDebugTrace(deepDebugLog) : null; + + return httpCalls == null && bidRequest == null && extDebugPgmetrics == null && extDebugTrace == null + ? null + : ExtResponseDebug.of(httpCalls, bidRequest, extDebugPgmetrics, extDebugTrace); + } + /** * Corresponds cacheId (or null if not present) to each {@link Bid}. */ @@ -702,7 +793,8 @@ private static CacheServiceResult addNotCachedBids(CacheServiceResult cacheResul } private static Map> toExtHttpCalls(List bidderResponses, - CacheServiceResult cacheResult) { + CacheServiceResult cacheResult, + Map> contextHttpCalls) { final Map> bidderHttpCalls = bidderResponses.stream() .collect(Collectors.toMap( BidderResponseInfo::getBidder, @@ -714,9 +806,15 @@ private static Map> toExtHttpCalls(List> contextExtHttpCalls = contextHttpCalls.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, serviceToHttpCall -> serviceToHttpCall.getValue().stream() + .map(BidResponseCreator::toExtHttpCall) + .collect(Collectors.toList()))); + final Map> httpCalls = new HashMap<>(); httpCalls.putAll(bidderHttpCalls); httpCalls.putAll(cacheHttpCalls); + httpCalls.putAll(contextExtHttpCalls); return httpCalls.isEmpty() ? null : httpCalls; } @@ -730,6 +828,38 @@ private static ExtHttpCall toExtHttpCall(DebugHttpCall debugHttpCall) { .build(); } + private static ExtDebugPgmetrics toExtDebugPgmetrics(TxnLog txnLog) { + final ExtDebugPgmetrics extDebugPgmetrics = ExtDebugPgmetrics.builder() + .matchedDomainTargeting(nullIfEmpty(txnLog.lineItemsMatchedDomainTargeting())) + .matchedWholeTargeting(nullIfEmpty(txnLog.lineItemsMatchedWholeTargeting())) + .matchedTargetingFcapped(nullIfEmpty(txnLog.lineItemsMatchedTargetingFcapped())) + .matchedTargetingFcapLookupFailed(nullIfEmpty(txnLog.lineItemsMatchedTargetingFcapLookupFailed())) + .readyToServe(nullIfEmpty(txnLog.lineItemsReadyToServe())) + .pacingDeferred(nullIfEmpty(txnLog.lineItemsPacingDeferred())) + .sentToBidder(nullIfEmpty(txnLog.lineItemsSentToBidder())) + .sentToBidderAsTopMatch(nullIfEmpty(txnLog.lineItemsSentToBidderAsTopMatch())) + .receivedFromBidder(nullIfEmpty(txnLog.lineItemsReceivedFromBidder())) + .responseInvalidated(nullIfEmpty(txnLog.lineItemsResponseInvalidated())) + .sentToClient(nullIfEmpty(txnLog.lineItemsSentToClient())) + .sentToClientAsTopMatch(nullIfEmpty(txnLog.lineItemSentToClientAsTopMatch())) + .build(); + return extDebugPgmetrics.equals(ExtDebugPgmetrics.EMPTY) ? null : extDebugPgmetrics; + } + + private static ExtDebugTrace toExtDebugTrace(DeepDebugLog deepDebugLog) { + final List entries = deepDebugLog.entries(); + final List dealsTrace = entries.stream() + .filter(extTraceDeal -> StringUtils.isEmpty(extTraceDeal.getLineItemId())) + .collect(Collectors.toList()); + final Map> lineItemsTrace = entries.stream() + .filter(extTraceDeal -> StringUtils.isNotEmpty(extTraceDeal.getLineItemId())) + .collect(Collectors.groupingBy(ExtTraceDeal::getLineItemId, Collectors.toList())); + return CollectionUtils.isNotEmpty(entries) + ? ExtDebugTrace.of(CollectionUtils.isEmpty(dealsTrace) ? null : dealsTrace, + MapUtils.isEmpty(lineItemsTrace) ? null : lineItemsTrace) + : null; + } + private Map> toExtBidderErrors(List bidderResponses, AuctionContext auctionContext, CacheServiceResult cacheResult, @@ -1239,10 +1369,21 @@ private static String integrationFrom(AuctionContext auctionContext) { private Events createEvents(String bidder, Account account, String bidId, - EventsContext eventsContext) { + EventsContext eventsContext, + String lineItemId) { - return eventsContext.isEnabledForAccount() && eventsContext.isEnabledForRequest() - ? eventsService.createEvent(bidId, bidder, account.getId(), eventsContext) + if (!eventsContext.isEnabledForAccount()) { + return null; + } + + return eventsContext.isEnabledForRequest() || StringUtils.isNotEmpty(lineItemId) + ? eventsService.createEvent( + bidId, + bidder, + account.getId(), + lineItemId, + eventsContext.isEnabledForRequest(), + eventsContext) : null; } @@ -1366,6 +1507,20 @@ private CacheAsset toCacheAsset(String cacheId) { return CacheAsset.of(cacheAssetUrlTemplate.concat(cacheId), cacheId); } + private static Set nullIfEmpty(Set set) { + if (set.isEmpty()) { + return null; + } + return Collections.unmodifiableSet(set); + } + + private static Map nullIfEmpty(Map map) { + if (map.isEmpty()) { + return null; + } + return Collections.unmodifiableMap(map); + } + /** * Creates {@link ExtBidPrebidVideo} from bid extension. */ diff --git a/src/main/java/org/prebid/server/auction/ExchangeService.java b/src/main/java/org/prebid/server/auction/ExchangeService.java index 5dbc6840370..bd3f6d166e1 100644 --- a/src/main/java/org/prebid/server/auction/ExchangeService.java +++ b/src/main/java/org/prebid/server/auction/ExchangeService.java @@ -8,7 +8,9 @@ import com.iab.openrtb.request.App; import com.iab.openrtb.request.BidRequest; import com.iab.openrtb.request.Content; +import com.iab.openrtb.request.Deal; import com.iab.openrtb.request.Imp; +import com.iab.openrtb.request.Pmp; import com.iab.openrtb.request.Site; import com.iab.openrtb.request.Source; import com.iab.openrtb.request.User; @@ -30,6 +32,7 @@ import org.prebid.server.auction.model.BidderResponse; import org.prebid.server.auction.model.MultiBidConfig; import org.prebid.server.auction.model.StoredResponseResult; +import org.prebid.server.auction.model.Tuple2; import org.prebid.server.bidder.Bidder; import org.prebid.server.bidder.BidderCatalog; import org.prebid.server.bidder.HttpBidderRequester; @@ -39,6 +42,9 @@ import org.prebid.server.bidder.model.BidderSeatBid; import org.prebid.server.cookie.UidsCookie; import org.prebid.server.currency.CurrencyConversionService; +import org.prebid.server.deals.DealsProcessor; +import org.prebid.server.deals.events.ApplicationEventService; +import org.prebid.server.deals.model.TxnLog; import org.prebid.server.exception.PreBidException; import org.prebid.server.execution.Timeout; import org.prebid.server.hooks.execution.HookStageExecutor; @@ -57,6 +63,7 @@ import org.prebid.server.hooks.v1.bidder.BidderRequestPayload; import org.prebid.server.hooks.v1.bidder.BidderResponsePayload; import org.prebid.server.json.JacksonMapper; +import org.prebid.server.log.CriteriaLogManager; import org.prebid.server.metric.MetricName; import org.prebid.server.metric.Metrics; import org.prebid.server.model.CaseInsensitiveMultiMap; @@ -64,6 +71,8 @@ import org.prebid.server.proto.openrtb.ext.request.BidAdjustmentMediaType; import org.prebid.server.proto.openrtb.ext.request.ExtApp; import org.prebid.server.proto.openrtb.ext.request.ExtBidderConfigOrtb; +import org.prebid.server.proto.openrtb.ext.request.ExtDeal; +import org.prebid.server.proto.openrtb.ext.request.ExtDealLine; import org.prebid.server.proto.openrtb.ext.request.ExtImpPrebid; import org.prebid.server.proto.openrtb.ext.request.ExtRequest; import org.prebid.server.proto.openrtb.ext.request.ExtRequestBidadjustmentfactors; @@ -95,6 +104,8 @@ import org.prebid.server.proto.openrtb.ext.response.ExtModulesTraceStage; import org.prebid.server.proto.openrtb.ext.response.ExtModulesTraceStageOutcome; import org.prebid.server.settings.model.Account; +import org.prebid.server.util.DealUtil; +import org.prebid.server.util.LineItemUtil; import org.prebid.server.util.StreamUtil; import org.prebid.server.validation.ResponseBidValidator; import org.prebid.server.validation.model.ValidationResult; @@ -110,7 +121,9 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -141,11 +154,13 @@ public class ExchangeService { private final ResponseBidValidator responseBidValidator; private final CurrencyConversionService currencyService; private final BidResponseCreator bidResponseCreator; + private final ApplicationEventService applicationEventService; private final BidResponsePostProcessor bidResponsePostProcessor; private final HookStageExecutor hookStageExecutor; private final Metrics metrics; private final Clock clock; private final JacksonMapper mapper; + private final CriteriaLogManager criteriaLogManager; public ExchangeService(long expectedCacheTime, BidderCatalog bidderCatalog, @@ -159,9 +174,11 @@ public ExchangeService(long expectedCacheTime, BidResponseCreator bidResponseCreator, BidResponsePostProcessor bidResponsePostProcessor, HookStageExecutor hookStageExecutor, + ApplicationEventService applicationEventService, Metrics metrics, Clock clock, - JacksonMapper mapper) { + JacksonMapper mapper, + CriteriaLogManager criteriaLogManager) { if (expectedCacheTime < 0) { throw new IllegalArgumentException("Expected cache time should be positive"); @@ -178,9 +195,11 @@ public ExchangeService(long expectedCacheTime, this.bidResponseCreator = Objects.requireNonNull(bidResponseCreator); this.bidResponsePostProcessor = Objects.requireNonNull(bidResponsePostProcessor); this.hookStageExecutor = Objects.requireNonNull(hookStageExecutor); + this.applicationEventService = applicationEventService; this.metrics = Objects.requireNonNull(metrics); this.clock = Objects.requireNonNull(clock); this.mapper = Objects.requireNonNull(mapper); + this.criteriaLogManager = Objects.requireNonNull(criteriaLogManager); } /** @@ -244,6 +263,9 @@ private Future runAuction(AuctionContext context) { context, cacheInfo, bidderToMultiBid)) + .map(bidResponse -> publishAuctionEvent(bidResponse, context)) + .map(bidResponse -> criteriaLogManager.traceResponse(logger, bidResponse, context.getBidRequest(), + context.getDebugContext().isDebugEnabled())) .compose(bidResponse -> bidResponsePostProcessor.postProcess( context.getHttpRequest(), uidsCookie, bidRequest, bidResponse, account)) .compose(bidResponse -> invokeResponseHooks(context, bidResponse)); @@ -419,6 +441,8 @@ private Future> extractBidderRequests(AuctionContext context Map bidderToMultiBid) { final List imps = storedResponseResult.getRequiredRequestImps().stream() .filter(imp -> bidderParamsFromImpExt(imp.getExt()) != null) + .map(imp -> DealsProcessor.removeDealsOnlyBiddersWithoutDeals(imp, context)) + .filter(Objects::nonNull) .collect(Collectors.toList()); // identify valid bidders and aliases out of imps final List bidders = imps.stream() @@ -481,7 +505,7 @@ private Future> makeBidderRequests(List bidders, .mask(context, bidderToUser, bidders, aliases) .map(bidderToPrivacyResult -> getBidderRequests(bidderToPrivacyResult, bidRequest, impBidderToStoredResponse, imps, - bidderToMultiBid, biddersToConfigs)); + bidderToMultiBid, biddersToConfigs, aliases)); } private Map getBiddersToConfigs(ExtRequestPrebid prebid) { @@ -687,7 +711,8 @@ private List getBidderRequests(List bidderPr Map> impBidderToStoredBidResponse, List imps, Map bidderToMultiBid, - Map biddersToConfigs) { + Map biddersToConfigs, + BidderAliases aliases) { final Map bidderToPrebidBidders = bidderToPrebidBidders(bidRequest); @@ -702,7 +727,8 @@ private List getBidderRequests(List bidderPr imps, bidderToMultiBid, biddersToConfigs, - bidderToPrebidBidders)) + bidderToPrebidBidders, + aliases)) .filter(Objects::nonNull) .collect(Collectors.toList()); @@ -740,7 +766,8 @@ private BidderRequest createBidderRequest(BidderPrivacyResult bidderPrivacyResul List imps, Map bidderToMultiBid, Map biddersToConfigs, - Map bidderToPrebidBidders) { + Map bidderToPrebidBidders, + BidderAliases bidderAliases) { final String bidder = bidderPrivacyResult.getRequestBidder(); if (bidderPrivacyResult.isBlockedRequestByTcf()) { @@ -767,7 +794,7 @@ private BidderRequest createBidderRequest(BidderPrivacyResult bidderPrivacyResul // User was already prepared above .user(bidderPrivacyResult.getUser()) .device(bidderPrivacyResult.getDevice()) - .imp(prepareImps(bidder, imps, useFirstPartyData)) + .imp(prepareImps(bidder, imps, useFirstPartyData, bidderAliases)) .app(prepareApp(bidRequestApp, fpdApp, useFirstPartyData)) .site(prepareSite(bidRequestSite, fpdSite, useFirstPartyData)) .source(prepareSource(bidder, bidRequest)) @@ -779,15 +806,63 @@ private BidderRequest createBidderRequest(BidderPrivacyResult bidderPrivacyResul * For each given imp creates a new imp with extension crafted to contain only "prebid", "context" and * bidder-specific extension. */ - private List prepareImps(String bidder, List imps, boolean useFirstPartyData) { + private List prepareImps(String bidder, List imps, boolean useFirstPartyData, BidderAliases aliases) { return imps.stream() .filter(imp -> bidderParamsFromImpExt(imp.getExt()).hasNonNull(bidder)) .map(imp -> imp.toBuilder() + .pmp(preparePmp(bidder, imp.getPmp(), aliases)) .ext(prepareImpExt(bidder, imp.getExt(), useFirstPartyData)) .build()) .collect(Collectors.toList()); } + /** + * Removes deal from {@link Pmp} if bidder's deals doesn't contain it. + */ + private Pmp preparePmp(String bidder, Pmp pmp, BidderAliases aliases) { + final List originalDeals = pmp != null ? pmp.getDeals() : null; + if (CollectionUtils.isEmpty(originalDeals)) { + return pmp; + } + + final List updatedDeals = originalDeals.stream() + .map(deal -> Tuple2.of(deal, toExtDeal(deal.getExt()))) + .filter((Tuple2 tuple) -> DealUtil.isBidderHasDeal(bidder, tuple.getRight(), aliases)) + .map((Tuple2 tuple) -> prepareDeal(tuple.getLeft(), tuple.getRight())) + .collect(Collectors.toList()); + + return pmp.toBuilder().deals(updatedDeals).build(); + } + + /** + * Returns {@link ExtDeal} from the given {@link ObjectNode}. + */ + private ExtDeal toExtDeal(ObjectNode ext) { + if (ext == null) { + return null; + } + try { + return mapper.mapper().treeToValue(ext, ExtDeal.class); + } catch (JsonProcessingException e) { + throw new PreBidException( + String.format("Error decoding bidRequest.imp.pmp.deal.ext: %s", e.getMessage()), e); + } + } + + /** + * Removes bidder from imp[].pmp.deal[].ext.line object if presents. + */ + private Deal prepareDeal(Deal deal, ExtDeal extDeal) { + final ExtDealLine line = extDeal != null ? extDeal.getLine() : null; + final ExtDealLine updatedLine = line != null + ? ExtDealLine.of(line.getLineItemId(), line.getExtLineItemId(), line.getSizes(), null) + : null; + + return updatedLine != null + ? deal.toBuilder().ext(mapper.mapper().valueToTree(ExtDeal.of(updatedLine))).build() + : deal; + } + /** * Creates a new imp extension for particular bidder having: *