Add impression tracking to VAST (Server-Side) (#437)

docs/config-app.md

@@ -55,6 +55,9 @@ This parameter affects how many CPU cores will be utilized by the application. R
 - `cookie-sync.coop-sync.default` - default value for coopSync when it missing in requests to `/cookie_sync` endpoint.
 - `cookie-sync.coop-sync.pri` - lists of bidders prioritised in groups.
 
+## Event
+- `event.url-template` - template string for impression tracking, suffixed to `external-url`, where `BIDID` placeholder is used for Bid's id and `ACCOUNT` - for Account's id
+
 ## Adapters
 - `adapters.*` - the section for bidder specific configuration options.
 

docs/differenceBetweenPBSGo-and-Java.md

@@ -23,6 +23,7 @@ and not the other for an interim period. This page tracks known differences that
 1) PBS-Java supports per-account cache TTL and event URLs configuration in the database in columns `banner_cache_ttl`, `video_cache_ttl` and `events_enabled`.
 1) PBS-Java does not support passing bidder extensions in `imp[...].ext.prebid.bidder`. PBS-Go [PR 846](https://github.com/prebid/prebid-server/pull/846)
 1) PBS-Java responds with active bidders only in `/info/bidders` endpoint, although PBS-Go returns all implemented ones.
+1) PBS-Java supports video impression tracking [issue 1015](https://github.com/prebid/prebid-server/issues/1015). PBS-Java [PR 437](https://github.com/rubicon-project/prebid-server-java/pull/437). 
 
 ## Minor differences
 

docs/pbs-java-and-go-features-review.md

@@ -16,6 +16,7 @@ Bid Categories |-|+
 Event endpoint |+|-
 Video Endpoint |-|+
 COPPA |+|+
+Video Impression Tracking |+|-
 All adapters ported to OpenRTB |+|-
 Bidder Generator |+|-
 

src/main/java/org/prebid/server/auction/ExchangeService.java

@@ -862,25 +862,21 @@ private static MetricName bidderErrorTypeToMetric(BidderError.Type errorType) {
     private Future<BidResponse> toBidResponse(List<BidderResponse> bidderResponses, BidRequest bidRequest,
                                               ExtRequestTargeting targeting, BidRequestCacheInfo cacheInfo,
                                               Account account, Timeout timeout, boolean debugEnabled) {
-        final Set<Bid> bids = bidderResponses.stream()
-                .map(BidderResponse::getSeatBid)
-                .filter(Objects::nonNull)
-                .map(BidderSeatBid::getBids)
-                .filter(Objects::nonNull)
-                .flatMap(Collection::stream)
-                .map(BidderBid::getBid)
-                .collect(Collectors.toSet());
-
-        return toBidsWithCacheIds(bids, bidRequest.getImp(), cacheInfo, account, timeout)
+        return toBidsWithCacheIds(bidderResponses, bidRequest.getImp(), cacheInfo, account, timeout)
                 .map(cacheResult -> bidResponseCreator.create(bidderResponses, bidRequest, targeting, cacheInfo,
                         cacheResult, account, debugEnabled));
     }
 
     /**
      * Corresponds cacheId (or null if not present) to each {@link Bid}.
      */
-    private Future<CacheServiceResult> toBidsWithCacheIds(Set<Bid> bids, List<Imp> imps, BidRequestCacheInfo cacheInfo,
-                                                          Account account, Timeout timeout) {
+    private Future<CacheServiceResult> toBidsWithCacheIds(List<BidderResponse> bidderResponses, List<Imp> imps,
+                                                          BidRequestCacheInfo cacheInfo, Account account,
+                                                          Timeout timeout) {
+        final Set<Bid> bids = bidderResponses.stream()
+                .flatMap(ExchangeService::getBids)
+                .collect(Collectors.toSet());
+
         final Future<CacheServiceResult> result;
 
         if (!cacheInfo.isDoCaching()) {
@@ -891,8 +887,20 @@ private Future<CacheServiceResult> toBidsWithCacheIds(Set<Bid> bids, List<Imp> i
                     .filter(bid -> bid.getPrice().compareTo(BigDecimal.ZERO) > 0)
                     .collect(Collectors.toList());
 
-            final CacheContext cacheContext = CacheContext.of(cacheInfo.isShouldCacheBids(),
-                    cacheInfo.getCacheBidsTtl(), cacheInfo.isShouldCacheVideoBids(), cacheInfo.getCacheVideoBidsTtl());
+            final boolean shouldCacheVideoBids = cacheInfo.isShouldCacheVideoBids();
+            final boolean eventsEnabled = Objects.equals(account.getEventsEnabled(), true);
+
+            final List<String> videoBidIdsToModify = shouldCacheVideoBids && eventsEnabled
+                    ? getVideoBidIdsToModify(bidderResponses, imps)
+                    : Collections.emptyList();
+
+            final CacheContext cacheContext = CacheContext.builder()
+                    .cacheBidsTtl(cacheInfo.getCacheBidsTtl())
+                    .cacheVideoBidsTtl(cacheInfo.getCacheVideoBidsTtl())
+                    .shouldCacheBids(cacheInfo.isShouldCacheBids())
+                    .shouldCacheVideoBids(shouldCacheVideoBids)
+                    .videoBidIdsToModify(videoBidIdsToModify)
+                    .build();
 
             result = cacheService.cacheBidsOpenrtb(bidsWithNonZeroPrice, imps, cacheContext, account, timeout)
                     .map(cacheResult -> addNotCachedBids(cacheResult, bids));
@@ -901,6 +909,32 @@ private Future<CacheServiceResult> toBidsWithCacheIds(Set<Bid> bids, List<Imp> i
         return result;
     }
 
+    private static Stream<Bid> getBids(BidderResponse bidderResponse) {
+        return Stream.of(bidderResponse)
+                .map(BidderResponse::getSeatBid)
+                .filter(Objects::nonNull)
+                .map(BidderSeatBid::getBids)
+                .filter(Objects::nonNull)
+                .flatMap(Collection::stream)
+                .map(BidderBid::getBid);
+    }
+
+    private List<String> getVideoBidIdsToModify(List<BidderResponse> bidderResponses, List<Imp> imps) {
+        return bidderResponses.stream()
+                .filter(bidderResponse -> bidderCatalog.isModifyingVastXmlAllowed(bidderResponse.getBidder()))
+                .flatMap(ExchangeService::getBids)
+                .filter(bid -> isVideoBid(bid, imps))
+                .map(Bid::getId)
+                .collect(Collectors.toList());
+    }
+
+    private static boolean isVideoBid(Bid bid, List<Imp> imps) {
+        return imps.stream()
+                .filter(imp -> imp.getVideo() != null)
+                .map(Imp::getId)
+                .anyMatch(impId -> bid.getImpid().equals(impId));
+    }
+
     /**
      * Creates a map with {@link Bid} as a key and null as a value.
      */

src/main/java/org/prebid/server/bidder/BidderCatalog.java

@@ -56,6 +56,13 @@ public boolean isValidName(String name) {
         return bidderDepsMap.containsKey(name);
     }
 
+    /**
+     * Tells if given bidder allows to modify video's Vast XML.
+     */
+    public boolean isModifyingVastXmlAllowed(String name) {
+        return bidderDepsMap.containsKey(name) && bidderDepsMap.get(name).getBidderInfo().isModifyingVastXmlAllowed();
+    }
+
     /**
      * Tells if given name corresponds to any of the registered deprecated bidder's name.
      */

src/main/java/org/prebid/server/cache/CacheService.java

@@ -55,18 +55,23 @@ public class CacheService {
 
     private static final Logger logger = LoggerFactory.getLogger(CacheService.class);
 
+    private static final String BID_ID_PLACEHOLDER = "BIDID";
+    private static final String ACCOUNT_PLACEHOLDER = "ACCOUNT";
+
     private final CacheTtl mediaTypeCacheTtl;
     private final HttpClient httpClient;
     private final URL endpointUrl;
     private final String cachedAssetUrlTemplate;
+    private final String eventsUrlTemplate;
     private final Clock clock;
 
     public CacheService(CacheTtl mediaTypeCacheTtl, HttpClient httpClient, URL endpointUrl,
-                        String cachedAssetUrlTemplate, Clock clock) {
+                        String cachedAssetUrlTemplate, String eventsUrlTemplate, Clock clock) {
         this.mediaTypeCacheTtl = Objects.requireNonNull(mediaTypeCacheTtl);
         this.httpClient = Objects.requireNonNull(httpClient);
         this.endpointUrl = Objects.requireNonNull(endpointUrl);
         this.cachedAssetUrlTemplate = Objects.requireNonNull(cachedAssetUrlTemplate);
+        this.eventsUrlTemplate = Objects.requireNonNull(eventsUrlTemplate);
         this.clock = Objects.requireNonNull(clock);
     }
 
@@ -152,20 +157,23 @@ public Future<CacheServiceResult> cacheBidsOpenrtb(List<com.iab.openrtb.response
             final Map<String, Integer> impIdToTtl = new HashMap<>(imps.size());
             boolean impWithNoExpExists = false; // indicates at least one impression without expire presents
             final List<String> videoImpIds = new ArrayList<>();
+            final boolean shouldCacheVideoBids = cacheContext.isShouldCacheVideoBids();
             for (Imp imp : imps) {
-                impIdToTtl.put(imp.getId(), imp.getExp());
+                final String impId = imp.getId();
+                impIdToTtl.put(impId, imp.getExp());
                 impWithNoExpExists |= imp.getExp() == null;
-                if (cacheContext.isShouldCacheVideoBids() && imp.getId() != null && imp.getVideo() != null) {
-                    videoImpIds.add(imp.getId());
+                if (shouldCacheVideoBids && impId != null && imp.getVideo() != null) {
+                    videoImpIds.add(impId);
                 }
             }
 
             final List<CacheBid> cacheBids = getCacheBids(cacheContext.isShouldCacheBids(), bids, impIdToTtl,
                     impWithNoExpExists, cacheContext.getCacheBidsTtl(), account);
-            final List<CacheBid> videoCacheBids = getVideoCacheBids(cacheContext.isShouldCacheVideoBids(), bids,
+            final List<CacheBid> videoCacheBids = getVideoCacheBids(shouldCacheVideoBids, bids,
                     impIdToTtl, videoImpIds, impWithNoExpExists, cacheContext.getCacheVideoBidsTtl(), account);
 
-            result = doCacheOpenrtb(cacheBids, videoCacheBids, timeout);
+            result = doCacheOpenrtb(cacheBids, videoCacheBids, cacheContext.getVideoBidIdsToModify(), account.getId(),
+                    timeout);
         }
 
         return result;
@@ -238,10 +246,12 @@ private CacheBid toCacheBid(com.iab.openrtb.response.Bid bid, Map<String, Intege
      * <p>
      * The returned result will always have the number of elements equals to sum of sizes of bids and video bids.
      */
-    private Future<CacheServiceResult> doCacheOpenrtb(List<CacheBid> bids, List<CacheBid> videoBids, Timeout timeout) {
+    private Future<CacheServiceResult> doCacheOpenrtb(
+            List<CacheBid> bids, List<CacheBid> videoBids, List<String> videoBidIdsToModify, String accountId,
+            Timeout timeout) {
         final List<PutObject> putObjects = Stream.concat(
                 bids.stream().map(CacheService::createJsonPutObjectOpenrtb),
-                videoBids.stream().map(CacheService::createXmlPutObjectOpenrtb))
+                videoBids.stream().map(cacheBid -> createXmlPutObjectOpenrtb(cacheBid, videoBidIdsToModify, accountId)))
                 .collect(Collectors.toList());
 
         if (putObjects.isEmpty()) {
@@ -326,16 +336,47 @@ private static PutObject createJsonPutObjectOpenrtb(CacheBid cacheBid) {
     /**
      * Makes XML type {@link PutObject} from {@link com.iab.openrtb.response.Bid}. Used for OpenRTB auction request.
      */
-    private static PutObject createXmlPutObjectOpenrtb(CacheBid cacheBid) {
-        if (cacheBid.getBid().getAdm() == null) {
-            return PutObject.of("xml", new TextNode("<VAST version=\"3.0\"><Ad><Wrapper>"
+    private PutObject createXmlPutObjectOpenrtb(CacheBid cacheBid, List<String> videoBidIdsToModify,
+                                                String accountId) {
+        final com.iab.openrtb.response.Bid bid = cacheBid.getBid();
+        String vastXml;
+        if (bid.getAdm() == null) {
+            vastXml = "<VAST version=\"3.0\"><Ad><Wrapper>"
                     + "<AdSystem>prebid.org wrapper</AdSystem>"
-                    + "<VASTAdTagURI><![CDATA[" + cacheBid.getBid().getNurl() + "]]></VASTAdTagURI>"
+                    + "<VASTAdTagURI><![CDATA[" + bid.getNurl() + "]]></VASTAdTagURI>"
                     + "<Impression></Impression><Creatives></Creatives>"
-                    + "</Wrapper></Ad></VAST>"), cacheBid.getTtl());
+                    + "</Wrapper></Ad></VAST>";
         } else {
-            return PutObject.of("xml", new TextNode(cacheBid.getBid().getAdm()), cacheBid.getTtl());
+            vastXml = bid.getAdm();
+        }
+
+        final String bidId = bid.getId();
+        if (CollectionUtils.isNotEmpty(videoBidIdsToModify) && videoBidIdsToModify.contains(bidId)) {
+            vastXml = modifyVastXml(vastXml, bidId, accountId);
+        }
+
+        return PutObject.of("xml", new TextNode(vastXml), cacheBid.getTtl());
+    }
+
+    private String modifyVastXml(String stringValue, String bidId, String accountId) {
+        final String closeTag = "</Impression>";
+        final int closeTagIndex = stringValue.indexOf(closeTag);
+
+        // no impression tag - pass it as it is
+        if (closeTagIndex == -1) {
+            return stringValue;
+        }
+
+        final String impressionUrl = eventsUrlTemplate.replace(BID_ID_PLACEHOLDER, bidId)
+                .replace(ACCOUNT_PLACEHOLDER, accountId);
+        final String openTag = "<Impression>";
+
+        // empty impression tag - just insert the link
+        if (closeTagIndex - stringValue.indexOf(openTag) == openTag.length()) {
+            return stringValue.replaceFirst(openTag, openTag + impressionUrl);
         }
+
+        return stringValue.replaceFirst(closeTag, closeTag + openTag + impressionUrl + closeTag);
     }
 
     /**

src/main/java/org/prebid/server/cache/model/CacheContext.java

@@ -1,12 +1,14 @@
 package org.prebid.server.cache.model;
 
-import lombok.AllArgsConstructor;
+import lombok.Builder;
 import lombok.Value;
 
+import java.util.List;
+
 /**
  * Holds the state needed to perform caching response bids.
  */
-@AllArgsConstructor(staticName = "of")
+@Builder
 @Value
 public class CacheContext {
 
@@ -17,4 +19,6 @@ public class CacheContext {
     boolean shouldCacheVideoBids;
 
     Integer cacheVideoBidsTtl;
+
+    List<String> videoBidIdsToModify;
 }

src/main/java/org/prebid/server/proto/response/BidderInfo.java

@@ -18,15 +18,17 @@ public class BidderInfo {
 
     GdprInfo gdpr;
 
+    boolean modifyingVastXmlAllowed;
+
     public static BidderInfo create(boolean enabled, String maintainerEmail, List<String> appMediaTypes,
-                                    List<String> siteMediaTypes, List<String> supportedVendors,
-                                    int vendorId, boolean enforceGdpr) {
+                                    List<String> siteMediaTypes, List<String> supportedVendors, int vendorId,
+                                    boolean enforceGdpr, boolean modifyingVastXmlAllowed) {
         final MaintainerInfo maintainer = new MaintainerInfo(maintainerEmail);
         final CapabilitiesInfo capabilities = new CapabilitiesInfo(platformInfo(appMediaTypes),
                 platformInfo(siteMediaTypes));
         final GdprInfo gdpr = new GdprInfo(vendorId, enforceGdpr);
 
-        return new BidderInfo(enabled, maintainer, capabilities, supportedVendors, gdpr);
+        return new BidderInfo(enabled, maintainer, capabilities, supportedVendors, gdpr, modifyingVastXmlAllowed);
     }
 
     private static PlatformInfo platformInfo(List<String> mediaTypes) {

src/main/java/org/prebid/server/spring/config/ServiceConfiguration.java

@@ -79,6 +79,8 @@ CacheService cacheService(
             @Value("${cache.query}") String query,
             @Value("${cache.banner-ttl-seconds:#{null}}") Integer bannerCacheTtl,
             @Value("${cache.video-ttl-seconds:#{null}}") Integer videoCacheTtl,
+            @Value("${external-url}") String externalUrl,
+            @Value("${event.url-template}") String eventUrlTemplate,
             HttpClient httpClient,
             Clock clock) {
 
@@ -87,7 +89,7 @@ CacheService cacheService(
                 httpClient,
                 CacheService.getCacheEndpointUrl(scheme, host, path),
                 CacheService.getCachedAssetUrlTemplate(scheme, host, path, query),
-                clock);
+                externalUrl + eventUrlTemplate, clock);
     }
 
     @Bean

src/main/java/org/prebid/server/spring/config/bidder/model/BidderConfigurationProperties.java

@@ -22,6 +22,9 @@ public class BidderConfigurationProperties {
     @NotNull
     private Boolean pbsEnforcesGdpr;
 
+    @NotNull
+    private Boolean modifyingVastXmlAllowed;
+
     @NotNull
     private List<String> deprecatedNames;
 

src/main/java/org/prebid/server/spring/config/bidder/util/BidderInfoCreator.java

@@ -19,6 +19,7 @@ public static BidderInfo create(BidderConfigurationProperties configurationPrope
                 metaInfo.getSiteMediaTypes(),
                 metaInfo.getSupportedVendors(),
                 metaInfo.getVendorId(),
-                configurationProperties.getPbsEnforcesGdpr());
+                configurationProperties.getPbsEnforcesGdpr(),
+                configurationProperties.getModifyingVastXmlAllowed());
     }
 }

src/main/resources/application.yaml

@@ -41,6 +41,8 @@ cookie-sync:
   coop-sync:
     default: true
   default-timeout-ms: 2000
+event:
+  url-template: /event?t=imp&b=BIDID&f=b&a=ACCOUNT
 currency-converter:
   enabled: true
   url: https://cdn.jsdelivr.net/gh/prebid/currency-file@1/latest.json

src/main/resources/bidder-config/adform.yaml

@@ -3,6 +3,7 @@ adapters:
     enabled: false
     endpoint: http://adx.adform.net/adx
     pbs-enforces-gdpr: true
+    modifying-vast-xml-allowed: true
     deprecated-names:
     aliases:
     meta-info:

src/main/resources/bidder-config/adkerneladn.yaml

@@ -3,6 +3,7 @@ adapters:
     enabled: false
     endpoint: http://tag.adkernel.com/rtbpub?account=
     pbs-enforces-gdpr: true
+    modifying-vast-xml-allowed: true
     deprecated-names:
     aliases:
     meta-info:

src/main/resources/bidder-config/adtelligent.yaml

@@ -3,6 +3,7 @@ adapters:
     enabled: false
     endpoint: http://hb.adtelligent.com/auction
     pbs-enforces-gdpr: true
+    modifying-vast-xml-allowed: true
     deprecated-names:
     aliases:
     meta-info:

src/main/resources/bidder-config/advangelists.yaml

@@ -3,6 +3,7 @@ adapters:
     enabled: false
     endpoint: http://nep.advangelists.com/xp/get?pubid=
     pbs-enforces-gdpr: true
+    modifying-vast-xml-allowed: true
     deprecated-names:
     aliases:
     meta-info:

src/main/resources/bidder-config/appnexus.yaml

@@ -3,6 +3,7 @@ adapters:
     enabled: false
     endpoint: http://ib.adnxs.com/openrtb2
     pbs-enforces-gdpr: true
+    modifying-vast-xml-allowed: true
     deprecated-names:
     aliases: districtm
     meta-info:

src/main/resources/bidder-config/beachfront.yaml

@@ -4,6 +4,7 @@ adapters:
     endpoint: https://display.bfmio.com/prebid_display
     video-endpoint: https://reachms.bfmio.com/bid.json?exchange_id=
     pbs-enforces-gdpr: true
+    modifying-vast-xml-allowed: true
     deprecated-names:
     aliases:
     meta-info:

src/main/resources/bidder-config/brightroll.yaml

@@ -3,6 +3,7 @@ adapters:
     enabled: false
     endpoint: http://east-bid.ybp.yahoo.com/bid/appnexuspbs
     pbs-enforces-gdpr: true
+    modifying-vast-xml-allowed: true
     deprecated-names:
     aliases:
     meta-info: