Add bid size and secure markup validations, controlled by host through configuration

docs/application-settings.md

@@ -23,6 +23,7 @@ There are two ways to configure application settings: database and file. This do
 - `truncate-target-attr` - Maximum targeting attributes size. Values between 1 and 255.
 - `default-integration` - Default integration to assume.
 - `analytics-config.auction-events.<channel>` - defines which channels are supported by analytics for this account
+- `bid-validations.banner-creative-max-size` - Overrides creative max size validation for banners.
 
 ```
 Purpose   | Purpose goal                    | Purpose meaning for PBS (n\a - not affected)  
@@ -187,6 +188,7 @@ Query to create accounts_account table:
 `truncate_target_attr` tinyint(3) unsigned DEFAULT NULL,
 `default_integration` varchar(64) DEFAULT NULL,
 `analytics_config` varchar(512) DEFAULT NULL,
+`bid_validations` json DEFAULT NULL,
 `status` enum('active','inactive') DEFAULT 'active',
 `updated_by` int(11) DEFAULT NULL,
 `updated_by_user` varchar(64) DEFAULT NULL,
@@ -198,16 +200,16 @@ ENGINE=InnoDB DEFAULT CHARSET=utf8'
 
 where tcf_config column is json with next format
 
-```
+```json
 {
   "enabled": true,
    "integration-enabled": {
       "video": true,
       "web": true,
       "app": true,
       "amp": true
-   }
-  "purpose-one-treatment-interpretation": "ignore"
+   },
+  "purpose-one-treatment-interpretation": "ignore",
   "purposes": {
     "p1": {
       "enforce-purpose": "full",
@@ -309,9 +311,17 @@ where tcf_config column is json with next format
 }
 ```
 
+and bid_validations column is json with next format
+
+```json
+{
+  "banner-creative-max-size": "enforce"
+}
+```
+
 Query used to get an account:
 ```
-SELECT uuid, price_granularity, banner_cache_ttl, video_cache_ttl, events_enabled, enforce_ccpa, tcf_config, analytics_sampling_factor, truncate_target_attr, default_integration, analytics_config 
+SELECT uuid, price_granularity, banner_cache_ttl, video_cache_ttl, events_enabled, enforce_ccpa, tcf_config, analytics_sampling_factor, truncate_target_attr, default_integration, analytics_config, bid_validations 
 FROM accounts_account where uuid = ?
 LIMIT 1
 

docs/config-app.md

@@ -76,6 +76,8 @@ Removes and downloads file again if depending service cant process probably corr
 - `auction.cache.only-winning-bids` - if equals to `true` only the winning bids would be cached. Has lower priority than request-specific flags.
 - `auction.generate-bid-id` - whether to generate seatbid[].bid[].ext.prebid.bidid in the OpenRTB response.
 - `auction.id-generator-type` - if generate-bid-id is on, then this defines how the ID should be generated. Currently onlye `uuid` is supported.
+- `auction.validations.banner-creative-max-size` - enables creative max size validation for banners. Possible values: `skip`, `enforce`, `warn`. Default is `skip`.
+- `auction.validations.secure-markup` - enables secure markup validation. Possible values: `skip`, `enforce`, `warn`. Default is `skip`.
 
 ## Amp (OpenRTB)
 - `amp.default-timeout-ms` - default operation timeout for OpenRTB Amp requests.

docs/metrics.md

@@ -80,10 +80,14 @@ where `[DATASOURCE]` is a data source name, `DEFAULT_DS` by defaul.
 - `adapter.<bidder-name>.(openrtb2-web|openrtb-app|amp|legacy).tcf.geo_masked` - number of requests made to `<bidder-name>` that required geo information removed as a result of TCF enforcement for that bidder
 - `adapter.<bidder-name>.(openrtb2-web|openrtb-app|amp|legacy).tcf.request_blocked` - number of requests made to `<bidder-name>` that were blocked as a result of TCF enforcement for that bidder
 - `adapter.<bidder-name>.(openrtb2-web|openrtb-app|amp|legacy).tcf.analytics_blocked` - number of requests made to `<bidder-name>` that required analytics blocked as a result of TCF enforcement for that bidder
+- `adapter.<bidder-name>.response.validation.size.(warn|err)` - number of banner bids received from the `<bidder-name>` that had invalid size
+- `adapter.<bidder-name>.response.validation.secure.(warn|err)` - number of bids received from the `<bidder-name>` that had insecure creative while in secure context
 
 ## Auction per-account metrics
 Following metrics are collected and submitted if account is configured with `basic` verbosity:   
 - `account.<account-id>.requests` - number of requests received from account with `<account-id>`
+- `account.<account-id>.response.validation.size.(warn|err)` - number of banner bids received from account with `<account-id>` that had invalid size
+- `account.<account-id>.response.validation.secure.(warn|err)` - number of bids received from account with `<account-id>` that had insecure creative while in secure context
 
 Following metrics are collected and submitted if account is configured with `detailed` verbosity:
 - `account.<account-id>.requests.type.(openrtb2-web,openrtb-app,amp,legacy)` - number of requests received from account with `<account-id>` broken down by type of incoming request

src/main/java/org/prebid/server/auction/ExchangeService.java

@@ -154,18 +154,19 @@ public Future<BidResponse> holdAuction(AuctionContext context) {
                 .compose(impsRequiredRequest -> extractBidderRequests(context, impsRequiredRequest, aliases))
                 .map(bidderRequests -> updateRequestMetric(
                         bidderRequests, uidsCookie, aliases, publisherId, context.getRequestTypeMetric()))
-                .compose(bidderRequests -> CompositeFuture.join(bidderRequests.stream()
-                        .map(bidderRequest -> requestBids(
-                                bidderRequest,
-                                auctionTimeout(timeout, cacheInfo.isDoCaching()),
-                                debugEnabled,
-                                aliases))
-                        .collect(Collectors.toList())))
+                .compose(bidderRequests -> CompositeFuture.join(
+                        bidderRequests.stream()
+                                .map(bidderRequest -> requestBids(
+                                        bidderRequest,
+                                        auctionTimeout(timeout, cacheInfo.isDoCaching()),
+                                        debugEnabled,
+                                        aliases))
+                                .collect(Collectors.toList())))
                 // send all the requests to the bidders and gathers results
                 .map(CompositeFuture::<BidderResponse>list)
                 .map(bidderResponses -> storedResponseProcessor.mergeWithBidderResponses(
                         bidderResponses, storedResponse, bidRequest.getImp()))
-                .map(bidderResponses -> validateAndAdjustBids(bidRequest, bidderResponses))
+                .map(bidderResponses -> validateAndAdjustBids(bidderResponses, context, aliases))
                 .map(bidderResponses -> updateMetricsFromResponses(bidderResponses, publisherId, aliases))
                 // produce response from bidder results
                 .compose(bidderResponses -> bidResponseCreator.create(
@@ -811,8 +812,10 @@ private static BigDecimal bidAdjustmentForBidder(BidRequest bidRequest, String b
      * Passes the request to a corresponding bidder and wraps response in {@link BidderResponse} which also holds
      * recorded response time.
      */
-    private Future<BidderResponse> requestBids(
-            BidderRequest bidderRequest, Timeout timeout, boolean debugEnabled, BidderAliases aliases) {
+    private Future<BidderResponse> requestBids(BidderRequest bidderRequest,
+                                               Timeout timeout,
+                                               boolean debugEnabled,
+                                               BidderAliases aliases) {
 
         final String bidderName = bidderRequest.getBidder();
         final Bidder<?> bidder = bidderCatalog.bidderByName(aliases.resolveBidder(bidderName));
@@ -822,10 +825,12 @@ private Future<BidderResponse> requestBids(
                 .map(seatBid -> BidderResponse.of(bidderName, seatBid, responseTime(startTime)));
     }
 
-    private List<BidderResponse> validateAndAdjustBids(BidRequest bidRequest, List<BidderResponse> bidderResponses) {
+    private List<BidderResponse> validateAndAdjustBids(
+            List<BidderResponse> bidderResponses, AuctionContext auctionContext, BidderAliases aliases) {
+
         return bidderResponses.stream()
-                .map(bidderResponse -> validBidderResponse(bidderResponse, bidRequest.getCur()))
-                .map(bidderResponse -> applyBidPriceChanges(bidderResponse, bidRequest))
+                .map(bidderResponse -> validBidderResponse(bidderResponse, auctionContext, aliases))
+                .map(bidderResponse -> applyBidPriceChanges(bidderResponse, auctionContext.getBidRequest()))
                 .collect(Collectors.toList());
     }
 
@@ -836,35 +841,48 @@ private List<BidderResponse> validateAndAdjustBids(BidRequest bidRequest, List<B
      * <p>
      * Returns input argument as the result if no errors found or creates new {@link BidderResponse} otherwise.
      */
-    private BidderResponse validBidderResponse(BidderResponse bidderResponse, List<String> requestCurrencies) {
-        final BidderSeatBid seatBid = bidderResponse.getSeatBid();
-        final List<BidderBid> bids = seatBid.getBids();
+    private BidderResponse validBidderResponse(
+            BidderResponse bidderResponse, AuctionContext auctionContext, BidderAliases aliases) {
 
-        final List<BidderBid> validBids = new ArrayList<>(bids.size());
+        final BidRequest bidRequest = auctionContext.getBidRequest();
+        final BidderSeatBid seatBid = bidderResponse.getSeatBid();
         final List<BidderError> errors = new ArrayList<>(seatBid.getErrors());
 
+        final List<String> requestCurrencies = bidRequest.getCur();
         if (requestCurrencies.size() > 1) {
             errors.add(BidderError.badInput(
                     String.format("Cur parameter contains more than one currency. %s will be used",
                             requestCurrencies.get(0))));
         }
 
-        for (BidderBid bid : bids) {
-            final ValidationResult validationResult = responseBidValidator.validate(bid);
+        final List<BidderBid> bids = seatBid.getBids();
+        final List<BidderBid> validBids = new ArrayList<>(bids.size());
+
+        for (final BidderBid bid : bids) {
+            final ValidationResult validationResult =
+                    responseBidValidator.validate(bid, bidderResponse.getBidder(), auctionContext, aliases);
+
+            if (validationResult.hasWarnings()) {
+                addAsBidderErrors(validationResult.getWarnings(), errors);
+            }
+
             if (validationResult.hasErrors()) {
-                for (String error : validationResult.getErrors()) {
-                    errors.add(BidderError.generic(error));
-                }
-            } else {
-                validBids.add(bid);
+                addAsBidderErrors(validationResult.getErrors(), errors);
+                continue;
             }
+
+            validBids.add(bid);
         }
 
         return errors.isEmpty()
                 ? bidderResponse
                 : bidderResponse.with(BidderSeatBid.of(validBids, seatBid.getHttpCalls(), errors));
     }
 
+    private void addAsBidderErrors(List<String> messages, List<BidderError> errors) {
+        messages.stream().map(BidderError::generic).forEach(errors::add);
+    }
+
     /**
      * Performs changes on {@link Bid}s price depends on different between adServerCurrency and bidCurrency,
      * and adjustment factor. Will drop bid if currency conversion is needed but not possible.

src/main/java/org/prebid/server/metric/AccountMetrics.java

@@ -21,6 +21,7 @@ class AccountMetrics extends UpdatableMetrics {
     private final Map<MetricName, RequestTypeMetrics> requestTypeMetrics;
     private final RequestMetrics requestsMetrics;
     private final CacheMetrics cacheMetrics;
+    private final ResponseMetrics responseMetrics;
 
     AccountMetrics(MetricRegistry metricRegistry, CounterType counterType, String account) {
         super(Objects.requireNonNull(metricRegistry), Objects.requireNonNull(counterType),
@@ -32,6 +33,7 @@ class AccountMetrics extends UpdatableMetrics {
         requestTypeMetrics = new HashMap<>();
         requestsMetrics = new RequestMetrics(metricRegistry, counterType, createPrefix(account));
         cacheMetrics = new CacheMetrics(metricRegistry, counterType, createPrefix(account));
+        responseMetrics = new ResponseMetrics(metricRegistry, counterType, createPrefix(account));
     }
 
     private static String createPrefix(String account) {
@@ -57,4 +59,8 @@ RequestMetrics requests() {
     CacheMetrics cache() {
         return cacheMetrics;
     }
+
+    ResponseMetrics response() {
+        return responseMetrics;
+    }
 }

src/main/java/org/prebid/server/metric/AdapterMetrics.java

@@ -17,6 +17,7 @@ class AdapterMetrics extends UpdatableMetrics {
     private final RequestMetrics requestMetrics;
     private final Function<String, BidTypeMetrics> bidTypeMetricsCreator;
     private final Map<String, BidTypeMetrics> bidTypeMetrics;
+    private final ResponseMetrics responseMetrics;
 
     AdapterMetrics(MetricRegistry metricRegistry, CounterType counterType, String adapterType) {
         super(Objects.requireNonNull(metricRegistry), Objects.requireNonNull(counterType),
@@ -29,6 +30,7 @@ class AdapterMetrics extends UpdatableMetrics {
         requestTypeMetrics = new HashMap<>();
         requestMetrics = new RequestMetrics(metricRegistry, counterType, createAdapterPrefix(adapterType));
         bidTypeMetrics = new HashMap<>();
+        responseMetrics = new ResponseMetrics(metricRegistry, counterType, createAdapterPrefix(adapterType));
     }
 
     AdapterMetrics(MetricRegistry metricRegistry, CounterType counterType, String account, String adapterType) {
@@ -44,6 +46,7 @@ class AdapterMetrics extends UpdatableMetrics {
         requestTypeMetrics = null;
         bidTypeMetricsCreator = null;
         bidTypeMetrics = null;
+        responseMetrics = null;
     }
 
     private static String createAdapterPrefix(String adapterType) {
@@ -69,4 +72,8 @@ RequestMetrics request() {
     BidTypeMetrics forBidType(String bidType) {
         return bidTypeMetrics.computeIfAbsent(bidType, bidTypeMetricsCreator);
     }
+
+    ResponseMetrics response() {
+        return responseMetrics;
+    }
 }

src/main/java/org/prebid/server/metric/MetricName.java

@@ -62,6 +62,9 @@ public enum MetricName {
     err,
     networkerr,
 
+    // bids validation
+    warn,
+
     // cookie sync
     cookie_sync_requests,
     opt_outs,

src/main/java/org/prebid/server/metric/Metrics.java

@@ -276,6 +276,16 @@ public void updateAdapterRequestErrorMetric(String bidder, MetricName errorMetri
         forAdapter(resolveMetricsBidderName(bidder)).request().incCounter(errorMetric);
     }
 
+    public void updateSizeValidationMetrics(String bidder, String accountId, MetricName type) {
+        forAdapter(resolveMetricsBidderName(bidder)).response().validation().size().incCounter(type);
+        forAccount(accountId).response().validation().size().incCounter(type);
+    }
+
+    public void updateSecureValidationMetrics(String bidder, String accountId, MetricName type) {
+        forAdapter(resolveMetricsBidderName(bidder)).response().validation().secure().incCounter(type);
+        forAccount(accountId).response().validation().secure().incCounter(type);
+    }
+
     public void updateUserSyncOptoutMetric() {
         userSync().incCounter(MetricName.opt_outs);
     }

src/main/java/org/prebid/server/metric/ResponseMetrics.java

@@ -0,0 +1,33 @@
+package org.prebid.server.metric;
+
+import com.codahale.metrics.MetricRegistry;
+
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Request metrics support.
+ */
+class ResponseMetrics extends UpdatableMetrics {
+
+    private final ValidationMetrics validationMetrics;
+
+    ResponseMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
+        super(Objects.requireNonNull(metricRegistry), Objects.requireNonNull(counterType),
+                nameCreator(createPrefix(Objects.requireNonNull(prefix))));
+
+        validationMetrics = new ValidationMetrics(metricRegistry, counterType, createPrefix(prefix));
+    }
+
+    private static Function<MetricName, String> nameCreator(String prefix) {
+        return metricName -> String.format("%s.%s", prefix, metricName.toString());
+    }
+
+    private static String createPrefix(String prefix) {
+        return String.format("%s.response", prefix);
+    }
+
+    ValidationMetrics validation() {
+        return validationMetrics;
+    }
+}

src/main/java/org/prebid/server/metric/SpecificValidationMetrics.java

@@ -0,0 +1,27 @@
+package org.prebid.server.metric;
+
+import com.codahale.metrics.MetricRegistry;
+
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Request metrics support.
+ */
+class SpecificValidationMetrics extends UpdatableMetrics {
+
+    SpecificValidationMetrics(
+            MetricRegistry metricRegistry, CounterType counterType, String prefix, String validation) {
+
+        super(Objects.requireNonNull(metricRegistry), Objects.requireNonNull(counterType),
+                nameCreator(createPrefix(Objects.requireNonNull(prefix), Objects.requireNonNull(validation))));
+    }
+
+    private static Function<MetricName, String> nameCreator(String prefix) {
+        return metricName -> String.format("%s.%s", prefix, metricName.toString());
+    }
+
+    private static String createPrefix(String prefix, String validation) {
+        return String.format("%s.%s", prefix, validation);
+    }
+}

src/main/java/org/prebid/server/metric/ValidationMetrics.java

@@ -0,0 +1,41 @@
+package org.prebid.server.metric;
+
+import com.codahale.metrics.MetricRegistry;
+
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Request metrics support.
+ */
+class ValidationMetrics extends UpdatableMetrics {
+
+    private final SpecificValidationMetrics sizeValidationMetrics;
+    private final SpecificValidationMetrics secureValidationMetrics;
+
+    ValidationMetrics(MetricRegistry metricRegistry, CounterType counterType, String prefix) {
+        super(Objects.requireNonNull(metricRegistry), Objects.requireNonNull(counterType),
+                nameCreator(createPrefix(Objects.requireNonNull(prefix))));
+
+        sizeValidationMetrics = new SpecificValidationMetrics(
+                metricRegistry, counterType, createPrefix(prefix), "size");
+        secureValidationMetrics = new SpecificValidationMetrics(
+                metricRegistry, counterType, createPrefix(prefix), "secure");
+    }
+
+    private static Function<MetricName, String> nameCreator(String prefix) {
+        return metricName -> String.format("%s.%s", prefix, metricName.toString());
+    }
+
+    private static String createPrefix(String prefix) {
+        return String.format("%s.validation", prefix);
+    }
+
+    SpecificValidationMetrics size() {
+        return sizeValidationMetrics;
+    }
+
+    SpecificValidationMetrics secure() {
+        return secureValidationMetrics;
+    }
+}