Implement default bid request support (#850)

docs/config-app.md

@@ -63,6 +63,9 @@ Removes and downloads file again if depending service cant process probably corr
 - `max-timeout-ms` - this setting controls maximum timeout for /auction endpoint.
 - `timeout-adjustment-ms` - reduces timeout value passed in legacy Auction request so that Prebid Server can handle timeouts from adapters and respond to the request before it times out.
 
+## Default bid request
+- `default-request.file.path` - path to a JSON file containing the default request
+
 ## Auction (OpenRTB)
 - `auction.blacklisted-accounts` - comma separated list of blacklisted account IDs.
 - `auction.blacklisted-apps` - comma separated list of blacklisted applications IDs, requests from which should not be processed.

docs/developers/default-request.md

@@ -0,0 +1,29 @@
+# Server Based Global Default Request
+
+This allows a default request to be defined that allows the server to set up some defaults for all incoming 
+requests. A stored request(s) referenced by a bid request override default request, and of course any options specified 
+directly in the bid request override both. The default request is only read on server startup, it is meant as an 
+installation static default rather than a dynamic tuning option.
+
+## Config Options
+
+Following config options are exposed to support this feature.
+```yaml
+default-request:
+  file:
+    path : path/to/default/request.json
+```
+
+The `path` option is the path to a JSON file containing the default request JSON.
+```json
+{
+    "tmax": "2000",
+    "regs": {
+        "ext": {
+            "gdpr": 1
+        }
+    }
+}
+```
+This will be JSON merged into the incoming requests at the top level. These will be used as fallbacks which can be 
+overridden by both Stored Requests _and_ the incoming HTTP request payload.

src/main/java/org/prebid/server/auction/FpdResolver.java

@@ -10,6 +10,7 @@
 import org.apache.commons.collections4.CollectionUtils;
 import org.prebid.server.exception.InvalidRequestException;
 import org.prebid.server.json.JacksonMapper;
+import org.prebid.server.json.JsonMerger;
 import org.prebid.server.proto.openrtb.ext.request.ExtApp;
 import org.prebid.server.proto.openrtb.ext.request.ExtBidderConfig;
 import org.prebid.server.proto.openrtb.ext.request.ExtBidderConfigFpd;
@@ -21,7 +22,6 @@
 import org.prebid.server.proto.openrtb.ext.request.ExtSite;
 import org.prebid.server.proto.openrtb.ext.request.ExtUser;
 import org.prebid.server.proto.request.Targeting;
-import org.prebid.server.util.JsonMergeUtil;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -53,12 +53,11 @@ public class FpdResolver {
             "privacypolicy", "mobile"));
 
     private final JacksonMapper jacksonMapper;
-    private final JsonMergeUtil jsonMergeUtil;
+    private final JsonMerger jsonMerger;
 
-    public FpdResolver(JacksonMapper jacksonMapper) {
+    public FpdResolver(JacksonMapper jacksonMapper, JsonMerger jsonMerger) {
         this.jacksonMapper = Objects.requireNonNull(jacksonMapper);
-
-        this.jsonMergeUtil = new JsonMergeUtil(jacksonMapper);
+        this.jsonMerger = Objects.requireNonNull(jsonMerger);
     }
 
     public User resolveUser(User originUser, ObjectNode fpdUser) {
@@ -183,7 +182,7 @@ public ObjectNode resolveImpExt(ObjectNode impExt, ObjectNode targeting) {
                 : null;
 
         final ObjectNode resolvedData = extImpContextData != null
-                ? (ObjectNode) jsonMergeUtil.merge(targeting, extImpContextData)
+                ? (ObjectNode) jsonMerger.merge(targeting, extImpContextData)
                 : targeting;
 
         return extImpContext != null && extImpContext.isObject()
@@ -253,7 +252,7 @@ private ObjectNode mergeExtData(JsonNode fpdData, JsonNode originData) {
         }
 
         if (originData != null && originData.isObject()) {
-            return (ObjectNode) jsonMergeUtil.merge(fpdData, originData);
+            return (ObjectNode) jsonMerger.merge(fpdData, originData);
         }
         return fpdData.isObject() ? (ObjectNode) fpdData : null;
     }

src/main/java/org/prebid/server/auction/OrtbTypesResolver.java

@@ -12,7 +12,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.prebid.server.exception.InvalidRequestException;
 import org.prebid.server.json.JacksonMapper;
-import org.prebid.server.util.JsonMergeUtil;
+import org.prebid.server.json.JsonMerger;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -69,11 +69,11 @@ public class OrtbTypesResolver {
     }
 
     private final JacksonMapper jacksonMapper;
-    private final JsonMergeUtil jsonMergeUtil;
+    private final JsonMerger jsonMerger;
 
-    public OrtbTypesResolver(JacksonMapper jacksonMapper) {
+    public OrtbTypesResolver(JacksonMapper jacksonMapper, JsonMerger jsonMerger) {
         this.jacksonMapper = Objects.requireNonNull(jacksonMapper);
-        this.jsonMergeUtil = new JsonMergeUtil(jacksonMapper);
+        this.jsonMerger = Objects.requireNonNull(jsonMerger);
     }
 
     /**
@@ -254,7 +254,7 @@ public void normalizeDataExtension(ObjectNode containerNode, String containerNam
         final JsonNode extData = containerNode.path(EXT).path(DATA);
         final JsonNode ext = containerNode.get(EXT);
         if (!extData.isNull() && !extData.isMissingNode()) {
-            final JsonNode resolvedExtData = jsonMergeUtil.merge(extData, data);
+            final JsonNode resolvedExtData = jsonMerger.merge(extData, data);
             ((ObjectNode) ext).set(DATA, resolvedExtData);
         } else {
             copyDataToExtData(containerNode, containerName, nodePrefix, warnings, data);

src/main/java/org/prebid/server/auction/StoredRequestProcessor.java

@@ -5,12 +5,14 @@
 import com.iab.openrtb.request.Imp;
 import com.iab.openrtb.request.Video;
 import io.vertx.core.Future;
+import io.vertx.core.file.FileSystem;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.prebid.server.exception.InvalidRequestException;
 import org.prebid.server.execution.Timeout;
 import org.prebid.server.execution.TimeoutFactory;
 import org.prebid.server.json.JacksonMapper;
+import org.prebid.server.json.JsonMerger;
 import org.prebid.server.metric.Metrics;
 import org.prebid.server.proto.openrtb.ext.request.ExtImp;
 import org.prebid.server.proto.openrtb.ext.request.ExtImpPrebid;
@@ -20,7 +22,6 @@
 import org.prebid.server.settings.ApplicationSettings;
 import org.prebid.server.settings.model.StoredDataResult;
 import org.prebid.server.settings.model.VideoStoredDataResult;
-import org.prebid.server.util.JsonMergeUtil;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -39,25 +40,48 @@
 public class StoredRequestProcessor {
 
     private final long defaultTimeout;
+    private final BidRequest defaultBidRequest;
     private final ApplicationSettings applicationSettings;
     private final TimeoutFactory timeoutFactory;
     private final Metrics metrics;
     private final JacksonMapper mapper;
-    private JsonMergeUtil jsonMergeUtil;
+    private final JsonMerger jsonMerger;
 
-    public StoredRequestProcessor(long defaultTimeout,
-                                  ApplicationSettings applicationSettings,
-                                  Metrics metrics,
-                                  TimeoutFactory timeoutFactory,
-                                  JacksonMapper mapper) {
+    private StoredRequestProcessor(long defaultTimeout,
+                                   BidRequest defaultBidRequest,
+                                   ApplicationSettings applicationSettings,
+                                   Metrics metrics,
+                                   TimeoutFactory timeoutFactory,
+                                   JacksonMapper mapper,
+                                   JsonMerger jsonMerger) {
 
         this.defaultTimeout = defaultTimeout;
-        this.applicationSettings = Objects.requireNonNull(applicationSettings);
-        this.timeoutFactory = Objects.requireNonNull(timeoutFactory);
-        this.metrics = Objects.requireNonNull(metrics);
-        this.mapper = Objects.requireNonNull(mapper);
+        this.defaultBidRequest = defaultBidRequest;
+        this.applicationSettings = applicationSettings;
+        this.timeoutFactory = timeoutFactory;
+        this.metrics = metrics;
+        this.mapper = mapper;
+        this.jsonMerger = jsonMerger;
+    }
 
-        jsonMergeUtil = new JsonMergeUtil(mapper);
+    public static StoredRequestProcessor create(long defaultTimeout,
+                                                String defaultBidRequestPath,
+                                                FileSystem fileSystem,
+                                                ApplicationSettings applicationSettings,
+                                                Metrics metrics,
+                                                TimeoutFactory timeoutFactory,
+                                                JacksonMapper mapper,
+                                                JsonMerger jsonMerger) {
+
+        return new StoredRequestProcessor(
+                defaultTimeout,
+                readBidRequest(
+                        defaultBidRequestPath, Objects.requireNonNull(fileSystem), Objects.requireNonNull(mapper)),
+                Objects.requireNonNull(applicationSettings),
+                Objects.requireNonNull(metrics),
+                Objects.requireNonNull(timeoutFactory),
+                Objects.requireNonNull(mapper),
+                Objects.requireNonNull(jsonMerger));
     }
 
     /**
@@ -89,25 +113,15 @@ Future<BidRequest> processStoredRequests(String accountId, BidRequest bidRequest
                 applicationSettings.getStoredData(accountId, requestIds, impIds, timeout(bidRequest))
                         .compose(storedDataResult -> updateMetrics(storedDataResult, requestIds, impIds));
 
-        return storedRequestsToBidRequest(storedDataFuture, bidRequest,
-                bidRequestToStoredRequestId.get(bidRequest), impToStoredRequestId);
-    }
-
-    private Future<StoredDataResult> updateMetrics(StoredDataResult storedDataResult, Set<String> requestIds,
-                                                   Set<String> impIds) {
-        requestIds.forEach(
-                id -> metrics.updateStoredRequestMetric(storedDataResult.getStoredIdToRequest().containsKey(id)));
-        impIds.forEach(
-                id -> metrics.updateStoredImpsMetric(storedDataResult.getStoredIdToImp().containsKey(id)));
-
-        return Future.succeededFuture(storedDataResult);
+        return storedRequestsToBidRequest(
+                storedDataFuture, bidRequest, bidRequestToStoredRequestId.get(bidRequest), impToStoredRequestId);
     }
 
     /**
      * Fetches AMP request from the source.
      */
     Future<BidRequest> processAmpRequest(String accountId, String ampRequestId) {
-        final BidRequest bidRequest = BidRequest.builder().build();
+        final BidRequest bidRequest = defaultBidRequest != null ? defaultBidRequest : BidRequest.builder().build();
         final Future<StoredDataResult> ampStoredDataFuture =
                 applicationSettings.getAmpStoredData(
                         accountId, Collections.singleton(ampRequestId), Collections.emptySet(), timeout(bidRequest))
@@ -132,9 +146,27 @@ Future<VideoStoredDataResult> videoStoredDataResult(String accountId, List<Imp>
                 .map(storedDataResult -> makeVideoStoredDataResult(storedDataResult, storedIdToImpId, errors));
     }
 
+    private Future<StoredDataResult> updateMetrics(StoredDataResult storedDataResult, Set<String> requestIds,
+                                                   Set<String> impIds) {
+        requestIds.forEach(
+                id -> metrics.updateStoredRequestMetric(storedDataResult.getStoredIdToRequest().containsKey(id)));
+        impIds.forEach(id -> metrics.updateStoredImpsMetric(storedDataResult.getStoredIdToImp().containsKey(id)));
+
+        return Future.succeededFuture(storedDataResult);
+    }
+
+    private static BidRequest readBidRequest(
+            String defaultBidRequestPath, FileSystem fileSystem, JacksonMapper mapper) {
+
+        return StringUtils.isNotBlank(defaultBidRequestPath)
+                ? mapper.decodeValue(fileSystem.readFileBlocking(defaultBidRequestPath), BidRequest.class)
+                : null;
+    }
+
     private VideoStoredDataResult makeVideoStoredDataResult(StoredDataResult storedDataResult,
                                                             Map<String, String> storedIdToImpId,
                                                             List<String> errors) {
+
         final Map<String, String> storedIdToStoredImp = storedDataResult.getStoredIdToImp();
         final Map<String, Video> impIdToStoredVideo = new HashMap<>();
 
@@ -172,8 +204,10 @@ private Video parseVideoFromImp(String storedJson) {
     }
 
     private Future<BidRequest> storedRequestsToBidRequest(Future<StoredDataResult> storedDataFuture,
-                                                          BidRequest bidRequest, String storedBidRequestId,
+                                                          BidRequest bidRequest,
+                                                          String storedBidRequestId,
                                                           Map<Imp, String> impsToStoredRequestId) {
+
         return storedDataFuture
                 .recover(exception -> Future.failedFuture(new InvalidRequestException(
                         String.format("Stored request fetching failed: %s", exception.getMessage()))))
@@ -187,30 +221,41 @@ private Future<BidRequest> storedRequestsToBidRequest(Future<StoredDataResult> s
     /**
      * Runs {@link BidRequest} and {@link Imp}s merge processes.
      */
-    private BidRequest mergeBidRequestAndImps(BidRequest bidRequest, String storedRequestId,
-                                              Map<Imp, String> impToStoredId, StoredDataResult storedDataResult) {
-        return mergeBidRequestImps(mergeBidRequest(bidRequest, storedRequestId, storedDataResult),
-                impToStoredId, storedDataResult);
+    private BidRequest mergeBidRequestAndImps(BidRequest bidRequest,
+                                              String storedRequestId,
+                                              Map<Imp, String> impToStoredId,
+                                              StoredDataResult storedDataResult) {
+
+        return mergeBidRequestImps(
+                mergeBidRequest(mergeDefaultRequest(bidRequest), storedRequestId, storedDataResult),
+                impToStoredId,
+                storedDataResult);
+    }
+
+    private BidRequest mergeDefaultRequest(BidRequest bidRequest) {
+        return jsonMerger.merge(bidRequest, defaultBidRequest, BidRequest.class);
     }
 
     /**
      * Merges original request with request from stored request source. Values from original request
      * has higher priority than stored request values.
      */
-    private BidRequest mergeBidRequest(BidRequest originalRequest, String storedRequestId,
-                                       StoredDataResult storedDataResult) {
+    private BidRequest mergeBidRequest(
+            BidRequest originalRequest, String storedRequestId, StoredDataResult storedDataResult) {
+
         final String storedRequest = storedDataResult.getStoredIdToRequest().get(storedRequestId);
         return StringUtils.isNotBlank(storedRequestId)
-                ? jsonMergeUtil.merge(originalRequest, storedRequest, storedRequestId, BidRequest.class)
+                ? jsonMerger.merge(originalRequest, storedRequest, storedRequestId, BidRequest.class)
                 : originalRequest;
     }
 
     /**
      * Merges {@link Imp}s from original request with Imps from stored request source. Values from original request
      * has higher priority than stored request values.
      */
-    private BidRequest mergeBidRequestImps(BidRequest bidRequest, Map<Imp, String> impToStoredId,
-                                           StoredDataResult storedDataResult) {
+    private BidRequest mergeBidRequestImps(
+            BidRequest bidRequest, Map<Imp, String> impToStoredId, StoredDataResult storedDataResult) {
+
         if (impToStoredId.isEmpty()) {
             return bidRequest;
         }
@@ -220,18 +265,18 @@ private BidRequest mergeBidRequestImps(BidRequest bidRequest, Map<Imp, String> i
             final String storedRequestId = impToStoredId.get(imp);
             if (storedRequestId != null) {
                 final String storedImp = storedDataResult.getStoredIdToImp().get(storedRequestId);
-                final Imp mergedImp = jsonMergeUtil.merge(imp, storedImp, storedRequestId, Imp.class);
+                final Imp mergedImp = jsonMerger.merge(imp, storedImp, storedRequestId, Imp.class);
                 mergedImps.set(i, mergedImp);
             }
         }
         return bidRequest.toBuilder().imp(mergedImps).build();
     }
 
     /**
-     * Maps object to its StoredRequestId if exists. If object's extension contains storedRequest field, expected that
-     * it includes id too, in another case error about missed id in stored request will be added to error list.
-     * Gathers all errors into list, and in case if it is not empty, throws {@link InvalidRequestException} with
-     * list of errors.
+     * Maps object to its StoredRequestId if exists. If object's extension contains storedRequest field, expected
+     * that it includes id too, in another case error about missed id in stored request will be added to error list.
+     * Gathers all errors into list, and in case if it is not empty, throws {@link InvalidRequestException} with list
+     * of errors.
      */
     private <K> Map<K, String> mapStoredRequestHolderToStoredRequestId(
             List<K> storedRequestHolders, Function<K, ExtStoredRequest> storedRequestExtractor) {

src/main/java/org/prebid/server/auction/VideoRequestFactory.java

@@ -42,10 +42,13 @@ public class VideoRequestFactory {
     private final TimeoutResolver timeoutResolver;
     private final JacksonMapper mapper;
 
-    public VideoRequestFactory(int maxRequestSize, boolean enforceStoredRequest,
+    public VideoRequestFactory(int maxRequestSize,
+                               boolean enforceStoredRequest,
                                VideoStoredRequestProcessor storedRequestProcessor,
-                               AuctionRequestFactory auctionRequestFactory, TimeoutResolver timeoutResolver,
+                               AuctionRequestFactory auctionRequestFactory,
+                               TimeoutResolver timeoutResolver,
                                JacksonMapper mapper) {
+
         this.enforceStoredRequest = enforceStoredRequest;
         this.maxRequestSize = maxRequestSize;
         this.storedRequestProcessor = Objects.requireNonNull(storedRequestProcessor);
@@ -175,8 +178,9 @@ private Future<WithPodErrors<BidRequest>> createBidRequest(RoutingContext routin
                                                                BidRequestVideo bidRequestVideo,
                                                                String storedVideoId,
                                                                Set<String> podConfigIds) {
-        return storedRequestProcessor.processVideoRequest(accountIdFrom(bidRequestVideo), storedVideoId, podConfigIds,
-                bidRequestVideo)
+
+        return storedRequestProcessor.processVideoRequest(
+                accountIdFrom(bidRequestVideo), storedVideoId, podConfigIds, bidRequestVideo)
                 .map(bidRequestToErrors -> fillImplicitParameters(routingContext, bidRequestToErrors))
                 .map(this::validateRequest);
     }