Merge branch 'master' into add-adocean-bidder

# Conflicts: # src/test/resources/org/prebid/server/it/test-application.properties
prebid · Sep 15, 2020 · 408ac85 · 408ac85
2 parents 4992d0b + 53ad16e
commit 408ac85
Show file tree

Hide file tree

Showing 104 changed files with 4,761 additions and 1,336 deletions.
diff --git a/docs/application-settings.md b/docs/application-settings.md
@@ -21,6 +21,8 @@ There are two ways to configure application settings: database and file. This do
 - `gdpr.purpose-one-treatment-interpretation` - option that allows to skip the Purpose one enforcement workflow. Values: ignore, no-access-allowed, access-allowed.
 - `analytics-sampling-factor` - Analytics sampling factor value. 
 - `truncate-target-attr` - Maximum targeting attributes size. Values between 1 and 255.
+- `default-integration` - Default integration to assume.
+- `analytics-config.auction-events.<channel>` - defines which channels are supported by analytics for this account
 
 ```
 Purpose   | Purpose goal                    | Purpose meaning for PBS (n\a - not affected)  
@@ -63,6 +65,10 @@ accounts:
     enforceCcpa: true
     analyticsSamplingFactor: 1
     truncateTargetAttr: 40
+    defaultIntegration: web
+    analytics-config:
+      auction-events:
+        amp: true
     gdpr:
       enabled: true
       integration-enabled:
@@ -179,13 +185,15 @@ Query to create accounts_account table:
 `tcf_config` json DEFAULT NULL,
 `analytics_sampling_factor` tinyint(4) DEFAULT NULL,
 `truncate_target_attr` tinyint(3) unsigned DEFAULT NULL,
+`default_integration` varchar(64) DEFAULT NULL,
+`analytics_config` varchar(512) DEFAULT NULL,
 `status` enum('active','inactive') DEFAULT 'active',
 `updated_by` int(11) DEFAULT NULL,
 `updated_by_user` varchar(64) DEFAULT NULL,
 `updated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 PRIMARY KEY (`id`),
 UNIQUE KEY `uuid` (`uuid`))
-ENGINE=InnoDB AUTO_INCREMENT=1726 DEFAULT CHARSET=utf8'
+ENGINE=InnoDB DEFAULT CHARSET=utf8'
 ```
 
 where tcf_config column is json with next format
@@ -303,7 +311,7 @@ where tcf_config column is json with next format
 
 Query used to get an account:
 ```
-SELECT uuid, price_granularity, banner_cache_ttl, video_cache_ttl, events_enabled, enforce_ccpa, tcf_config, analytics_sampling_factor, truncate_target_attr 
+SELECT uuid, price_granularity, banner_cache_ttl, video_cache_ttl, events_enabled, enforce_ccpa, tcf_config, analytics_sampling_factor, truncate_target_attr, default_integration, analytics_config 
 FROM accounts_account where uuid = ?
 LIMIT 1
 

diff --git a/docs/config-app.md b/docs/config-app.md
@@ -114,8 +114,11 @@ There are several typical keys:
 
 But feel free to add additional bidder's specific options.
 
-## Logger Level Modifier
-- `logger-level-modifier.enabled` - enable the `/admin` endpoint.
+## Logging
+- `logging.http-interaction.max-limit` - maximum value for the number of interactions to log in one take.
+
+## Logging
+- `logging.change-level.max-duration-ms` - maximum duration (in milliseconds) for which logging level could be changed.
 
 ## Currency Converter
 - `currency-converter.external-rates.enabled` - if equals to `true` the currency conversion service will be enabled to fetch updated rates and convert bid currencies from external source. Also enables `/currency-rates` endpoint on admin port.
@@ -149,10 +152,10 @@ But feel free to add additional bidder's specific options.
 - `admin-endpoints.cache-invalidation.on-application-port` - when equals to `false` endpoint will be bound to `admin.port`.
 - `admin-endpoints.cache-invalidation.protected` - when equals to `true` endpoint will be protected by basic authentication configured in `admin-endpoints.credentials` 
 
-- `admin-endpoints.logger-level-modifier.enabled` - if equals to `true` the endpoint will be available.
-- `admin-endpoints.logger-level-modifier.path` - the server context path where the endpoint will be accessible.
-- `admin-endpoints.logger-level-modifier.on-application-port` - when equals to `false` endpoint will be bound to `admin.port`.
-- `admin-endpoints.logger-level-modifier.protected` - when equals to `true` endpoint will be protected by basic authentication configured in `admin-endpoints.credentials` 
+- `admin-endpoints.logging-httpinteraction.enabled` - if equals to `true` the endpoint will be available.
+- `admin-endpoints.logging-httpinteraction.path` - the server context path where the endpoint will be accessible.
+- `admin-endpoints.logging-httpinteraction.on-application-port` - when equals to `false` endpoint will be bound to `admin.port`.
+- `admin-endpoints.logging-httpinteraction.protected` - when equals to `true` endpoint will be protected by basic authentication configured in `admin-endpoints.credentials` 
 
 - `admin-endpoints.credentials` - user and password for access to admin endpoints if `admin-endpoints.[NAME].protected` is true`.
 

diff --git a/docs/endpoints/admin.md b/docs/endpoints/admin.md
@@ -1,10 +0,0 @@
-# Admin Endpoint
-
-Unavailable if `logger level modifier` is disabled (`logger-level-modifier.enabled` config property).
-
-This endpoint will set the desirable logging level and number of logs for `400` responses.
-
-### Query Params
-
-- `logging` - Desirable logging level: `info`, `warn`, `trace`, `error`, `fatal`, `debug`.
-- `records` - numbers of logs with changed logging level. (0 < n < 100_000).

diff --git a/docs/endpoints/logging/changelevel.md b/docs/endpoints/logging/changelevel.md
@@ -0,0 +1,10 @@
+# Change logging level endpoint
+
+This endpoint has a path `/logging/changelevel` by default (can be configured).
+
+This endpoint allows changing `org.prebid.server` logger level temporarily, mainly for troubleshooting production issues.
+
+### Query Params
+- `level` - desired logging level to set; must be one of `error`, `warn`, `info`, `debug`
+- `duration` - for how long to change level before it gets reset to original; there is an upper threshold for this 
+value set in [configuration](../../config-app.md)
diff --git a/docs/endpoints/logging/httpinteraction.md b/docs/endpoints/logging/httpinteraction.md
@@ -0,0 +1,18 @@
+# Enable HTTP interaction logging endpoint
+
+This endpoint has a path `/logging/httpinteraction` by default (can be configured).
+
+This endpoint turns on temporary logging of raw HTTP requests and responses, mainly for troubleshooting production issues. 
+
+Interaction is logged at `INFO` level using `http-interaction` logback logger so make sure this logger has at least 
+`INFO` or more verbose level set ([logback configuration](../../../src/main/resources/logback-spring.xml) bundled in JAR 
+file sets this logger to `INFO` level).
+
+### Query Params
+- `endpoint` - endpoint to be affected; valid values: [auction](../openrtb2/auction.md), [amp](../openrtb2/amp.md); 
+if omitted all valid endpoints will be affected
+- `statusCode` - specifies that only interactions resulting in this response status code should be logged; 
+valid values: >=200 and <=500
+- `account` - specifies that only interactions involving this account should be logged
+- `limit` - number of interactions to log; there is an upper threshold for this value set in 
+[configuration](../../config-app.md) 
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>org.prebid</groupId>
     <artifactId>prebid-server</artifactId>
-    <version>1.42.0-SNAPSHOT</version>
+    <version>1.43.0-SNAPSHOT</version>
 
     <name>prebid-server</name>
     <description>Prebid Server (Server-side Header Bidding)</description>

diff --git a/src/main/java/org/prebid/server/analytics/model/NotificationEvent.java b/src/main/java/org/prebid/server/analytics/model/NotificationEvent.java
@@ -21,6 +21,8 @@ public class NotificationEvent {
 
     Long timestamp;
 
+    String integration;
+
     HttpContext httpContext;
 
     public enum Type {

diff --git a/src/main/java/org/prebid/server/auction/AmpRequestFactory.java b/src/main/java/org/prebid/server/auction/AmpRequestFactory.java
@@ -35,6 +35,7 @@
 import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebidCache;
 import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebidCacheBids;
 import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebidCacheVastxml;
+import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebidChannel;
 import org.prebid.server.proto.openrtb.ext.request.ExtRequestTargeting;
 import org.prebid.server.proto.openrtb.ext.request.ExtSite;
 import org.prebid.server.proto.openrtb.ext.request.ExtUser;
@@ -67,24 +68,29 @@ public class AmpRequestFactory {
     private static final String TIMEOUT_REQUEST_PARAM = "timeout";
     private static final String GDPR_CONSENT_PARAM = "gdpr_consent";
     private static final String CONSENT_PARAM = "consent_string";
+
     private static final int NO_LIMIT_SPLIT_MODE = -1;
+    private static final String AMP_CHANNEL = "amp";
 
     private final StoredRequestProcessor storedRequestProcessor;
     private final AuctionRequestFactory auctionRequestFactory;
     private final OrtbTypesResolver ortbTypesResolver;
+    private final ImplicitParametersExtractor implicitParametersExtractor;
     private final FpdResolver fpdResolver;
     private final TimeoutResolver timeoutResolver;
     private final JacksonMapper mapper;
 
     public AmpRequestFactory(StoredRequestProcessor storedRequestProcessor,
                              AuctionRequestFactory auctionRequestFactory,
                              OrtbTypesResolver ortbTypesResolver,
+                             ImplicitParametersExtractor implicitParametersExtractor,
                              FpdResolver fpdResolver,
                              TimeoutResolver timeoutResolver,
                              JacksonMapper mapper) {
         this.storedRequestProcessor = Objects.requireNonNull(storedRequestProcessor);
         this.auctionRequestFactory = Objects.requireNonNull(auctionRequestFactory);
         this.ortbTypesResolver = Objects.requireNonNull(ortbTypesResolver);
+        this.implicitParametersExtractor = Objects.requireNonNull(implicitParametersExtractor);
         this.fpdResolver = Objects.requireNonNull(fpdResolver);
         this.timeoutResolver = Objects.requireNonNull(timeoutResolver);
         this.mapper = Objects.requireNonNull(mapper);
@@ -108,8 +114,7 @@ public Future<AuctionContext> fromRequest(RoutingContext routingContext, long st
      * Creates {@link BidRequest} and sets properties which were not set explicitly by the client, but can be
      * updated by values derived from headers and other request attributes.
      */
-    private Future<Tuple2<BidRequest, List<String>>> createBidRequest(RoutingContext context,
-                                                                      String tagId) {
+    private Future<Tuple2<BidRequest, List<String>>> createBidRequest(RoutingContext context, String tagId) {
         final List<String> errors = new ArrayList<>();
         return storedRequestProcessor.processAmpRequest(tagId)
                 .map(bidRequest -> validateStoredBidRequest(tagId, bidRequest))
@@ -169,17 +174,23 @@ private BidRequest fillExplicitParameters(BidRequest bidRequest, RoutingContext
         final boolean setDefaultTargeting;
         final boolean setDefaultCache;
 
+        final boolean setChannel;
+
         if (prebid == null) {
             setDefaultTargeting = true;
             setDefaultCache = true;
+            setChannel = true;
         } else {
             final ExtRequestTargeting targeting = prebid.getTargeting();
             setDefaultTargeting = targeting == null
                     || targeting.getIncludewinners() == null
                     || targeting.getIncludebidderkeys() == null
                     || targeting.getPricegranularity() == null || targeting.getPricegranularity().isNull();
+
             final ExtRequestPrebidCache cache = prebid.getCache();
             setDefaultCache = cache == null || cache.equals(ExtRequestPrebidCache.EMPTY);
+
+            setChannel = prebid.getChannel() == null;
         }
 
         final Integer debugQueryParam = debugFromQueryStringParam(context);
@@ -200,6 +211,7 @@ private BidRequest fillExplicitParameters(BidRequest bidRequest, RoutingContext
         if (setSecure
                 || setDefaultTargeting
                 || setDefaultCache
+                || setChannel
                 || updatedTest != null
                 || updatedDebug != null
                 || updatedAmpData != null) {
@@ -208,7 +220,12 @@ private BidRequest fillExplicitParameters(BidRequest bidRequest, RoutingContext
                     .imp(setSecure ? Collections.singletonList(imps.get(0).toBuilder().secure(1).build()) : imps)
                     .test(ObjectUtils.defaultIfNull(updatedTest, test))
                     .ext(extRequest(
-                            bidRequest, prebid, setDefaultTargeting, setDefaultCache, updatedDebug, updatedAmpData))
+                            bidRequest,
+                            setDefaultTargeting,
+                            setDefaultCache,
+                            setChannel,
+                            updatedDebug,
+                            updatedAmpData))
                     .build();
         } else {
             result = bidRequest;
@@ -249,7 +266,7 @@ private BidRequest overrideParameters(BidRequest bidRequest, HttpServerRequest r
 
         final String requestTargeting = request.getParam(TARGETING_REQUEST_PARAM);
         final ObjectNode targetingNode = readTargeting(requestTargeting);
-        ortbTypesResolver.normalizeStandardFpdFields(targetingNode, errors, "targeting");
+        ortbTypesResolver.normalizeTargeting(targetingNode, errors, implicitParametersExtractor.refererFrom(request));
         final Targeting targeting = parseTargeting(targetingNode);
 
         final Site updatedSite = overrideSite(bidRequest.getSite(), request);
@@ -551,14 +568,16 @@ private static Map<String, String> updateAmpData(ExtRequestPrebid prebid, HttpSe
      * Creates updated bidrequest.ext {@link ObjectNode}.
      */
     private ExtRequest extRequest(BidRequest bidRequest,
-                                  ExtRequestPrebid prebid,
                                   boolean setDefaultTargeting,
                                   boolean setDefaultCache,
+                                  boolean setChannel,
                                   Integer updatedDebug,
                                   Map<String, String> updatedAmpData) {
 
         final ExtRequest result;
-        if (setDefaultTargeting || setDefaultCache || updatedDebug != null || updatedAmpData != null) {
+        if (setDefaultTargeting || setDefaultCache || setChannel || updatedDebug != null || updatedAmpData != null) {
+            final ExtRequest requestExt = bidRequest.getExt();
+            final ExtRequestPrebid prebid = requestExt != null ? requestExt.getPrebid() : null;
             final ExtRequestPrebid.ExtRequestPrebidBuilder prebidBuilder = prebid != null
                     ? prebid.toBuilder()
                     : ExtRequestPrebid.builder();
@@ -570,6 +589,9 @@ private ExtRequest extRequest(BidRequest bidRequest,
                 prebidBuilder.cache(ExtRequestPrebidCache.of(ExtRequestPrebidCacheBids.of(null, null),
                         ExtRequestPrebidCacheVastxml.of(null, null), null));
             }
+            if (setChannel) {
+                prebidBuilder.channel(ExtRequestPrebidChannel.of(AMP_CHANNEL));
+            }
             if (updatedDebug != null) {
                 prebidBuilder.debug(updatedDebug);
             }