Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Turn off debug by Publisher, by bidder #1443

Merged
merged 22 commits into from
Oct 7, 2021
Merged

Turn off debug by Publisher, by bidder #1443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
4d31654
Work in progress
And1sS Aug 26, 2021
3b98aba
Fixed all but one test
And1sS Aug 26, 2021
0d3fde3
Added DebugResolverTest
And1sS Aug 26, 2021
a1538e1
Added unit tests for exchange service
And1sS Aug 26, 2021
b79a1ba
Added tests for request factories
And1sS Aug 26, 2021
7412189
Updated test for account settings
And1sS Aug 26, 2021
80fb9e1
Added filtration for bidder debug http calls
And1sS Aug 26, 2021
4487c8a
Fixed styling
And1sS Aug 26, 2021
9a362aa
Removed redundant Bidder alias
And1sS Aug 26, 2021
46bd779
Fixed test in AuctionRequestFactoryTest
And1sS Aug 26, 2021
264ae96
Finished implementation
And1sS Aug 27, 2021
b93d941
Fixed styling
And1sS Aug 27, 2021
80b2916
Added unit tests for DebugResolver
And1sS Aug 27, 2021
d645fb2
Replaced AccountDebugConfig with debugAllow flag
And1sS Sep 2, 2021
733845b
Merge branch 'master' into turn-off-debug-by-publisher-by-bidder
And1sS Sep 2, 2021
6d25bcf
Fixed styling
And1sS Sep 2, 2021
35109ad
Merge branch 'master' into turn-off-debug-by-publisher-by-bidder
Sep 22, 2021
b7775bc
Merge branch 'master' into turn-off-debug-by-publisher-by-bidder
Sep 22, 2021
2e1efde
Merge branch 'master' into turn-off-debug-by-publisher-by-bidder
Oct 1, 2021
009ab8d
Turn off debug by Publisher, by bidder - proposal (#1513)
rpanchyk Oct 6, 2021
9f6f8e4
Merge branch 'master' into turn-off-debug-by-publisher-by-bidder
Oct 6, 2021
8307291
Fixes after review
Oct 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 21 additions & 19 deletions src/main/java/org/prebid/server/auction/BidResponseCreator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import io.vertx.core.CompositeFuture;
import io.vertx.core.Future;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.ObjectUtils;
Expand All @@ -28,6 +27,7 @@
import org.prebid.server.auction.model.BidRequestCacheInfo;
import org.prebid.server.auction.model.BidderResponse;
import org.prebid.server.auction.model.BidderResponseInfo;
import org.prebid.server.auction.model.DebugContext;
import org.prebid.server.auction.model.MultiBidConfig;
import org.prebid.server.auction.model.TargetingInfo;
import org.prebid.server.bidder.BidderCatalog;
Expand Down Expand Up @@ -346,13 +346,6 @@ private ExtBidPrebid updateBidExtPrebid(
.build();
}

private JsonNode getAndRemoveProperty(String propertyName, ObjectNode node) {
final JsonNode property = node.get(propertyName);
node.remove(propertyName);

return property;
}

/**
* Checks whether bidder responses are empty or contain no bids.
*/
Expand Down Expand Up @@ -647,7 +640,7 @@ private List<BidInfo> injectTargeting(List<BidInfo> bidderImpIdBidInfos,

final int multiBidSize = bidderImpIdBidInfos.size();
for (int i = 0; i < multiBidSize; i++) {
// first bid have highest value and can't be extra bid.
// first bid have the highest value and can't be extra bid
final boolean isFirstBid = i == 0;
final String targetingBidderCode = isFirstBid
? bidder
Expand Down Expand Up @@ -690,11 +683,11 @@ private ExtBidResponse toExtBidResponse(List<BidderResponseInfo> bidderResponseI
long auctionTimestamp,
Map<String, List<ExtBidderError>> bidErrors) {

final BidRequest bidRequest = auctionContext.getBidRequest();
final boolean debugEnabled = auctionContext.getDebugContext().isDebugEnabled();
final DebugContext debugContext = auctionContext.getDebugContext();
final boolean debugEnabled = debugContext.isDebugEnabled();

final ExtResponseDebug extResponseDebug =
toExtResponseDebug(bidderResponseInfos, auctionContext, cacheResult, debugEnabled);
final ExtResponseDebug extResponseDebug = toExtResponseDebug(
bidderResponseInfos, auctionContext, cacheResult, debugEnabled);
final Map<String, List<ExtBidderError>> errors =
toExtBidderErrors(bidderResponseInfos, auctionContext, cacheResult, videoStoredDataResult, bidErrors);
final Map<String, List<ExtBidderError>> warnings = debugEnabled
Expand All @@ -707,7 +700,7 @@ private ExtBidResponse toExtBidResponse(List<BidderResponseInfo> bidderResponseI
.errors(errors)
.warnings(warnings)
.responsetimemillis(responseTimeMillis)
.tmaxrequest(bidRequest.getTmax())
.tmaxrequest(auctionContext.getBidRequest().getTmax())
.prebid(ExtBidResponsePrebid.of(auctionTimestamp, null))
.build();
}
Expand All @@ -716,12 +709,15 @@ private ExtResponseDebug toExtResponseDebug(List<BidderResponseInfo> bidderRespo
AuctionContext auctionContext,
CacheServiceResult cacheResult,
boolean debugEnabled) {

final DeepDebugLog deepDebugLog = auctionContext.getDeepDebugLog();

final Map<String, List<ExtHttpCall>> httpCalls = debugEnabled
? toExtHttpCalls(bidderResponseInfos, cacheResult, auctionContext.getDebugHttpCalls())
: null;

final BidRequest bidRequest = debugEnabled ? auctionContext.getBidRequest() : null;

final ExtDebugPgmetrics extDebugPgmetrics = debugEnabled ? toExtDebugPgmetrics(
auctionContext.getTxnLog()) : null;
final ExtDebugTrace extDebugTrace = deepDebugLog.isDeepDebugEnabled() ? toExtDebugTrace(deepDebugLog) : null;
Expand Down Expand Up @@ -792,13 +788,15 @@ private static CacheServiceResult addNotCachedBids(CacheServiceResult cacheResul
return cacheResult;
}

private static Map<String, List<ExtHttpCall>> toExtHttpCalls(List<BidderResponseInfo> bidderResponses,
CacheServiceResult cacheResult,
Map<String, List<DebugHttpCall>> contextHttpCalls) {
private Map<String, List<ExtHttpCall>> toExtHttpCalls(List<BidderResponseInfo> bidderResponses,
CacheServiceResult cacheResult,
Map<String, List<DebugHttpCall>> contextHttpCalls) {

final Map<String, List<ExtHttpCall>> bidderHttpCalls = bidderResponses.stream()
.filter(bidderResponse -> CollectionUtils.isNotEmpty(bidderResponse.getSeatBid().getHttpCalls()))
.collect(Collectors.toMap(
BidderResponseInfo::getBidder,
bidderResponse -> ListUtils.emptyIfNull(bidderResponse.getSeatBid().getHttpCalls())));
bidderResponse -> bidderResponse.getSeatBid().getHttpCalls()));

final DebugHttpCall httpCall = cacheResult.getHttpCall();
final ExtHttpCall cacheExtHttpCall = httpCall != null ? toExtHttpCall(httpCall) : null;
Expand Down Expand Up @@ -848,12 +846,14 @@ private static ExtDebugPgmetrics toExtDebugPgmetrics(TxnLog txnLog) {

private static ExtDebugTrace toExtDebugTrace(DeepDebugLog deepDebugLog) {
final List<ExtTraceDeal> entries = deepDebugLog.entries();

final List<ExtTraceDeal> dealsTrace = entries.stream()
.filter(extTraceDeal -> StringUtils.isEmpty(extTraceDeal.getLineItemId()))
.collect(Collectors.toList());
final Map<String, List<ExtTraceDeal>> lineItemsTrace = entries.stream()
.filter(extTraceDeal -> StringUtils.isNotEmpty(extTraceDeal.getLineItemId()))
.collect(Collectors.groupingBy(ExtTraceDeal::getLineItemId, Collectors.toList()));

return CollectionUtils.isNotEmpty(entries)
? ExtDebugTrace.of(CollectionUtils.isEmpty(dealsTrace) ? null : dealsTrace,
MapUtils.isEmpty(lineItemsTrace) ? null : lineItemsTrace)
Expand All @@ -865,6 +865,7 @@ private Map<String, List<ExtBidderError>> toExtBidderErrors(List<BidderResponseI
CacheServiceResult cacheResult,
VideoStoredDataResult videoStoredDataResult,
Map<String, List<ExtBidderError>> bidErrors) {

final Map<String, List<ExtBidderError>> errors = new HashMap<>();

errors.putAll(extractBidderErrors(bidderResponses));
Expand Down Expand Up @@ -1356,6 +1357,7 @@ private long auctionTimestamp(AuctionContext auctionContext) {
final ExtRequest requestExt = auctionContext.getBidRequest().getExt();
final ExtRequestPrebid prebid = requestExt != null ? requestExt.getPrebid() : null;
final Long auctionTimestamp = prebid != null ? prebid.getAuctiontimestamp() : null;

return auctionTimestamp != null ? auctionTimestamp : clock.millis();
}

Expand Down Expand Up @@ -1392,6 +1394,7 @@ private TargetingKeywordsCreator resolveKeywordsCreator(BidType bidType,
boolean isApp,
BidRequest bidRequest,
Account account) {

final Map<BidType, TargetingKeywordsCreator> keywordsCreatorByBidType =
keywordsCreatorByBidType(targeting, isApp, bidRequest, account);

Expand Down Expand Up @@ -1534,7 +1537,6 @@ private ExtBidPrebid getExtPrebid(ObjectNode bidExt) {
if (bidExt == null || !bidExt.hasNonNull(PREBID_EXT)) {
return null;
}

return mapper.mapper().convertValue(bidExt.get(PREBID_EXT), ExtBidPrebid.class);
}
}
91 changes: 91 additions & 0 deletions src/main/java/org/prebid/server/auction/DebugResolver.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package org.prebid.server.auction;

import com.iab.openrtb.request.BidRequest;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.prebid.server.auction.model.AuctionContext;
import org.prebid.server.auction.model.DebugContext;
import org.prebid.server.bidder.BidderCatalog;
import org.prebid.server.model.HttpRequestContext;
import org.prebid.server.proto.openrtb.ext.request.ExtRequest;
import org.prebid.server.proto.openrtb.ext.request.ExtRequestPrebid;
import org.prebid.server.proto.openrtb.ext.request.TraceLevel;
import org.prebid.server.settings.model.Account;
import org.prebid.server.settings.model.AccountAuctionConfig;
import org.prebid.server.util.ObjectUtil;

import java.util.Objects;

public class DebugResolver {

private static final String DEBUG_OVERRIDE_HEADER = "x-pbs-debug-override";
private static final Boolean DEFAULT_DEBUG_ALLOWED_BY_ACCOUNT = true;

private final BidderCatalog bidderCatalog;
private final String debugOverrideToken;

public DebugResolver(BidderCatalog bidderCatalog, String debugOverrideToken) {
this.bidderCatalog = Objects.requireNonNull(bidderCatalog);
this.debugOverrideToken = debugOverrideToken;
}

public DebugContext debugContextFrom(AuctionContext auctionContext) {
final boolean debugEnabled = isDebugEnabled(auctionContext);
final TraceLevel traceLevel = getTraceLevel(auctionContext.getBidRequest());
return DebugContext.of(debugEnabled, traceLevel);
}

private boolean isDebugEnabled(AuctionContext auctionContext) {
final BidRequest bidRequest = auctionContext.getBidRequest();
final boolean debugOverride = isDebugOverridden(auctionContext.getHttpRequest());
final boolean debugEnabledForRequest = isDebugEnabledForRequest(bidRequest);
final boolean debugAllowedByAccount = isDebugAllowedByAccount(auctionContext.getAccount());

if (debugEnabledForRequest && !debugOverride && !debugAllowedByAccount) {
auctionContext.getDebugWarnings()
.add("Debug turned off for account");
}

return debugOverride || (debugEnabledForRequest && debugAllowedByAccount);
}

private boolean isDebugOverridden(HttpRequestContext httpRequest) {
return StringUtils.isNotEmpty(debugOverrideToken)
&& StringUtils.equals(httpRequest.getHeaders().get(DEBUG_OVERRIDE_HEADER), debugOverrideToken);
}

private boolean isDebugEnabledForRequest(BidRequest bidRequest) {
return Objects.equals(bidRequest.getTest(), 1)
|| Objects.equals(ObjectUtil.getIfNotNull(getExtRequestPrebid(bidRequest),
ExtRequestPrebid::getDebug), 1);
}

private boolean isDebugAllowedByAccount(Account account) {
final AccountAuctionConfig auctionConfig = ObjectUtil.getIfNotNull(account, Account::getAuction);
final Boolean debugAllowed = ObjectUtil.getIfNotNull(auctionConfig, AccountAuctionConfig::getDebugAllow);
return ObjectUtils.defaultIfNull(debugAllowed, DEFAULT_DEBUG_ALLOWED_BY_ACCOUNT);
}

private static TraceLevel getTraceLevel(BidRequest bidRequest) {
return ObjectUtil.getIfNotNull(getExtRequestPrebid(bidRequest), ExtRequestPrebid::getTrace);
}

private static ExtRequestPrebid getExtRequestPrebid(BidRequest bidRequest) {
return ObjectUtil.getIfNotNull(
ObjectUtil.getIfNotNull(bidRequest, BidRequest::getExt), ExtRequest::getPrebid);
}

public boolean resolveDebugForBidder(AuctionContext auctionContext, String bidder) {
final DebugContext debugContext = auctionContext.getDebugContext();
final boolean debugEnabled = debugContext.isDebugEnabled();
final boolean debugOverride = isDebugOverridden(auctionContext.getHttpRequest());
final boolean debugAllowedByBidder = bidderCatalog.isDebugAllowed(bidder);

if (debugEnabled && !debugOverride && !debugAllowedByBidder) {
auctionContext.getDebugWarnings()
.add(String.format("Debug turned off for bidder: %s", bidder));
}

return debugOverride || (debugEnabled && debugAllowedByBidder);
}
}
60 changes: 32 additions & 28 deletions src/main/java/org/prebid/server/auction/ExchangeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ public class ExchangeService {
private final PrivacyEnforcementService privacyEnforcementService;
private final FpdResolver fpdResolver;
private final SchainResolver schainResolver;
private final DebugResolver debugResolver;
private final HttpBidderRequester httpBidderRequester;
private final ResponseBidValidator responseBidValidator;
private final CurrencyConversionService currencyService;
Expand All @@ -170,6 +171,7 @@ public ExchangeService(long expectedCacheTime,
PrivacyEnforcementService privacyEnforcementService,
FpdResolver fpdResolver,
SchainResolver schainResolver,
DebugResolver debugResolver,
HttpBidderRequester httpBidderRequester,
ResponseBidValidator responseBidValidator,
CurrencyConversionService currencyService,
Expand All @@ -192,6 +194,7 @@ public ExchangeService(long expectedCacheTime,
this.privacyEnforcementService = Objects.requireNonNull(privacyEnforcementService);
this.fpdResolver = Objects.requireNonNull(fpdResolver);
this.schainResolver = Objects.requireNonNull(schainResolver);
this.debugResolver = Objects.requireNonNull(debugResolver);
this.httpBidderRequester = Objects.requireNonNull(httpBidderRequester);
this.responseBidValidator = Objects.requireNonNull(responseBidValidator);
this.currencyService = Objects.requireNonNull(currencyService);
Expand Down Expand Up @@ -1096,12 +1099,11 @@ private Future<BidderResponse> invokeHooksAndRequestBids(AuctionContext auctionC
Timeout timeout,
BidderAliases aliases) {

final CaseInsensitiveMultiMap headers = auctionContext.getHttpRequest().getHeaders();
final boolean debugEnabled = auctionContext.getDebugContext().isDebugEnabled();

return hookStageExecutor.executeBidderRequestStage(bidderRequest, auctionContext)

.compose(stageResult -> requestBidsOrRejectBidder(
stageResult, bidderRequest, timeout, headers, debugEnabled, aliases))
stageResult, bidderRequest, auctionContext, timeout, aliases))

.compose(bidderResponse -> hookStageExecutor.executeRawBidderResponseStage(
bidderResponse, auctionContext)
.map(stageResult -> rejectBidderResponseOrProceed(stageResult, bidderResponse)));
Expand All @@ -1110,50 +1112,52 @@ private Future<BidderResponse> invokeHooksAndRequestBids(AuctionContext auctionC
private Future<BidderResponse> requestBidsOrRejectBidder(
HookStageExecutionResult<BidderRequestPayload> hookStageResult,
BidderRequest bidderRequest,
AuctionContext auctionContext,
Timeout timeout,
CaseInsensitiveMultiMap requestHeaders,
boolean debugEnabled,
BidderAliases aliases) {

return hookStageResult.isShouldReject()
? Future.succeededFuture(BidderResponse.of(bidderRequest.getBidder(), BidderSeatBid.empty(), 0))
: requestBids(
bidderRequest.with(hookStageResult.getPayload().bidRequest()),
timeout,
requestHeaders,
debugEnabled,
aliases);
}

private BidderResponse rejectBidderResponseOrProceed(HookStageExecutionResult<BidderResponsePayload> stageResult,
BidderResponse bidderResponse) {

final List<BidderBid> bids = stageResult.isShouldReject()
? Collections.emptyList()
: stageResult.getPayload().bids();
if (hookStageResult.isShouldReject()) {
return Future.succeededFuture(BidderResponse.of(bidderRequest.getBidder(), BidderSeatBid.empty(), 0));
}

return bidderResponse
.with(bidderResponse.getSeatBid().with(bids));
final BidderRequest enrichedBidderRequest = bidderRequest.with(hookStageResult.getPayload().bidRequest());
return requestBids(enrichedBidderRequest, auctionContext, timeout, aliases);
}

/**
* Passes the request to a corresponding bidder and wraps response in {@link BidderResponse} which also holds
* recorded response time.
*/
private Future<BidderResponse> requestBids(BidderRequest bidderRequest,
AuctionContext auctionContext,
Timeout timeout,
CaseInsensitiveMultiMap requestHeaders,
boolean debugEnabled,
BidderAliases aliases) {

final CaseInsensitiveMultiMap requestHeaders = auctionContext.getHttpRequest().getHeaders();

final String bidderName = bidderRequest.getBidder();
final Bidder<?> bidder = bidderCatalog.bidderByName(aliases.resolveBidder(bidderName));
final String resolvedBidderName = aliases.resolveBidder(bidderName);
final Bidder<?> bidder = bidderCatalog.bidderByName(resolvedBidderName);

final boolean debugEnabledForBidder = debugResolver.resolveDebugForBidder(auctionContext, resolvedBidderName);

final long startTime = clock.millis();

return httpBidderRequester.requestBids(bidder, bidderRequest, timeout, requestHeaders, debugEnabled)
return httpBidderRequester.requestBids(bidder, bidderRequest, timeout, requestHeaders, debugEnabledForBidder)
.map(seatBid -> BidderResponse.of(bidderName, seatBid, responseTime(startTime)));
}

private BidderResponse rejectBidderResponseOrProceed(HookStageExecutionResult<BidderResponsePayload> stageResult,
BidderResponse bidderResponse) {

final List<BidderBid> bids = stageResult.isShouldReject()
? Collections.emptyList()
: stageResult.getPayload().bids();

return bidderResponse
.with(bidderResponse.getSeatBid().with(bids));
}

private List<BidderResponse> dropZeroNonDealBids(List<BidderResponse> bidderResponses, List<String> debugWarnings) {

return bidderResponses.stream()
Expand Down
6 changes: 6 additions & 0 deletions src/main/java/org/prebid/server/auction/model/AuctionContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,12 @@ public AuctionContext with(MetricName requestTypeMetric) {
.build();
}

public AuctionContext with(DebugContext debugContext) {
return this.toBuilder()
.debugContext(debugContext)
.build();
}

public AuctionContext withRequestRejected() {
return this.toBuilder()
.requestRejected(true)
Expand Down
Loading