Skip to content

Commit

Permalink
JSON block that has a full data-center specific URL cache info (#1104)
Browse files Browse the repository at this point in the history
  • Loading branch information
@guscarreon @SyntaxNode
guscarreon authored and SyntaxNode committed Nov 21, 2019
1 parent 4eb5d76 commit a7d6b3b
Show file tree
Hide file tree
Showing 13 changed files with 323 additions and 41 deletions.
3 changes: 3 additions & 0 deletions exchange/auction_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,9 @@ type cacheComparator struct {
func (c *mockCache) GetExtCacheData() (string, string) {
return "", ""
}
func (c *mockCache) GetPutUrl() string {
return ""
}
func (c *mockCache) PutJson(ctx context.Context, values []prebid_cache_client.Cacheable) ([]string, []error) {
c.items = values
return []string{"", "", "", "", ""}, nil
Expand Down
45 changes: 36 additions & 9 deletions exchange/exchange.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,14 +129,15 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque

// If we need to cache bids, then it will take some time to call prebid cache.
// We should reduce the amount of time the bidders have, to compensate.
auctionCtx, cancel := e.makeAuctionContext(ctx, shouldCacheBids)
auctionCtx, cancel := e.makeAuctionContext(ctx, shouldCacheBids) //Why no context for `shouldCacheVast`?
defer cancel()

// Get currency rates conversions for the auction
conversions := e.currencyConverter.Rates()

adapterBids, adapterExtra, anyBidsReturned := e.getAllBids(auctionCtx, cleanRequests, aliases, bidAdjustmentFactors, blabels, conversions)

var auc *auction = nil
if anyBidsReturned {

var bidCategory map[string]string
Expand All @@ -149,7 +150,7 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque
}
}

auc := newAuction(adapterBids, len(bidRequest.Imp))
auc = newAuction(adapterBids, len(bidRequest.Imp))

if targData != nil {
auc.setRoundedPrices(targData.priceGranularity)
Expand All @@ -162,7 +163,7 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque
}

// Build the response
return e.buildBidResponse(ctx, liveAdapters, adapterBids, bidRequest, resolvedRequest, adapterExtra, errs)
return e.buildBidResponse(ctx, liveAdapters, adapterBids, bidRequest, resolvedRequest, adapterExtra, auc, errs)
}

func (e *exchange) makeAuctionContext(ctx context.Context, needsCache bool) (auctionCtx context.Context, cancel context.CancelFunc) {
Expand Down Expand Up @@ -304,7 +305,7 @@ func errsToBidderErrors(errs []error) []openrtb_ext.ExtBidderError {
}

// This piece takes all the bids supplied by the adapters and crafts an openRTB response to send back to the requester
func (e *exchange) buildBidResponse(ctx context.Context, liveAdapters []openrtb_ext.BidderName, adapterBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid, bidRequest *openrtb.BidRequest, resolvedRequest json.RawMessage, adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra, errList []error) (*openrtb.BidResponse, error) {
func (e *exchange) buildBidResponse(ctx context.Context, liveAdapters []openrtb_ext.BidderName, adapterBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid, bidRequest *openrtb.BidRequest, resolvedRequest json.RawMessage, adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra, auc *auction, errList []error) (*openrtb.BidResponse, error) {
bidResponse := new(openrtb.BidResponse)

bidResponse.ID = bidRequest.ID
Expand All @@ -319,7 +320,7 @@ func (e *exchange) buildBidResponse(ctx context.Context, liveAdapters []openrtb_
for _, a := range liveAdapters {
//while processing every single bib, do we need to handle categories here?
if adapterBids[a] != nil && len(adapterBids[a].bids) > 0 {
sb := e.makeSeatBid(adapterBids[a], a, adapterExtra)
sb := e.makeSeatBid(adapterBids[a], a, adapterExtra, auc)
seatBids = append(seatBids, *sb)
}
}
Expand Down Expand Up @@ -542,7 +543,7 @@ func (e *exchange) makeExtBidResponse(adapterBids map[openrtb_ext.BidderName]*pb

// Return an openrtb seatBid for a bidder
// BuildBidResponse is responsible for ensuring nil bid seatbids are not included
func (e *exchange) makeSeatBid(adapterBid *pbsOrtbSeatBid, adapter openrtb_ext.BidderName, adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra) *openrtb.SeatBid {
func (e *exchange) makeSeatBid(adapterBid *pbsOrtbSeatBid, adapter openrtb_ext.BidderName, adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra, auc *auction) *openrtb.SeatBid {
seatBid := new(openrtb.SeatBid)
seatBid.Seat = adapter.String()
// Prebid cannot support roadblocking
Expand All @@ -565,7 +566,7 @@ func (e *exchange) makeSeatBid(adapterBid *pbsOrtbSeatBid, adapter openrtb_ext.B
}

var errList []error
seatBid.Bid, errList = e.makeBid(adapterBid.bids, adapter)
seatBid.Bid, errList = e.makeBid(adapterBid.bids, adapter, auc)
if len(errList) > 0 {
adapterExtra[adapter].Errors = append(adapterExtra[adapter].Errors, errsToBidderErrors(errList)...)
}
Expand All @@ -574,7 +575,7 @@ func (e *exchange) makeSeatBid(adapterBid *pbsOrtbSeatBid, adapter openrtb_ext.B
}

// Create the Bid array inside of SeatBid
func (e *exchange) makeBid(Bids []*pbsOrtbBid, adapter openrtb_ext.BidderName) ([]openrtb.Bid, []error) {
func (e *exchange) makeBid(Bids []*pbsOrtbBid, adapter openrtb_ext.BidderName, auc *auction) ([]openrtb.Bid, []error) {
bids := make([]openrtb.Bid, 0, len(Bids))
errList := make([]error, 0, 1)
for _, thisBid := range Bids {
Expand All @@ -586,7 +587,11 @@ func (e *exchange) makeBid(Bids []*pbsOrtbBid, adapter openrtb_ext.BidderName) (
Video: thisBid.bidVideo,
},
}

if cacheInfo, found := e.getBidCacheInfo(thisBid, auc); found {
bidExt.Prebid.Cache = &openrtb_ext.ExtBidPrebidCache{
Bids: &cacheInfo,
}
}
ext, err := json.Marshal(bidExt)
if err != nil {
errList = append(errList, err)
Expand All @@ -598,6 +603,28 @@ func (e *exchange) makeBid(Bids []*pbsOrtbBid, adapter openrtb_ext.BidderName) (
return bids, errList
}

// If bid got cached inside `(a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, targData *targetData, bidRequest *openrtb.BidRequest, ttlBuffer int64, defaultTTLs *config.DefaultTTLs, bidCategory map[string]string)`,
// a UUID should be found inside `a.cacheIds` or `a.vastCacheIds`. This function returns the UUID along with the internal cache URL
func (e *exchange) getBidCacheInfo(bid *pbsOrtbBid, auc *auction) (openrtb_ext.ExtBidPrebidCacheBids, bool) {
var cacheInfo openrtb_ext.ExtBidPrebidCacheBids
var cacheUUID string
var found bool = false

if auc != nil {
var extCacheHost, extCachePath string
if cacheUUID, found = auc.cacheIds[bid.bid]; found {
cacheInfo.CacheId = cacheUUID
extCacheHost, extCachePath = e.cache.GetExtCacheData()
cacheInfo.Url = extCacheHost + "/" + extCachePath + "?uuid=" + cacheUUID
} else if cacheUUID, found = auc.vastCacheIds[bid.bid]; found {
cacheInfo.CacheId = cacheUUID
extCacheHost, extCachePath = e.cache.GetExtCacheData()
cacheInfo.Url = extCacheHost + "/" + extCachePath + "?uuid=" + cacheUUID
}
}
return cacheInfo, found
}

// Returns a snapshot of resolved bid request for debug if test field is set in the incomming request
func buildResolvedRequest(bidRequest *openrtb.BidRequest) (json.RawMessage, error) {
if bidRequest.Test == 1 {
Expand Down
230 changes: 226 additions & 4 deletions exchange/exchange_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/prebid/prebid-server/openrtb_ext"
"github.com/prebid/prebid-server/pbsmetrics"
metricsConf "github.com/prebid/prebid-server/pbsmetrics/config"
pbc "github.com/prebid/prebid-server/prebid_cache_client"
"github.com/rcrowley/go-metrics"
"github.com/stretchr/testify/assert"
"github.com/yudai/gojsondiff"
Expand Down Expand Up @@ -125,7 +126,7 @@ func TestCharacterEscape(t *testing.T) {
var errList []error

/* 4) Build bid response */
bid_resp, err := e.buildBidResponse(context.Background(), liveAdapters, adapterBids, bidRequest, resolvedRequest, adapterExtra, errList)
bidResp, err := e.buildBidResponse(context.Background(), liveAdapters, adapterBids, bidRequest, resolvedRequest, adapterExtra, nil, errList)

/* 5) Assert we have no errors and one '&' character as we are supposed to */
if err != nil {
Expand All @@ -134,11 +135,232 @@ func TestCharacterEscape(t *testing.T) {
if len(errList) > 0 {
t.Errorf("exchange.buildBidResponse returned %d errors", len(errList))
}
if bytes.Contains(bid_resp.Ext, []byte("u0026")) {
t.Errorf("exchange.buildBidResponse() did not correctly print the '&' characters %s", string(bid_resp.Ext))
if bytes.Contains(bidResp.Ext, []byte("u0026")) {
t.Errorf("exchange.buildBidResponse() did not correctly print the '&' characters %s", string(bidResp.Ext))
}
}

func TestGetBidCacheInfo(t *testing.T) {
testUUID := "CACHE_UUID_1234"
testExternalCacheHost := "https://www.externalprebidcache.net"
testExternalCachePath := "endpoints/cache"

/* 1) An adapter */
bidderName := openrtb_ext.BidderName("appnexus")

cfg := &config.Configuration{
Adapters: map[string]config.Adapter{
string(bidderName): {
Endpoint: "http://ib.adnxs.com/endpoint",
},
},
CacheURL: config.Cache{
Host: "www.internalprebidcache.net",
},
ExtCacheURL: config.ExternalCache{
Host: testExternalCacheHost,
Path: testExternalCachePath,
},
}
adapterList := make([]openrtb_ext.BidderName, 0, 2)
testEngine := metricsConf.NewMetricsEngine(cfg, adapterList)

/* 2) Init new exchange with said configuration */
handlerNoBidServer := func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) }
server := httptest.NewServer(http.HandlerFunc(handlerNoBidServer))
defer server.Close()

e := NewExchange(server.Client(), pbc.NewClient(&cfg.CacheURL, &cfg.ExtCacheURL, testEngine), cfg, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), adapters.ParseBidderInfos(cfg.Adapters, "../static/bidder-info", openrtb_ext.BidderList()), gdpr.AlwaysAllow{}, currencies.NewRateConverterDefault()).(*exchange)

/* 3) Build all the parameters e.buildBidResponse(ctx.Background(), liveA... ) needs */
liveAdapters := []openrtb_ext.BidderName{bidderName}

//adapterBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid,
bids := []*openrtb.Bid{
{
ID: "some-imp-id",
ImpID: "",
Price: 9.517803,
NURL: "",
BURL: "",
LURL: "",
AdM: "",
AdID: "",
ADomain: nil,
Bundle: "",
IURL: "",
CID: "",
CrID: "",
Tactic: "",
Cat: nil,
Attr: nil,
API: 0,
Protocol: 0,
QAGMediaRating: 0,
Language: "",
DealID: "",
W: 300,
H: 250,
WRatio: 0,
HRatio: 0,
Exp: 0,
Ext: nil,
},
}
auc := &auction{
cacheIds: map[*openrtb.Bid]string{
bids[0]: testUUID,
},
}
aPbsOrtbBidArr := []*pbsOrtbBid{
{
bid: bids[0],
bidType: openrtb_ext.BidTypeBanner,
bidTargets: map[string]string{
"pricegranularity": "med",
"includewinners": "true",
"includebidderkeys": "false",
},
},
}
adapterBids := map[openrtb_ext.BidderName]*pbsOrtbSeatBid{
bidderName: {
bids: aPbsOrtbBidArr,
currency: "USD",
},
}

//resolvedRequest json.RawMessage
resolvedRequest := json.RawMessage(`{"id": "some-request-id","site": {"page": "prebid.org"},"imp": [{"id": "some-impression-id","banner": {"format": [{"w": 300,"h": 250},{"w": 300,"h": 600}]},"ext": {"appnexus": {"placementId": 10433394}}}],"tmax": 500}`)

//adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra,
adapterExtra := map[openrtb_ext.BidderName]*seatResponseExtra{
bidderName: {
ResponseTimeMillis: 5,
Errors: []openrtb_ext.ExtBidderError{
{
Code: 999,
Message: "Post ib.adnxs.com/openrtb2?query1&query2: unsupported protocol scheme \"\"",
},
},
},
}
bidRequest := &openrtb.BidRequest{
ID: "some-request-id",
TMax: 1000,
Imp: []openrtb.Imp{
{
ID: "test-div",
Secure: openrtb.Int8Ptr(0),
Banner: &openrtb.Banner{Format: []openrtb.Format{{W: 300, H: 250}}},
Ext: json.RawMessage(` {
"rubicon": {
"accountId": 1001,
"siteId": 113932,
"zoneId": 535510
},
"appnexus": { "placementId": 10433394 },
"pubmatic": { "publisherId": "156209", "adSlot": "pubmatic_test2@300x250" },
"pulsepoint": { "cf": "300X250", "cp": 512379, "ct": 486653 },
"conversant": { "site_id": "108060" },
"ix": { "siteId": "287415" }
}`),
},
},
Site: &openrtb.Site{
Page: "http://rubitest.com/index.html",
Publisher: &openrtb.Publisher{ID: "1001"},
},
Test: 1,
Ext: json.RawMessage(`{"prebid": { "cache": { "bids": {}, "vastxml": {} }, "targeting": { "pricegranularity": "med", "includewinners": true, "includebidderkeys": false } }}`),
}

var errList []error

/* 4) Build bid response */
bid_resp, err := e.buildBidResponse(context.Background(), liveAdapters, adapterBids, bidRequest, resolvedRequest, adapterExtra, auc, errList)

/* 5) Assert we have no errors and the bid response we expected*/
assert.NoError(t, err, "[TestGetBidCacheInfo] buildBidResponse() threw an error")

expectedBidResponse := &openrtb.BidResponse{
SeatBid: []openrtb.SeatBid{
{
Seat: string(bidderName),
Bid: []openrtb.Bid{
{
Ext: json.RawMessage(`{ "prebid": { "cache": { "bids": { "cacheId": "` + testUUID + `", "url": "` + testExternalCacheHost + `/` + testExternalCachePath + `?uuid=` + testUUID + `" }, "key": "", "url": "" }`),
},
},
},
},
}
// compare cache UUID
expCacheUUID, err := jsonparser.GetString(expectedBidResponse.SeatBid[0].Bid[0].Ext, "prebid", "cache", "bids", "cacheId")
assert.NoErrorf(t, err, "[TestGetBidCacheInfo] Error found while trying to json parse the cacheId field from expected build response. Message: %v \n", err)

cacheUUID, err := jsonparser.GetString(bid_resp.SeatBid[0].Bid[0].Ext, "prebid", "cache", "bids", "cacheId")
assert.NoErrorf(t, err, "[TestGetBidCacheInfo] bid_resp.SeatBid[0].Bid[0].Ext = %s \n", bid_resp.SeatBid[0].Bid[0].Ext)

assert.Equal(t, expCacheUUID, cacheUUID, "[TestGetBidCacheInfo] cacheId field in ext should equal \"%s\" \n", expCacheUUID)

// compare cache UUID
expCacheURL, err := jsonparser.GetString(expectedBidResponse.SeatBid[0].Bid[0].Ext, "prebid", "cache", "bids", "url")
assert.NoErrorf(t, err, "[TestGetBidCacheInfo] Error found while trying to json parse the url field from expected build response. Message: %v \n", err)

cacheURL, err := jsonparser.GetString(bid_resp.SeatBid[0].Bid[0].Ext, "prebid", "cache", "bids", "url")
assert.NoErrorf(t, err, "[TestGetBidCacheInfo] Error found while trying to json parse the url field from actual build response. Message: %v \n", err)

assert.Equal(t, expCacheURL, cacheURL, "[TestGetBidCacheInfo] cacheId field in ext should equal \"%s\" \n", expCacheURL)
}

func buildBidResponseParams(bidderName openrtb_ext.BidderName, bidRequest *openrtb.BidRequest) ([]openrtb_ext.BidderName, map[openrtb_ext.BidderName]*pbsOrtbSeatBid, json.RawMessage, map[openrtb_ext.BidderName]*seatResponseExtra) {
//liveAdapters []openrtb_ext.BidderName,
liveAdapters := []openrtb_ext.BidderName{bidderName}

//adapterBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid,
adapterBids := map[openrtb_ext.BidderName]*pbsOrtbSeatBid{
bidderName: {
bids: []*pbsOrtbBid{
{
bid: &openrtb.Bid{
ID: "some-imp-id",
Price: 9.517803,
W: 300,
H: 250,
},
bidType: openrtb_ext.BidTypeBanner,
bidTargets: map[string]string{
"pricegranularity": "med",
"includewinners": "true",
"includebidderkeys": "false",
},
},
},
currency: "USD",
//ext: bidRequest.Ext,
},
}

//resolvedRequest json.RawMessage
resolvedRequest := json.RawMessage(`{"id": "some-request-id","site": {"page": "prebid.org"},"imp": [{"id": "some-impression-id","banner": {"format": [{"w": 300,"h": 250},{"w": 300,"h": 600}]},"ext": {"appnexus": {"placementId": 10433394}}}],"tmax": 500}`)

//adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra,
adapterExtra := map[openrtb_ext.BidderName]*seatResponseExtra{
bidderName: {
ResponseTimeMillis: 5,
Errors: []openrtb_ext.ExtBidderError{
{
Code: 999,
Message: "Post ib.adnxs.com/openrtb2?query1&query2: unsupported protocol scheme \"\"",
},
},
},
}

return liveAdapters, adapterBids, resolvedRequest, adapterExtra
}

// TestRaceIntegration runs an integration test using all the sample params from
// adapters/{bidder}/{bidder}test/params/race/*.json files.
//
Expand Down Expand Up @@ -1069,7 +1291,7 @@ func mockHandler(statusCode int, getBody string, postBody string) http.Handler {
type wellBehavedCache struct{}

func (c *wellBehavedCache) GetExtCacheData() (string, string) {
return "www.pbcserver.com", "/pbcache/endpoint"
return "www.pbcserver.com", "pbcache/endpoint"
}

func (c *wellBehavedCache) PutJson(ctx context.Context, values []prebid_cache_client.Cacheable) ([]string, []error) {
Expand Down
Loading

0 comments on commit a7d6b3b

Please sign in to comment.