Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JSON block that has a full data-center specific URL cache info #1104

Merged
merged 5 commits into from
Nov 21, 2019
Merged

JSON block that has a full data-center specific URL cache info #1104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
87fde55
First draft, test casei needs correction
Nov 6, 2019
07f0510
Stepping out real quick
Nov 11, 2019
3658f79
Implemented corrections
Nov 13, 2019
8419b52
test case corrections
Nov 13, 2019
208c2c1
Cleaning up before pushing PR
Nov 13, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions exchange/auction_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,9 @@ type cacheComparator struct {
func (c *mockCache) GetExtCacheData() (string, string) {
return "", ""
}
func (c *mockCache) GetPutUrl() string {
return ""
}
func (c *mockCache) PutJson(ctx context.Context, values []prebid_cache_client.Cacheable) ([]string, []error) {
c.items = values
return []string{"", "", "", "", ""}, nil
Expand Down
45 changes: 36 additions & 9 deletions exchange/exchange.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,14 +129,15 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque

// If we need to cache bids, then it will take some time to call prebid cache.
// We should reduce the amount of time the bidders have, to compensate.
auctionCtx, cancel := e.makeAuctionContext(ctx, shouldCacheBids)
auctionCtx, cancel := e.makeAuctionContext(ctx, shouldCacheBids) //Why no context for `shouldCacheVast`?
defer cancel()

// Get currency rates conversions for the auction
conversions := e.currencyConverter.Rates()

adapterBids, adapterExtra, anyBidsReturned := e.getAllBids(auctionCtx, cleanRequests, aliases, bidAdjustmentFactors, blabels, conversions)

var auc *auction = nil
if anyBidsReturned {

var bidCategory map[string]string
Expand All @@ -149,7 +150,7 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque
}
}

auc := newAuction(adapterBids, len(bidRequest.Imp))
auc = newAuction(adapterBids, len(bidRequest.Imp))

if targData != nil {
auc.setRoundedPrices(targData.priceGranularity)
Expand All @@ -162,7 +163,7 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque
}

// Build the response
return e.buildBidResponse(ctx, liveAdapters, adapterBids, bidRequest, resolvedRequest, adapterExtra, errs)
return e.buildBidResponse(ctx, liveAdapters, adapterBids, bidRequest, resolvedRequest, adapterExtra, auc, errs)
}

func (e *exchange) makeAuctionContext(ctx context.Context, needsCache bool) (auctionCtx context.Context, cancel context.CancelFunc) {
Expand Down Expand Up @@ -304,7 +305,7 @@ func errsToBidderErrors(errs []error) []openrtb_ext.ExtBidderError {
}

// This piece takes all the bids supplied by the adapters and crafts an openRTB response to send back to the requester
func (e *exchange) buildBidResponse(ctx context.Context, liveAdapters []openrtb_ext.BidderName, adapterBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid, bidRequest *openrtb.BidRequest, resolvedRequest json.RawMessage, adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra, errList []error) (*openrtb.BidResponse, error) {
func (e *exchange) buildBidResponse(ctx context.Context, liveAdapters []openrtb_ext.BidderName, adapterBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid, bidRequest *openrtb.BidRequest, resolvedRequest json.RawMessage, adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra, auc *auction, errList []error) (*openrtb.BidResponse, error) {
bidResponse := new(openrtb.BidResponse)

bidResponse.ID = bidRequest.ID
Expand All @@ -319,7 +320,7 @@ func (e *exchange) buildBidResponse(ctx context.Context, liveAdapters []openrtb_
for _, a := range liveAdapters {
//while processing every single bib, do we need to handle categories here?
if adapterBids[a] != nil && len(adapterBids[a].bids) > 0 {
sb := e.makeSeatBid(adapterBids[a], a, adapterExtra)
sb := e.makeSeatBid(adapterBids[a], a, adapterExtra, auc)
seatBids = append(seatBids, *sb)
}
}
Expand Down Expand Up @@ -531,7 +532,7 @@ func (e *exchange) makeExtBidResponse(adapterBids map[openrtb_ext.BidderName]*pb

// Return an openrtb seatBid for a bidder
// BuildBidResponse is responsible for ensuring nil bid seatbids are not included
func (e *exchange) makeSeatBid(adapterBid *pbsOrtbSeatBid, adapter openrtb_ext.BidderName, adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra) *openrtb.SeatBid {
func (e *exchange) makeSeatBid(adapterBid *pbsOrtbSeatBid, adapter openrtb_ext.BidderName, adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra, auc *auction) *openrtb.SeatBid {
seatBid := new(openrtb.SeatBid)
seatBid.Seat = adapter.String()
// Prebid cannot support roadblocking
Expand All @@ -554,7 +555,7 @@ func (e *exchange) makeSeatBid(adapterBid *pbsOrtbSeatBid, adapter openrtb_ext.B
}

var errList []error
seatBid.Bid, errList = e.makeBid(adapterBid.bids, adapter)
seatBid.Bid, errList = e.makeBid(adapterBid.bids, adapter, auc)
if len(errList) > 0 {
adapterExtra[adapter].Errors = append(adapterExtra[adapter].Errors, errsToBidderErrors(errList)...)
}
Expand All @@ -563,7 +564,7 @@ func (e *exchange) makeSeatBid(adapterBid *pbsOrtbSeatBid, adapter openrtb_ext.B
}

// Create the Bid array inside of SeatBid
func (e *exchange) makeBid(Bids []*pbsOrtbBid, adapter openrtb_ext.BidderName) ([]openrtb.Bid, []error) {
func (e *exchange) makeBid(Bids []*pbsOrtbBid, adapter openrtb_ext.BidderName, auc *auction) ([]openrtb.Bid, []error) {
bids := make([]openrtb.Bid, 0, len(Bids))
errList := make([]error, 0, 1)
for _, thisBid := range Bids {
Expand All @@ -575,7 +576,11 @@ func (e *exchange) makeBid(Bids []*pbsOrtbBid, adapter openrtb_ext.BidderName) (
Video: thisBid.bidVideo,
},
}

if cacheInfo, found := e.getBidCacheInfo(thisBid, auc); found {
bidExt.Prebid.Cache = &openrtb_ext.ExtBidPrebidCache{
Bids: &cacheInfo,
}
}
ext, err := json.Marshal(bidExt)
if err != nil {
errList = append(errList, err)
Expand All @@ -587,6 +592,28 @@ func (e *exchange) makeBid(Bids []*pbsOrtbBid, adapter openrtb_ext.BidderName) (
return bids, errList
}

// If bid got cached inside `(a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, targData *targetData, bidRequest *openrtb.BidRequest, ttlBuffer int64, defaultTTLs *config.DefaultTTLs, bidCategory map[string]string)`,
// a UUID should be found inside `a.cacheIds` or `a.vastCacheIds`. This function returns the UUID along with the internal cache URL
func (e *exchange) getBidCacheInfo(bid *pbsOrtbBid, auc *auction) (openrtb_ext.ExtBidPrebidCacheBids, bool) {
var cacheInfo openrtb_ext.ExtBidPrebidCacheBids
var cacheUUID string
var found bool = false

if auc != nil {
var extCacheHost, extCachePath string
if cacheUUID, found = auc.cacheIds[bid.bid]; found {
cacheInfo.CacheId = cacheUUID
extCacheHost, extCachePath = e.cache.GetExtCacheData()
cacheInfo.Url = extCacheHost + "/" + extCachePath + "?uuid=" + cacheUUID
} else if cacheUUID, found = auc.vastCacheIds[bid.bid]; found {
cacheInfo.CacheId = cacheUUID
extCacheHost, extCachePath = e.cache.GetExtCacheData()
cacheInfo.Url = extCacheHost + "/" + extCachePath + "?uuid=" + cacheUUID
}
}
return cacheInfo, found
}

// Returns a snapshot of resolved bid request for debug if test field is set in the incomming request
func buildResolvedRequest(bidRequest *openrtb.BidRequest) (json.RawMessage, error) {
if bidRequest.Test == 1 {
Expand Down
230 changes: 226 additions & 4 deletions exchange/exchange_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/prebid/prebid-server/openrtb_ext"
"github.com/prebid/prebid-server/pbsmetrics"
metricsConf "github.com/prebid/prebid-server/pbsmetrics/config"
pbc "github.com/prebid/prebid-server/prebid_cache_client"
"github.com/rcrowley/go-metrics"
"github.com/stretchr/testify/assert"
"github.com/yudai/gojsondiff"
Expand Down Expand Up @@ -125,7 +126,7 @@ func TestCharacterEscape(t *testing.T) {
var errList []error

/* 4) Build bid response */
bid_resp, err := e.buildBidResponse(context.Background(), liveAdapters, adapterBids, bidRequest, resolvedRequest, adapterExtra, errList)
bidResp, err := e.buildBidResponse(context.Background(), liveAdapters, adapterBids, bidRequest, resolvedRequest, adapterExtra, nil, errList)

/* 5) Assert we have no errors and one '&' character as we are supposed to */
if err != nil {
Expand All @@ -134,11 +135,232 @@ func TestCharacterEscape(t *testing.T) {
if len(errList) > 0 {
t.Errorf("exchange.buildBidResponse returned %d errors", len(errList))
}
if bytes.Contains(bid_resp.Ext, []byte("u0026")) {
t.Errorf("exchange.buildBidResponse() did not correctly print the '&' characters %s", string(bid_resp.Ext))
if bytes.Contains(bidResp.Ext, []byte("u0026")) {
t.Errorf("exchange.buildBidResponse() did not correctly print the '&' characters %s", string(bidResp.Ext))
}
}

func TestGetBidCacheInfo(t *testing.T) {
testUUID := "CACHE_UUID_1234"
testExternalCacheHost := "https://www.externalprebidcache.net"
testExternalCachePath := "endpoints/cache"

/* 1) An adapter */
bidderName := openrtb_ext.BidderName("appnexus")

cfg := &config.Configuration{
Adapters: map[string]config.Adapter{
string(bidderName): {
Endpoint: "http://ib.adnxs.com/endpoint",
},
},
CacheURL: config.Cache{
Host: "www.internalprebidcache.net",
},
ExtCacheURL: config.ExternalCache{
Host: testExternalCacheHost,
Path: testExternalCachePath,
},
}
adapterList := make([]openrtb_ext.BidderName, 0, 2)
testEngine := metricsConf.NewMetricsEngine(cfg, adapterList)

/* 2) Init new exchange with said configuration */
handlerNoBidServer := func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) }
server := httptest.NewServer(http.HandlerFunc(handlerNoBidServer))
defer server.Close()

e := NewExchange(server.Client(), pbc.NewClient(&cfg.CacheURL, &cfg.ExtCacheURL, testEngine), cfg, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), adapters.ParseBidderInfos(cfg.Adapters, "../static/bidder-info", openrtb_ext.BidderList()), gdpr.AlwaysAllow{}, currencies.NewRateConverterDefault()).(*exchange)

/* 3) Build all the parameters e.buildBidResponse(ctx.Background(), liveA... ) needs */
liveAdapters := []openrtb_ext.BidderName{bidderName}

//adapterBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid,
bids := []*openrtb.Bid{
{
ID: "some-imp-id",
ImpID: "",
Price: 9.517803,
NURL: "",
BURL: "",
LURL: "",
AdM: "",
AdID: "",
ADomain: nil,
Bundle: "",
IURL: "",
CID: "",
CrID: "",
Tactic: "",
Cat: nil,
Attr: nil,
API: 0,
Protocol: 0,
QAGMediaRating: 0,
Language: "",
DealID: "",
W: 300,
H: 250,
WRatio: 0,
HRatio: 0,
Exp: 0,
Ext: nil,
},
}
auc := &auction{
cacheIds: map[*openrtb.Bid]string{
bids[0]: testUUID,
},
}
aPbsOrtbBidArr := []*pbsOrtbBid{
{
bid: bids[0],
bidType: openrtb_ext.BidTypeBanner,
bidTargets: map[string]string{
"pricegranularity": "med",
"includewinners": "true",
"includebidderkeys": "false",
},
},
}
adapterBids := map[openrtb_ext.BidderName]*pbsOrtbSeatBid{
bidderName: {
bids: aPbsOrtbBidArr,
currency: "USD",
},
}

//resolvedRequest json.RawMessage
resolvedRequest := json.RawMessage(`{"id": "some-request-id","site": {"page": "prebid.org"},"imp": [{"id": "some-impression-id","banner": {"format": [{"w": 300,"h": 250},{"w": 300,"h": 600}]},"ext": {"appnexus": {"placementId": 10433394}}}],"tmax": 500}`)

//adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra,
adapterExtra := map[openrtb_ext.BidderName]*seatResponseExtra{
bidderName: {
ResponseTimeMillis: 5,
Errors: []openrtb_ext.ExtBidderError{
{
Code: 999,
Message: "Post ib.adnxs.com/openrtb2?query1&query2: unsupported protocol scheme \"\"",
},
},
},
}
bidRequest := &openrtb.BidRequest{
ID: "some-request-id",
TMax: 1000,
Imp: []openrtb.Imp{
{
ID: "test-div",
Secure: openrtb.Int8Ptr(0),
Banner: &openrtb.Banner{Format: []openrtb.Format{{W: 300, H: 250}}},
Ext: json.RawMessage(` {
"rubicon": {
"accountId": 1001,
"siteId": 113932,
"zoneId": 535510
},
"appnexus": { "placementId": 10433394 },
"pubmatic": { "publisherId": "156209", "adSlot": "pubmatic_test2@300x250" },
"pulsepoint": { "cf": "300X250", "cp": 512379, "ct": 486653 },
"conversant": { "site_id": "108060" },
"ix": { "siteId": "287415" }
}`),
},
},
Site: &openrtb.Site{
Page: "http://rubitest.com/index.html",
Publisher: &openrtb.Publisher{ID: "1001"},
},
Test: 1,
Ext: json.RawMessage(`{"prebid": { "cache": { "bids": {}, "vastxml": {} }, "targeting": { "pricegranularity": "med", "includewinners": true, "includebidderkeys": false } }}`),
}

var errList []error

/* 4) Build bid response */
bid_resp, err := e.buildBidResponse(context.Background(), liveAdapters, adapterBids, bidRequest, resolvedRequest, adapterExtra, auc, errList)

/* 5) Assert we have no errors and the bid response we expected*/
assert.NoError(t, err, "[TestGetBidCacheInfo] buildBidResponse() threw an error")

expectedBidResponse := &openrtb.BidResponse{
SeatBid: []openrtb.SeatBid{
{
Seat: string(bidderName),
Bid: []openrtb.Bid{
{
Ext: json.RawMessage(`{ "prebid": { "cache": { "bids": { "cacheId": "` + testUUID + `", "url": "` + testExternalCacheHost + `/` + testExternalCachePath + `?uuid=` + testUUID + `" }, "key": "", "url": "" }`),
},
},
},
},
}
// compare cache UUID
expCacheUUID, err := jsonparser.GetString(expectedBidResponse.SeatBid[0].Bid[0].Ext, "prebid", "cache", "bids", "cacheId")
assert.NoErrorf(t, err, "[TestGetBidCacheInfo] Error found while trying to json parse the cacheId field from expected build response. Message: %v \n", err)

cacheUUID, err := jsonparser.GetString(bid_resp.SeatBid[0].Bid[0].Ext, "prebid", "cache", "bids", "cacheId")
assert.NoErrorf(t, err, "[TestGetBidCacheInfo] bid_resp.SeatBid[0].Bid[0].Ext = %s \n", bid_resp.SeatBid[0].Bid[0].Ext)

assert.Equal(t, expCacheUUID, cacheUUID, "[TestGetBidCacheInfo] cacheId field in ext should equal \"%s\" \n", expCacheUUID)

// compare cache UUID
expCacheURL, err := jsonparser.GetString(expectedBidResponse.SeatBid[0].Bid[0].Ext, "prebid", "cache", "bids", "url")
assert.NoErrorf(t, err, "[TestGetBidCacheInfo] Error found while trying to json parse the url field from expected build response. Message: %v \n", err)

cacheURL, err := jsonparser.GetString(bid_resp.SeatBid[0].Bid[0].Ext, "prebid", "cache", "bids", "url")
assert.NoErrorf(t, err, "[TestGetBidCacheInfo] Error found while trying to json parse the url field from actual build response. Message: %v \n", err)

assert.Equal(t, expCacheURL, cacheURL, "[TestGetBidCacheInfo] cacheId field in ext should equal \"%s\" \n", expCacheURL)
}

func buildBidResponseParams(bidderName openrtb_ext.BidderName, bidRequest *openrtb.BidRequest) ([]openrtb_ext.BidderName, map[openrtb_ext.BidderName]*pbsOrtbSeatBid, json.RawMessage, map[openrtb_ext.BidderName]*seatResponseExtra) {
//liveAdapters []openrtb_ext.BidderName,
liveAdapters := []openrtb_ext.BidderName{bidderName}

//adapterBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid,
adapterBids := map[openrtb_ext.BidderName]*pbsOrtbSeatBid{
bidderName: {
bids: []*pbsOrtbBid{
{
bid: &openrtb.Bid{
ID: "some-imp-id",
Price: 9.517803,
W: 300,
H: 250,
},
bidType: openrtb_ext.BidTypeBanner,
bidTargets: map[string]string{
"pricegranularity": "med",
"includewinners": "true",
"includebidderkeys": "false",
},
},
},
currency: "USD",
//ext: bidRequest.Ext,
},
}

//resolvedRequest json.RawMessage
resolvedRequest := json.RawMessage(`{"id": "some-request-id","site": {"page": "prebid.org"},"imp": [{"id": "some-impression-id","banner": {"format": [{"w": 300,"h": 250},{"w": 300,"h": 600}]},"ext": {"appnexus": {"placementId": 10433394}}}],"tmax": 500}`)

//adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra,
adapterExtra := map[openrtb_ext.BidderName]*seatResponseExtra{
bidderName: {
ResponseTimeMillis: 5,
Errors: []openrtb_ext.ExtBidderError{
{
Code: 999,
Message: "Post ib.adnxs.com/openrtb2?query1&query2: unsupported protocol scheme \"\"",
},
},
},
}

return liveAdapters, adapterBids, resolvedRequest, adapterExtra
}

// TestRaceIntegration runs an integration test using all the sample params from
// adapters/{bidder}/{bidder}test/params/race/*.json files.
//
Expand Down Expand Up @@ -938,7 +1160,7 @@ func mockHandler(statusCode int, getBody string, postBody string) http.Handler {
type wellBehavedCache struct{}

func (c *wellBehavedCache) GetExtCacheData() (string, string) {
return "www.pbcserver.com", "/pbcache/endpoint"
return "www.pbcserver.com", "pbcache/endpoint"
}

func (c *wellBehavedCache) PutJson(ctx context.Context, values []prebid_cache_client.Cacheable) ([]string, []error) {
Expand Down
Loading