From 655585c8d73026ffaa2ff0a1916aff4230a27ed8 Mon Sep 17 00:00:00 2001 From: Robert Ray Martinez III Date: Tue, 15 Mar 2022 07:20:18 -0700 Subject: [PATCH] Rubicon Analytics Adapter: pass along billing events (#8182) * rubicon listens to billing events * billingId is required * remove log --- modules/rubiconAnalyticsAdapter.js | 155 ++++++++--- .../modules/rubiconAnalyticsAdapter_spec.js | 259 +++++++++++++++++- test/spec/modules/rubiconAnalyticsSchema.json | 46 +++- 3 files changed, 422 insertions(+), 38 deletions(-) diff --git a/modules/rubiconAnalyticsAdapter.js b/modules/rubiconAnalyticsAdapter.js index 7bb84a39f73..41df83deb1d 100644 --- a/modules/rubiconAnalyticsAdapter.js +++ b/modules/rubiconAnalyticsAdapter.js @@ -1,4 +1,4 @@ -import { generateUUID, mergeDeep, deepAccess, parseUrl, logError, pick, isEmpty, logWarn, debugTurnedOn, parseQS, getWindowLocation, isAdUnitCodeMatchingSlot, isNumber, isGptPubadsDefined, _each, deepSetValue } from '../src/utils.js'; +import { generateUUID, mergeDeep, deepAccess, parseUrl, logError, pick, isEmpty, logWarn, debugTurnedOn, parseQS, getWindowLocation, isAdUnitCodeMatchingSlot, isNumber, isGptPubadsDefined, _each, deepSetValue, deepClone, logInfo } from '../src/utils.js'; import adapter from '../src/AnalyticsAdapter.js'; import adapterManager from '../src/adapterManager.js'; import CONSTANTS from '../src/constants.json'; @@ -12,6 +12,7 @@ export const storage = getStorageManager({gvlid: RUBICON_GVL_ID, moduleName: 'ru const COOKIE_NAME = 'rpaSession'; const LAST_SEEN_EXPIRE_TIME = 1800000; // 30 mins const END_EXPIRE_TIME = 21600000; // 6 hours +const MODULE_NAME = 'Rubicon Analytics'; const pbsErrorMap = { 1: 'timeout-error', @@ -31,7 +32,8 @@ const { BIDDER_DONE, BID_TIMEOUT, BID_WON, - SET_TARGETING + SET_TARGETING, + BILLABLE_EVENT }, STATUS: { GOOD, @@ -55,13 +57,19 @@ const cache = { targeting: {}, timeouts: {}, gpt: {}, + billing: {} }; const BID_REJECTED_IPF = 'rejected-ipf'; export let rubiConf = { pvid: generateUUID().slice(0, 8), - analyticsEventDelay: 0 + analyticsEventDelay: 0, + dmBilling: { + enabled: false, + vendors: [], + waitForAuction: true + } }; // we are saving these as global to this module so that if a pub accidentally overwrites the entire // rubicon object, then we do not lose other data @@ -76,7 +84,7 @@ export function getHostNameFromReferer(referer) { try { rubiconAdapter.referrerHostname = parseUrl(referer, { noDecodeWholeURL: true }).hostname; } catch (e) { - logError('Rubicon Analytics: Unable to parse hostname from supplied url: ', referer, e); + logError(`${MODULE_NAME}: Unable to parse hostname from supplied url: `, referer, e); rubiconAdapter.referrerHostname = ''; } return rubiconAdapter.referrerHostname @@ -115,6 +123,55 @@ function formatSource(src) { return src.toLowerCase(); } +function getBillingPayload(event) { + // for now we are mapping all events to type "general", later we will expand support for specific types + let billingEvent = deepClone(event); + billingEvent.type = 'general'; + billingEvent.accountId = accountId; + // mark as sent + deepSetValue(cache.billing, `${event.vendor}.${event.billingId}`, true); + return billingEvent; +} + +function sendBillingEvent(event) { + let message = getBasicEventDetails(undefined, 'soloBilling'); + message.billableEvents = [getBillingPayload(event)]; + ajax( + rubiconAdapter.getUrl(), + null, + JSON.stringify(message), + { + contentType: 'application/json' + } + ); +} + +function getBasicEventDetails(auctionId, trigger) { + let auctionCache = cache.auctions[auctionId]; + let referrer = config.getConfig('pageUrl') || pageReferer || (auctionCache && auctionCache.referrer); + let message = { + timestamps: { + prebidLoaded: rubiconAdapter.MODULE_INITIALIZED_TIME, + auctionEnded: auctionCache ? auctionCache.endTs : undefined, + eventTime: Date.now() + }, + trigger, + integration: rubiConf.int_type || DEFAULT_INTEGRATION, + version: '$prebid.version$', + referrerUri: referrer, + referrerHostname: rubiconAdapter.referrerHostname || getHostNameFromReferer(referrer), + channel: 'web', + }; + if (rubiConf.wrapperName) { + message.wrapper = { + name: rubiConf.wrapperName, + family: rubiConf.wrapperFamily, + rule: rubiConf.rule_name + } + } + return message; +} + function sendMessage(auctionId, bidWonId, trigger) { function formatBid(bid) { return pick(bid, [ @@ -160,28 +217,8 @@ function sendMessage(auctionId, bidWonId, trigger) { samplingFactor }); } + let message = getBasicEventDetails(auctionId, trigger); let auctionCache = cache.auctions[auctionId]; - let referrer = config.getConfig('pageUrl') || (auctionCache && auctionCache.referrer); - let message = { - timestamps: { - prebidLoaded: rubiconAdapter.MODULE_INITIALIZED_TIME, - auctionEnded: auctionCache.endTs, - eventTime: Date.now() - }, - trigger, - integration: rubiConf.int_type || DEFAULT_INTEGRATION, - version: '$prebid.version$', - referrerUri: referrer, - referrerHostname: rubiconAdapter.referrerHostname || getHostNameFromReferer(referrer), - channel: 'web', - }; - if (rubiConf.wrapperName) { - message.wrapper = { - name: rubiConf.wrapperName, - family: rubiConf.wrapperFamily, - rule: rubiConf.rule_name - } - } if (auctionCache && !auctionCache.sent) { let adUnitMap = Object.keys(auctionCache.bids).reduce((adUnits, bidId) => { let bid = auctionCache.bids[bidId]; @@ -321,6 +358,12 @@ function sendMessage(auctionId, bidWonId, trigger) { ]; } + // if we have not sent any billingEvents send them + const pendingBillingEvents = getPendingBillingEvents(auctionCache); + if (pendingBillingEvents && pendingBillingEvents.length) { + message.billableEvents = pendingBillingEvents; + } + ajax( this.getUrl(), null, @@ -331,6 +374,17 @@ function sendMessage(auctionId, bidWonId, trigger) { ); } +function getPendingBillingEvents(auctionCache) { + if (auctionCache && auctionCache.billing && auctionCache.billing.length) { + return auctionCache.billing.reduce((accum, billingEvent) => { + if (deepAccess(cache.billing, `${billingEvent.vendor}.${billingEvent.billingId}`) === false) { + accum.push(getBillingPayload(billingEvent)); + } + return accum; + }, []); + } +} + function adUnitIsOnlyInstream(adUnit) { return adUnit.mediaTypes && Object.keys(adUnit.mediaTypes).length === 1 && deepAccess(adUnit, 'mediaTypes.video.context') === 'instream'; } @@ -359,7 +413,7 @@ function getBidPrice(bid) { try { return Number(prebidGlobal.convertCurrency(cpm, currency, 'USD')); } catch (err) { - logWarn('Rubicon Analytics Adapter: Could not determine the bidPriceUSD of the bid ', bid); + logWarn(`${MODULE_NAME}: Could not determine the bidPriceUSD of the bid `, bid); } } @@ -449,7 +503,7 @@ function getRpaCookie() { try { return JSON.parse(window.atob(encodedCookie)); } catch (e) { - logError(`Rubicon Analytics: Unable to decode ${COOKIE_NAME} value: `, e); + logError(`${MODULE_NAME}: Unable to decode ${COOKIE_NAME} value: `, e); } } return {}; @@ -459,7 +513,7 @@ function setRpaCookie(decodedCookie) { try { storage.setDataInLocalStorage(COOKIE_NAME, window.btoa(JSON.stringify(decodedCookie))); } catch (e) { - logError(`Rubicon Analytics: Unable to encode ${COOKIE_NAME} value: `, e); + logError(`${MODULE_NAME}: Unable to encode ${COOKIE_NAME} value: `, e); } } @@ -524,6 +578,30 @@ function subscribeToGamSlots() { }); } +let pageReferer; + +const isBillingEventValid = event => { + // vendor is whitelisted + const isWhitelistedVendor = rubiConf.dmBilling.vendors.includes(event.vendor); + // event is not duplicated + const isNotDuplicate = typeof deepAccess(cache.billing, `${event.vendor}.${event.billingId}`) !== 'boolean'; + // billingId is defined and a string + return typeof event.billingId === 'string' && isWhitelistedVendor && isNotDuplicate; +} + +const sendOrAddEventToQueue = event => { + // if any auction is not sent yet, then add it to the auction queue + const pendingAuction = Object.keys(cache.auctions).find(auctionId => !cache.auctions[auctionId].sent); + + if (rubiConf.dmBilling.waitForAuction && pendingAuction) { + cache.auctions[pendingAuction].billing = cache.auctions[pendingAuction].billing || []; + cache.auctions[pendingAuction].billing.push(event); + } else { + // send it + sendBillingEvent(event); + } +} + let baseAdapter = adapter({ analyticsType: 'endpoint' }); let rubiconAdapter = Object.assign({}, baseAdapter, { MODULE_INITIALIZED_TIME: Date.now(), @@ -539,7 +617,7 @@ let rubiconAdapter = Object.assign({}, baseAdapter, { if (config.options.endpoint) { this.getUrl = () => config.options.endpoint; } else { - logError('required endpoint missing from rubicon analytics'); + logError(`${MODULE_NAME}: required endpoint missing`); error = true; } if (typeof config.options.sampling !== 'undefined') { @@ -547,7 +625,7 @@ let rubiconAdapter = Object.assign({}, baseAdapter, { } if (typeof config.options.samplingFactor !== 'undefined') { if (typeof config.options.sampling !== 'undefined') { - logWarn('Both options.samplingFactor and options.sampling enabled in rubicon analytics, defaulting to samplingFactor'); + logWarn(`${MODULE_NAME}: Both options.samplingFactor and options.sampling enabled defaulting to samplingFactor`); } samplingFactor = parseFloat(config.options.samplingFactor); config.options.sampling = 1 / samplingFactor; @@ -557,10 +635,10 @@ let rubiconAdapter = Object.assign({}, baseAdapter, { let validSamplingFactors = [1, 10, 20, 40, 100]; if (validSamplingFactors.indexOf(samplingFactor) === -1) { error = true; - logError('invalid samplingFactor for rubicon analytics: ' + samplingFactor + ', must be one of ' + validSamplingFactors.join(', ')); + logError(`${MODULE_NAME}: invalid samplingFactor ${samplingFactor} - must be one of ${validSamplingFactors.join(', ')}`); } else if (!accountId) { error = true; - logError('required accountId missing for rubicon analytics'); + logError(`${MODULE_NAME}: required accountId missing for rubicon analytics`); } if (!error) { @@ -572,6 +650,7 @@ let rubiconAdapter = Object.assign({}, baseAdapter, { accountId = undefined; rubiConf = {}; cache.gpt.registered = false; + cache.billing = {}; baseAdapter.disableAnalytics.apply(this, arguments); }, track({ eventType, args }) { @@ -586,8 +665,8 @@ let rubiconAdapter = Object.assign({}, baseAdapter, { cacheEntry.bids = {}; cacheEntry.bidsWon = {}; cacheEntry.gamHasRendered = {}; + cacheEntry.referrer = pageReferer = deepAccess(args, 'bidderRequests.0.refererInfo.referer'); cacheEntry.bidderOrder = []; - cacheEntry.referrer = deepAccess(args, 'bidderRequests.0.refererInfo.referer'); const floorData = deepAccess(args, 'bidderRequests.0.bids.0.floorData'); if (floorData) { cacheEntry.floorData = { ...floorData }; @@ -716,7 +795,7 @@ let rubiconAdapter = Object.assign({}, baseAdapter, { auctionEntry.floorData.enforcements = { ...args.floorData.enforcements }; } if (!bid) { - logError('Rubicon Anlytics Adapter Error: Could not find associated bid request for bid response with requestId: ', args.requestId); + logError(`${MODULE_NAME}: Could not find associated bid request for bid response with requestId: `, args.requestId); break; } bid.source = formatSource(bid.source || args.source); @@ -814,6 +893,14 @@ let rubiconAdapter = Object.assign({}, baseAdapter, { } }); break; + case BILLABLE_EVENT: + if (rubiConf.dmBilling.enabled && isBillingEventValid(args)) { + // add to the map indicating it has not been sent yet + deepSetValue(cache.billing, `${args.vendor}.${args.billingId}`, false); + sendOrAddEventToQueue(args); + } else { + logInfo(`${MODULE_NAME}: Billing event ignored`, args); + } } } }); diff --git a/test/spec/modules/rubiconAnalyticsAdapter_spec.js b/test/spec/modules/rubiconAnalyticsAdapter_spec.js index 584fe718c36..262276039b0 100644 --- a/test/spec/modules/rubiconAnalyticsAdapter_spec.js +++ b/test/spec/modules/rubiconAnalyticsAdapter_spec.js @@ -13,6 +13,7 @@ import { setConfig, addBidResponseHook, } from 'modules/currency.js'; +import truncate from 'lodash.truncate'; let Ajv = require('ajv'); let schema = require('./rubiconAnalyticsSchema.json'); let ajv = new Ajv({ @@ -39,7 +40,8 @@ const { BIDDER_DONE, BID_WON, BID_TIMEOUT, - SET_TARGETING + SET_TARGETING, + BILLABLE_EVENT } } = CONSTANTS; @@ -677,6 +679,11 @@ describe('rubicon analytics adapter', function () { }); expect(rubiConf).to.deep.equal({ analyticsEventDelay: 0, + dmBilling: { + enabled: false, + vendors: [], + waitForAuction: true + }, pvid: '12345678', wrapperName: '1001_general', int_type: 'dmpbjs', @@ -696,6 +703,11 @@ describe('rubicon analytics adapter', function () { }); expect(rubiConf).to.deep.equal({ analyticsEventDelay: 0, + dmBilling: { + enabled: false, + vendors: [], + waitForAuction: true + }, pvid: '12345678', wrapperName: '1001_general', int_type: 'dmpbjs', @@ -717,6 +729,11 @@ describe('rubicon analytics adapter', function () { }); expect(rubiConf).to.deep.equal({ analyticsEventDelay: 0, + dmBilling: { + enabled: false, + vendors: [], + waitForAuction: true + }, pvid: '12345678', wrapperName: '1001_general', int_type: 'dmpbjs', @@ -2163,6 +2180,246 @@ describe('rubicon analytics adapter', function () { }); }); + describe('billing events integration', () => { + beforeEach(function () { + rubiconAnalyticsAdapter.enableAnalytics({ + options: { + endpoint: '//localhost:9999/event', + accountId: 1001 + } + }); + // default dmBilling + config.setConfig({ + rubicon: { + dmBilling: { + enabled: false, + vendors: [], + waitForAuction: true + } + } + }) + }); + afterEach(function () { + rubiconAnalyticsAdapter.disableAnalytics(); + }); + const basicBillingAuction = (billingEvents = []) => { + events.emit(AUCTION_INIT, MOCK.AUCTION_INIT); + events.emit(BID_REQUESTED, MOCK.BID_REQUESTED); + + // emit billing events + billingEvents.forEach(ev => events.emit(BILLABLE_EVENT, ev)); + events.emit(BID_RESPONSE, MOCK.BID_RESPONSE[0]); + events.emit(BID_RESPONSE, MOCK.BID_RESPONSE[1]); + events.emit(BIDDER_DONE, MOCK.BIDDER_DONE); + events.emit(AUCTION_END, MOCK.AUCTION_END); + events.emit(SET_TARGETING, MOCK.SET_TARGETING); + events.emit(BID_WON, MOCK.BID_WON[0]); + events.emit(BID_WON, MOCK.BID_WON[1]); + } + it('should ignore billing events when not enabled', () => { + basicBillingAuction([{ + vendor: 'vendorName', + type: 'auction', + billingId: 'f8558d41-62de-4349-bc7b-2dbee1e69965' + }]); + expect(server.requests.length).to.equal(1); + const request = server.requests[0]; + const message = JSON.parse(request.requestBody); + expect(message.billableEvents).to.be.undefined; + }); + it('should ignore billing events when enabled but vendor is not whitelisted', () => { + // off by default + config.setConfig({ + rubicon: { + dmBilling: { + enabled: true + } + } + }); + basicBillingAuction([{ + vendor: 'vendorName', + type: 'auction', + billingId: 'f8558d41-62de-4349-bc7b-2dbee1e69965' + }]); + expect(server.requests.length).to.equal(1); + const request = server.requests[0]; + const message = JSON.parse(request.requestBody); + expect(message.billableEvents).to.be.undefined; + }); + it('should ignore billing events if billingId is not defined or billingId is not a string', () => { + // off by default + config.setConfig({ + rubicon: { + dmBilling: { + enabled: true, + vendors: ['vendorName'] + } + } + }); + basicBillingAuction([ + { + vendor: 'vendorName', + type: 'auction', + }, + { + vendor: 'vendorName', + type: 'auction', + billingId: true + }, + { + vendor: 'vendorName', + type: 'auction', + billingId: 1233434 + }, + { + vendor: 'vendorName', + type: 'auction', + billingId: null + } + ]); + expect(server.requests.length).to.equal(1); + const request = server.requests[0]; + const message = JSON.parse(request.requestBody); + expect(message.billableEvents).to.be.undefined; + }); + it('should pass along billing event in same payload', () => { + // off by default + config.setConfig({ + rubicon: { + dmBilling: { + enabled: true, + vendors: ['vendorName'] + } + } + }); + basicBillingAuction([{ + vendor: 'vendorName', + type: 'auction', + billingId: 'f8558d41-62de-4349-bc7b-2dbee1e69965' + }]); + expect(server.requests.length).to.equal(1); + const request = server.requests[0]; + const message = JSON.parse(request.requestBody); + expect(message).to.haveOwnProperty('auctions'); + expect(message.billableEvents).to.deep.equal([{ + accountId: 1001, + vendor: 'vendorName', + type: 'general', // mapping all events to endpoint as 'general' for now + billingId: 'f8558d41-62de-4349-bc7b-2dbee1e69965' + }]); + }); + it('should pass along multiple billing events but filter out duplicates', () => { + // off by default + config.setConfig({ + rubicon: { + dmBilling: { + enabled: true, + vendors: ['vendorName'] + } + } + }); + basicBillingAuction([ + { + vendor: 'vendorName', + type: 'auction', + billingId: 'f8558d41-62de-4349-bc7b-2dbee1e69965' + }, + { + vendor: 'vendorName', + type: 'auction', + billingId: '743db6e3-21f2-44d4-917f-cb3488c6076f' + }, + { + vendor: 'vendorName', + type: 'auction', + billingId: 'f8558d41-62de-4349-bc7b-2dbee1e69965' + } + ]); + expect(server.requests.length).to.equal(1); + const request = server.requests[0]; + const message = JSON.parse(request.requestBody); + expect(message).to.haveOwnProperty('auctions'); + expect(message.billableEvents).to.deep.equal([ + { + accountId: 1001, + vendor: 'vendorName', + type: 'general', + billingId: 'f8558d41-62de-4349-bc7b-2dbee1e69965' + }, + { + accountId: 1001, + vendor: 'vendorName', + type: 'general', + billingId: '743db6e3-21f2-44d4-917f-cb3488c6076f' + } + ]); + }); + it('should pass along event right away if no pending auction', () => { + // off by default + config.setConfig({ + rubicon: { + dmBilling: { + enabled: true, + vendors: ['vendorName'] + } + } + }); + + events.emit(BILLABLE_EVENT, { + vendor: 'vendorName', + type: 'auction', + billingId: 'f8558d41-62de-4349-bc7b-2dbee1e69965' + }); + expect(server.requests.length).to.equal(1); + const request = server.requests[0]; + const message = JSON.parse(request.requestBody); + expect(message).to.not.haveOwnProperty('auctions'); + expect(message.billableEvents).to.deep.equal([ + { + accountId: 1001, + vendor: 'vendorName', + type: 'general', + billingId: 'f8558d41-62de-4349-bc7b-2dbee1e69965' + } + ]); + }); + it('should pass along event right away if pending auction but not waiting', () => { + // off by default + config.setConfig({ + rubicon: { + dmBilling: { + enabled: true, + vendors: ['vendorName'], + waitForAuction: false + } + } + }); + // should fire right away, and then auction later + basicBillingAuction([{ + vendor: 'vendorName', + type: 'auction', + billingId: 'f8558d41-62de-4349-bc7b-2dbee1e69965' + }]); + expect(server.requests.length).to.equal(2); + const billingRequest = server.requests[0]; + const billingMessage = JSON.parse(billingRequest.requestBody); + expect(billingMessage).to.not.haveOwnProperty('auctions'); + expect(billingMessage.billableEvents).to.deep.equal([ + { + accountId: 1001, + vendor: 'vendorName', + type: 'general', + billingId: 'f8558d41-62de-4349-bc7b-2dbee1e69965' + } + ]); + // auction event after + const auctionRequest = server.requests[1]; + const auctionMessage = JSON.parse(auctionRequest.requestBody); + // should not double pass events! + expect(auctionMessage).to.not.haveOwnProperty('billableEvents'); + }); + }); + describe('wrapper details passed in', () => { it('should correctly pass in the wrapper details if provided', () => { config.setConfig({ diff --git a/test/spec/modules/rubiconAnalyticsSchema.json b/test/spec/modules/rubiconAnalyticsSchema.json index 0fa666d087c..07239ca06ed 100644 --- a/test/spec/modules/rubiconAnalyticsSchema.json +++ b/test/spec/modules/rubiconAnalyticsSchema.json @@ -17,6 +17,11 @@ "required": [ "bidsWon" ] + }, + { + "required": [ + "billableEvents" + ] } ], "properties": { @@ -237,9 +242,44 @@ ] } }, - "wrapperName": { - "type": "string" - } + "billableEvents":{ + "type":"array", + "minItems":1, + "items":{ + "type":"object", + "required":[ + "accountId", + "vendor", + "type", + "billingId" + ], + "properties":{ + "vendor":{ + "type":"string", + "description":"The name of the vendor who emitted the billable event" + }, + "type":{ + "type":"string", + "description":"The type of billable event", + "enum":[ + "impression", + "pageLoad", + "auction", + "request", + "general" + ] + }, + "billingId":{ + "type":"string", + "description":"A UUID which is responsible more mapping this event to" + }, + "accountId": { + "type": "number", + "description": "The account id for the rubicon publisher" + } + } + } + } }, "definitions": { "gam": {