From 575884f46c4d24fafe5ae9859fac48d10ff9676f Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Mon, 5 Sep 2022 14:46:25 +0530 Subject: [PATCH 01/13] Topics: Initial Topics iframe implementation --- modules/topicsFpdModule.js | 75 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index dbd1d3e90e9..2529fbe662b 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -2,6 +2,12 @@ import {logError, logWarn, mergeDeep} from '../src/utils.js'; import {getRefererInfo} from '../src/refererDetection.js'; import {submodule} from '../src/hook.js'; import {GreedyPromise} from '../src/utils/promise.js'; +import {getGlobal} from '../src/prebidGlobal.js'; +import {config} from '../src/config.js'; +import { getStorageManager } from '../src/storageManager.js'; + +export const storage = getStorageManager(); +export const initialTopicName = 'tps_'; const TAXONOMIES = { // map from topic taxonomyVersion to IAB segment taxonomy @@ -58,10 +64,31 @@ export function getTopics(doc = document) { return topics; } +// function to fetch the cached topic data from storage for bidders and return it +export function getTpsForBidderFromStorage(){ + let cachedTopicData = []; + const topics = config.getConfig('userSync.topics'); + let biddersList = Object.keys(topics.bidders || []); + Object.keys(window.localStorage).forEach((storeItem) => { + // Get Storage Item starting only with tps_. + if (storeItem.startsWith(initialTopicName)) { + // Check bidder exist in config for cached bidder data and if not delete it since it is stale cached data + if(biddersList.includes(storeItem.slice(initialTopicName.length))){ + let segmentData = JSON.parse(storage.getDataFromLocalStorage(storeItem) ); + segmentData.forEach(segment => cachedTopicData.push(segment)); + }else{ + storage.removeDataFromLocalStorage(storeItem) + } + } + }) + return cachedTopicData; +} + const topicsData = getTopics().then((topics) => getTopicsData(getRefererInfo().domain, topics)); export function processFpd(config, {global}, {data = topicsData} = {}) { return data.then((data) => { + data = [].concat(data, getTpsForBidderFromStorage()); // Add cached data in FPD data. if (data.length) { mergeDeep(global, { user: { @@ -73,6 +100,54 @@ export function processFpd(config, {global}, {data = topicsData} = {}) { }); } +// Added function addListenerToFetchTopics to listen for topic data retured from iframe loaded for SSPs(bidder) +function addListenerToFetchTopics() { + window.addEventListener('message', (e) => { + if (e && e.data) { + try { + let data = JSON.parse(e.data); + if (data?.segment?.topics) { + let {domain, topics, bidder} = data.segment; + let segmentData = getTopicsData(domain, topics); + let updateStore = []; + let bidderFoundInStore = false; + let storedSegments = JSON.parse(storage.getDataFromLocalStorage(`${initialTopicName}${bidder}`)); + storedSegments?.forEach((segment) => { + if (segment.name === segmentData[0].name && segment.ext.segclass == segmentData[0].ext.segclass) { + updateStore.push(segmentData[0]); + bidderFoundInStore = true; + } else { + updateStore.push(segment); + }; + }); + !bidderFoundInStore && updateStore.push(segmentData[0]); + storage.setDataInLocalStorage(`${initialTopicName}${bidder}`, JSON.stringify(updateStore)); + } + } catch (err) { } + } + }); +} + +function loadTopicsForBidders() { + addListenerToFetchTopics(); + const topics = config.getConfig('userSync.topics'); + if (topics) { + const bidders = Object.keys(topics.bidders ||[]); + bidders?.forEach((bidder) => { + let ifrm = document.createElement('iframe'); + ifrm.name = `ifrm_${bidder}`; + ifrm.src = `${topics['bidders'][bidder]['iframeURL']}?bidder=${bidder}`; + ifrm.style.display = "none"; + window.document.documentElement.appendChild(ifrm); + }); + } else { + logWarn(`${MODULE_NAME} - Topics : Topics config not defined under userSync Object`); + } +} + +// Exposing loadTopicsForBidders function in global-name-space so that API calling from domain can get the data and send it in oRTB format. +(getGlobal()).loadTopicsForBidders = loadTopicsForBidders; + submodule('firstPartyData', { name: 'topics', queue: 1, From bda432574373c5662884549c0a6b7ec6a4a1f028 Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Mon, 5 Sep 2022 15:28:33 +0530 Subject: [PATCH 02/13] Topics API: LINT errors solved --- modules/topicsFpdModule.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index 2529fbe662b..78c7462ae0e 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -65,7 +65,7 @@ export function getTopics(doc = document) { } // function to fetch the cached topic data from storage for bidders and return it -export function getTpsForBidderFromStorage(){ +export function getTpsForBidderFromStorage() { let cachedTopicData = []; const topics = config.getConfig('userSync.topics'); let biddersList = Object.keys(topics.bidders || []); @@ -73,10 +73,10 @@ export function getTpsForBidderFromStorage(){ // Get Storage Item starting only with tps_. if (storeItem.startsWith(initialTopicName)) { // Check bidder exist in config for cached bidder data and if not delete it since it is stale cached data - if(biddersList.includes(storeItem.slice(initialTopicName.length))){ - let segmentData = JSON.parse(storage.getDataFromLocalStorage(storeItem) ); + if (biddersList.includes(storeItem.slice(initialTopicName.length))) { + let segmentData = JSON.parse(storage.getDataFromLocalStorage(storeItem)); segmentData.forEach(segment => cachedTopicData.push(segment)); - }else{ + } else { storage.removeDataFromLocalStorage(storeItem) } } @@ -132,16 +132,16 @@ function loadTopicsForBidders() { addListenerToFetchTopics(); const topics = config.getConfig('userSync.topics'); if (topics) { - const bidders = Object.keys(topics.bidders ||[]); + const bidders = Object.keys(topics.bidders || []); bidders?.forEach((bidder) => { let ifrm = document.createElement('iframe'); ifrm.name = `ifrm_${bidder}`; ifrm.src = `${topics['bidders'][bidder]['iframeURL']}?bidder=${bidder}`; - ifrm.style.display = "none"; + ifrm.style.display = 'none'; window.document.documentElement.appendChild(ifrm); }); } else { - logWarn(`${MODULE_NAME} - Topics : Topics config not defined under userSync Object`); + logWarn(`Topics config not defined under userSync Object`); } } From 1cbd1627c7a51207b256fa7d6f53a560666ed310 Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Tue, 6 Sep 2022 17:59:30 +0530 Subject: [PATCH 03/13] Added Empty Topics Check --- modules/topicsFpdModule.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index 78c7462ae0e..dc9f202126a 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -1,4 +1,4 @@ -import {logError, logWarn, mergeDeep} from '../src/utils.js'; +import {logError, logWarn, mergeDeep, isEmpty} from '../src/utils.js'; import {getRefererInfo} from '../src/refererDetection.js'; import {submodule} from '../src/hook.js'; import {GreedyPromise} from '../src/utils/promise.js'; @@ -106,7 +106,7 @@ function addListenerToFetchTopics() { if (e && e.data) { try { let data = JSON.parse(e.data); - if (data?.segment?.topics) { + if (data && data.segment && !isEmpty(data.segment.topics)) { let {domain, topics, bidder} = data.segment; let segmentData = getTopicsData(domain, topics); let updateStore = []; From 2cbb16aa3df8b42ff760b4f30cd35912b854057c Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Fri, 16 Sep 2022 18:13:06 +0530 Subject: [PATCH 04/13] Topics: Storage Map logic and added message listener secure check --- modules/topicsFpdModule.js | 128 +++++++++++++++++++++---------------- 1 file changed, 74 insertions(+), 54 deletions(-) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index dc9f202126a..b4db430ef3a 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -1,14 +1,14 @@ -import {logError, logWarn, mergeDeep, isEmpty} from '../src/utils.js'; +import {logError, logWarn, mergeDeep, isEmpty, safeJSONParse} from '../src/utils.js'; import {getRefererInfo} from '../src/refererDetection.js'; import {submodule} from '../src/hook.js'; import {GreedyPromise} from '../src/utils/promise.js'; -import {getGlobal} from '../src/prebidGlobal.js'; import {config} from '../src/config.js'; import { getStorageManager } from '../src/storageManager.js'; export const storage = getStorageManager(); -export const initialTopicName = 'tps_'; +export const topicStorageName = 'prebid:topics'; +const iframeLoadedURL = []; const TAXONOMIES = { // map from topic taxonomyVersion to IAB segment taxonomy '1': 600 @@ -23,6 +23,20 @@ function partitionBy(field, items) { }, {}); } +/** + * function to get list of loaded Iframes calling Topics API + */ +function getLoadedIframeURL() { + return iframeLoadedURL; +} + +/** + * function to set/push iframe in the list which is loaded to called topics API. + */ +function setLoadedIframeURL(url) { + return iframeLoadedURL.push(url); +} + export function getTopicsData(name, topics, taxonomies = TAXONOMIES) { return Object.entries(partitionBy('taxonomyVersion', topics)) .filter(([taxonomyVersion]) => { @@ -64,31 +78,12 @@ export function getTopics(doc = document) { return topics; } -// function to fetch the cached topic data from storage for bidders and return it -export function getTpsForBidderFromStorage() { - let cachedTopicData = []; - const topics = config.getConfig('userSync.topics'); - let biddersList = Object.keys(topics.bidders || []); - Object.keys(window.localStorage).forEach((storeItem) => { - // Get Storage Item starting only with tps_. - if (storeItem.startsWith(initialTopicName)) { - // Check bidder exist in config for cached bidder data and if not delete it since it is stale cached data - if (biddersList.includes(storeItem.slice(initialTopicName.length))) { - let segmentData = JSON.parse(storage.getDataFromLocalStorage(storeItem)); - segmentData.forEach(segment => cachedTopicData.push(segment)); - } else { - storage.removeDataFromLocalStorage(storeItem) - } - } - }) - return cachedTopicData; -} - const topicsData = getTopics().then((topics) => getTopicsData(getRefererInfo().domain, topics)); export function processFpd(config, {global}, {data = topicsData} = {}) { + loadTopicsForBidders(); return data.then((data) => { - data = [].concat(data, getTpsForBidderFromStorage()); // Add cached data in FPD data. + data = [].concat(data, getCachedTopics()); // Add cached data in FPD data. if (data.length) { mergeDeep(global, { user: { @@ -100,44 +95,72 @@ export function processFpd(config, {global}, {data = topicsData} = {}) { }); } -// Added function addListenerToFetchTopics to listen for topic data retured from iframe loaded for SSPs(bidder) -function addListenerToFetchTopics() { - window.addEventListener('message', (e) => { - if (e && e.data) { - try { - let data = JSON.parse(e.data); - if (data && data.segment && !isEmpty(data.segment.topics)) { - let {domain, topics, bidder} = data.segment; - let segmentData = getTopicsData(domain, topics); - let updateStore = []; - let bidderFoundInStore = false; - let storedSegments = JSON.parse(storage.getDataFromLocalStorage(`${initialTopicName}${bidder}`)); - storedSegments?.forEach((segment) => { - if (segment.name === segmentData[0].name && segment.ext.segclass == segmentData[0].ext.segclass) { - updateStore.push(segmentData[0]); - bidderFoundInStore = true; - } else { - updateStore.push(segment); - }; - }); - !bidderFoundInStore && updateStore.push(segmentData[0]); - storage.setDataInLocalStorage(`${initialTopicName}${bidder}`, JSON.stringify(updateStore)); - } - } catch (err) { } +/** + * function to fetch the cached topic data from storage for bidders and return it + */ +function getCachedTopics() { + let cachedTopicData = []; + const topics = config.getConfig('userSync.topics'); + const bidderList = Object.keys(topics.bidders || []); + new Map(safeJSONParse(storage.getDataFromLocalStorage(topicStorageName))).forEach((value, key) => { + let bidder = `${key}`.split(':')[0]; + // Check bidder exist in config for cached bidder data and then only retrieve the cached data + if (bidderList.includes(bidder)) { + cachedTopicData.push(value); } }); + return cachedTopicData; +} + +/** + * Recieve messages from iframe loaded for bidders to fetch topic + * @param {MessageEvent} evt + */ +function receiveMessage(evt) { + if (evt && evt.data) { + try { + let data = safeJSONParse(evt.data); + if (getLoadedIframeURL().includes(evt.origin) && data && data.segment && !isEmpty(data.segment.topics)) { + const {domain, topics, bidder} = data.segment; + const iframeTopicsData = getTopicsData(domain, topics)[0]; + iframeTopicsData && storeInLocalStorage(bidder, iframeTopicsData); + } + } catch (err) { } + } +} + +/** +Function to store Topics data recieved from iframe in storage(name: "prebid:topics") +* @param {Topics} topics +*/ +export function storeInLocalStorage(bidder, topics) { + const storedSegments = new Map(safeJSONParse(storage.getDataFromLocalStorage(topicStorageName))); + storedSegments.set(`${bidder}:${topics.ext.segclass}`, topics); + storage.setDataInLocalStorage(topicStorageName, JSON.stringify([...storedSegments])); } +/** + * function to add listener for message receiving from IFRAME + */ +function listenMessagesFromTopicIframe() { + window.addEventListener('message', receiveMessage, false); +} + +/** + * function to load the iframes of the bidder to load the topics data + */ function loadTopicsForBidders() { - addListenerToFetchTopics(); const topics = config.getConfig('userSync.topics'); if (topics) { + listenMessagesFromTopicIframe(); const bidders = Object.keys(topics.bidders || []); - bidders?.forEach((bidder) => { + bidders && bidders.forEach((bidder) => { + const iframeURL = `${topics['bidders'][bidder]['iframeURL']}`; let ifrm = document.createElement('iframe'); ifrm.name = `ifrm_${bidder}`; - ifrm.src = `${topics['bidders'][bidder]['iframeURL']}?bidder=${bidder}`; + ifrm.src = `${iframeURL}?bidder=${bidder}`; ifrm.style.display = 'none'; + setLoadedIframeURL(new URL(iframeURL).origin); window.document.documentElement.appendChild(ifrm); }); } else { @@ -145,9 +168,6 @@ function loadTopicsForBidders() { } } -// Exposing loadTopicsForBidders function in global-name-space so that API calling from domain can get the data and send it in oRTB format. -(getGlobal()).loadTopicsForBidders = loadTopicsForBidders; - submodule('firstPartyData', { name: 'topics', queue: 1, From bb10172bc6b10976b16e61d1e912f33199369b93 Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Tue, 11 Oct 2022 12:43:32 +0530 Subject: [PATCH 05/13] Topics: Iframe implementation for bidders --- modules/topicsFpdModule.js | 82 ++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 20 deletions(-) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index b4db430ef3a..5d6f10ee0cf 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -3,10 +3,12 @@ import {getRefererInfo} from '../src/refererDetection.js'; import {submodule} from '../src/hook.js'; import {GreedyPromise} from '../src/utils/promise.js'; import {config} from '../src/config.js'; -import { getStorageManager } from '../src/storageManager.js'; +import {getStorageManager} from '../src/storageManager.js'; +import {includes} from '../src/polyfill.js'; export const storage = getStorageManager(); export const topicStorageName = 'prebid:topics'; +export const lastUpdated = 'lastUpdated'; const iframeLoadedURL = []; const TAXONOMIES = { @@ -14,6 +16,16 @@ const TAXONOMIES = { '1': 600 } +const DEFAULT_EXPIRATION_DAYS = 21; + +const bidderIframeList = { + maxTopicCaller: 1, + bidders: [{ + bidder: 'pubmatic', + iframeURL: 'https://pubmatic.com:8080/topics/fpd/topic.html' // dummy URL for NOW + }] +} + function partitionBy(field, items) { return items.reduce((partitions, item) => { const key = item[field]; @@ -100,13 +112,22 @@ export function processFpd(config, {global}, {data = topicsData} = {}) { */ function getCachedTopics() { let cachedTopicData = []; - const topics = config.getConfig('userSync.topics'); - const bidderList = Object.keys(topics.bidders || []); - new Map(safeJSONParse(storage.getDataFromLocalStorage(topicStorageName))).forEach((value, key) => { - let bidder = `${key}`.split(':')[0]; + const topics = config.getConfig('userSync.topics') || bidderIframeList; + const bidderList = topics.bidders || []; + let storedSegments = new Map(safeJSONParse(storage.getDataFromLocalStorage(topicStorageName))); + storedSegments && storedSegments.forEach((value, cachedBidder) => { // Check bidder exist in config for cached bidder data and then only retrieve the cached data - if (bidderList.includes(bidder)) { - cachedTopicData.push(value); + let isBidderConfigured = bidderList.some(({bidder}) => cachedBidder == bidder) + if (isBidderConfigured) { + if (!isCachedDataExpired(value[lastUpdated], isBidderConfigured?.expiry || DEFAULT_EXPIRATION_DAYS)) { + Object.keys(value).forEach((segData) => { + value != lastUpdated && cachedTopicData.push(value[segData]); + }) + } else { + // delete the specific bidder map from the store and store the updated maps + storedSegments.delete(cachedBidder); + storage.setDataInLocalStorage(topicStorageName, JSON.stringify([...storedSegments])); + } } }); return cachedTopicData; @@ -120,7 +141,7 @@ function receiveMessage(evt) { if (evt && evt.data) { try { let data = safeJSONParse(evt.data); - if (getLoadedIframeURL().includes(evt.origin) && data && data.segment && !isEmpty(data.segment.topics)) { + if (includes(getLoadedIframeURL(), evt.origin) && data && data.segment && !isEmpty(data.segment.topics)) { const {domain, topics, bidder} = data.segment; const iframeTopicsData = getTopicsData(domain, topics)[0]; iframeTopicsData && storeInLocalStorage(bidder, iframeTopicsData); @@ -135,10 +156,30 @@ Function to store Topics data recieved from iframe in storage(name: "prebid:topi */ export function storeInLocalStorage(bidder, topics) { const storedSegments = new Map(safeJSONParse(storage.getDataFromLocalStorage(topicStorageName))); - storedSegments.set(`${bidder}:${topics.ext.segclass}`, topics); + if (storedSegments.has(bidder)) { + storedSegments.get(bidder)[topics['ext']['segclass']] = topics; + storedSegments.get(bidder)[lastUpdated] = new Date().getTime(); + storedSegments.set(bidder, storedSegments.get(bidder)); + } else { + storedSegments.set(bidder, {[topics.ext.segclass]: topics, [lastUpdated]: new Date().getTime()}) + } storage.setDataInLocalStorage(topicStorageName, JSON.stringify([...storedSegments])); } +function isCachedDataExpired(storedTime, cacheTime) { + const _MS_PER_DAY = 1000 * 60 * 60 * 24; + const now = new Date().getTime(); + const daysDifference = Math.floor((storedTime - now) / _MS_PER_DAY); + return daysDifference > cacheTime; +} + +/** +* Function to get random bidders based on count passed with array of bidders +**/ +function getRandomBidders(arr, count) { + return ([...arr].sort(() => 0.5 - Math.random())).slice(0, count) +} + /** * function to add listener for message receiving from IFRAME */ @@ -150,19 +191,20 @@ function listenMessagesFromTopicIframe() { * function to load the iframes of the bidder to load the topics data */ function loadTopicsForBidders() { - const topics = config.getConfig('userSync.topics'); + const topics = config.getConfig('userSync.topics') || bidderIframeList; if (topics) { listenMessagesFromTopicIframe(); - const bidders = Object.keys(topics.bidders || []); - bidders && bidders.forEach((bidder) => { - const iframeURL = `${topics['bidders'][bidder]['iframeURL']}`; - let ifrm = document.createElement('iframe'); - ifrm.name = `ifrm_${bidder}`; - ifrm.src = `${iframeURL}?bidder=${bidder}`; - ifrm.style.display = 'none'; - setLoadedIframeURL(new URL(iframeURL).origin); - window.document.documentElement.appendChild(ifrm); - }); + const randomBidders = getRandomBidders(topics.bidders || [], topics.maxTopicCaller || 1) + randomBidders && randomBidders.forEach(({ bidder, iframeURL }) => { + if (bidder && iframeURL) { + let ifrm = document.createElement('iframe'); + ifrm.name = 'ifrm_'.concat(bidder); + ifrm.src = ''.concat(iframeURL, '?bidder=').concat(bidder); + ifrm.style.display = 'none'; + setLoadedIframeURL(new URL(iframeURL).origin); + iframeURL && window.document.documentElement.appendChild(ifrm); + } + }) } else { logWarn(`Topics config not defined under userSync Object`); } From 3f34d3e7aea96dc9d7efe4925e92133015ecc077 Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Wed, 12 Oct 2022 16:32:18 +0530 Subject: [PATCH 06/13] Added topics_iframe html in example for reference --- integrationExamples/gpt/topics_frame.html | 43 +++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 integrationExamples/gpt/topics_frame.html diff --git a/integrationExamples/gpt/topics_frame.html b/integrationExamples/gpt/topics_frame.html new file mode 100644 index 00000000000..7a12b030a6a --- /dev/null +++ b/integrationExamples/gpt/topics_frame.html @@ -0,0 +1,43 @@ + + + + Topics demo + + + + + + + + + \ No newline at end of file From 9abc2bd7363149ccfaca06a980f33f68b946c040 Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Mon, 31 Oct 2022 15:28:05 +0530 Subject: [PATCH 07/13] Added Pubmatic Topic iframe URL --- modules/topicsFpdModule.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index 5d6f10ee0cf..cb234d96de0 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -22,7 +22,7 @@ const bidderIframeList = { maxTopicCaller: 1, bidders: [{ bidder: 'pubmatic', - iframeURL: 'https://pubmatic.com:8080/topics/fpd/topic.html' // dummy URL for NOW + iframeURL: 'https://ads.pubmatic.com/AdServer/js/topics/topics_frame.html' // dummy URL for NOW }] } From d18ea65ccd81e898601315b8486ef9046899339f Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Mon, 31 Oct 2022 15:29:53 +0530 Subject: [PATCH 08/13] Added Pubmatic Topic iframe URL- Removed comment --- modules/topicsFpdModule.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index cb234d96de0..70fe76012b6 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -22,7 +22,7 @@ const bidderIframeList = { maxTopicCaller: 1, bidders: [{ bidder: 'pubmatic', - iframeURL: 'https://ads.pubmatic.com/AdServer/js/topics/topics_frame.html' // dummy URL for NOW + iframeURL: 'https://ads.pubmatic.com/AdServer/js/topics/topics_frame.html' }] } From 3e9a034a36345f33331922ed840014faee8fb191 Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Mon, 28 Nov 2022 16:54:37 +0530 Subject: [PATCH 09/13] Topics Module: Consent management logic added --- modules/topicsFpdModule.js | 77 +++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index 70fe76012b6..f929b3f2cf8 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -1,22 +1,17 @@ -import {logError, logWarn, mergeDeep, isEmpty, safeJSONParse} from '../src/utils.js'; +import {logError, logWarn, mergeDeep, isEmpty, safeJSONParse, logInfo} from '../src/utils.js'; import {getRefererInfo} from '../src/refererDetection.js'; import {submodule} from '../src/hook.js'; import {GreedyPromise} from '../src/utils/promise.js'; import {config} from '../src/config.js'; -import {getStorageManager} from '../src/storageManager.js'; +import {getCoreStorageManager} from '../src/storageManager.js'; import {includes} from '../src/polyfill.js'; +import {gdprDataHandler} from '../src/adapterManager.js'; -export const storage = getStorageManager(); -export const topicStorageName = 'prebid:topics'; -export const lastUpdated = 'lastUpdated'; - -const iframeLoadedURL = []; -const TAXONOMIES = { - // map from topic taxonomyVersion to IAB segment taxonomy - '1': 600 -} - +const MODULE_NAME = 'topicsFpd'; const DEFAULT_EXPIRATION_DAYS = 21; +const TCF_REQUIRED_PURPOSES = ['1', '2', '3', '4']; +let HAS_GDPR_CONSENT = true; +let LOAD_TOPICS_INITIALISE = false; const bidderIframeList = { maxTopicCaller: 1, @@ -25,6 +20,15 @@ const bidderIframeList = { iframeURL: 'https://ads.pubmatic.com/AdServer/js/topics/topics_frame.html' }] } +export const coreStorage = getCoreStorageManager(MODULE_NAME); +export const topicStorageName = 'prebid:topics'; +export const lastUpdated = 'lastUpdated'; + +const iframeLoadedURL = []; +const TAXONOMIES = { + // map from topic taxonomyVersion to IAB segment taxonomy + '1': 600 +} function partitionBy(field, items) { return items.reduce((partitions, item) => { @@ -93,7 +97,10 @@ export function getTopics(doc = document) { const topicsData = getTopics().then((topics) => getTopicsData(getRefererInfo().domain, topics)); export function processFpd(config, {global}, {data = topicsData} = {}) { - loadTopicsForBidders(); + if (!LOAD_TOPICS_INITIALISE) { + loadTopicsForBidders(); + LOAD_TOPICS_INITIALISE = true; + } return data.then((data) => { data = [].concat(data, getCachedTopics()); // Add cached data in FPD data. if (data.length) { @@ -112,9 +119,12 @@ export function processFpd(config, {global}, {data = topicsData} = {}) { */ function getCachedTopics() { let cachedTopicData = []; + if (!HAS_GDPR_CONSENT) { + return cachedTopicData; + } const topics = config.getConfig('userSync.topics') || bidderIframeList; const bidderList = topics.bidders || []; - let storedSegments = new Map(safeJSONParse(storage.getDataFromLocalStorage(topicStorageName))); + let storedSegments = new Map(safeJSONParse(coreStorage.getDataFromLocalStorage(topicStorageName))); storedSegments && storedSegments.forEach((value, cachedBidder) => { // Check bidder exist in config for cached bidder data and then only retrieve the cached data let isBidderConfigured = bidderList.some(({bidder}) => cachedBidder == bidder) @@ -126,7 +136,7 @@ function getCachedTopics() { } else { // delete the specific bidder map from the store and store the updated maps storedSegments.delete(cachedBidder); - storage.setDataInLocalStorage(topicStorageName, JSON.stringify([...storedSegments])); + coreStorage.setDataInLocalStorage(topicStorageName, JSON.stringify([...storedSegments])); } } }); @@ -155,7 +165,7 @@ Function to store Topics data recieved from iframe in storage(name: "prebid:topi * @param {Topics} topics */ export function storeInLocalStorage(bidder, topics) { - const storedSegments = new Map(safeJSONParse(storage.getDataFromLocalStorage(topicStorageName))); + const storedSegments = new Map(safeJSONParse(coreStorage.getDataFromLocalStorage(topicStorageName))); if (storedSegments.has(bidder)) { storedSegments.get(bidder)[topics['ext']['segclass']] = topics; storedSegments.get(bidder)[lastUpdated] = new Date().getTime(); @@ -163,7 +173,7 @@ export function storeInLocalStorage(bidder, topics) { } else { storedSegments.set(bidder, {[topics.ext.segclass]: topics, [lastUpdated]: new Date().getTime()}) } - storage.setDataInLocalStorage(topicStorageName, JSON.stringify([...storedSegments])); + coreStorage.setDataInLocalStorage(topicStorageName, JSON.stringify([...storedSegments])); } function isCachedDataExpired(storedTime, cacheTime) { @@ -187,10 +197,43 @@ function listenMessagesFromTopicIframe() { window.addEventListener('message', receiveMessage, false); } +function checkTCFv2(vendorData, requiredPurposes = TCF_REQUIRED_PURPOSES) { + const {gdprApplies, purpose} = vendorData; + if (!gdprApplies) { + return true; + } + return requiredPurposes.map((purposeNo) => { + const purposeConsent = purpose.consents ? purpose.consents[purposeNo] : false; + if (purposeConsent) { + return true; + } + return false; + }).reduce((a, b) => a && b, true); +} + +function hasGDPRConsent() { + // Check for GDPR consent for purpose 1,2,3,4 and return false if consent has not been given + const gdprConsent = gdprDataHandler.getConsentData(); + const hasGdpr = (gdprConsent && typeof gdprConsent.gdprApplies === 'boolean' && gdprConsent.gdprApplies) ? 1 : 0; + const gdprConsentString = hasGdpr ? gdprConsent.consentString : ''; + if (hasGdpr) { + if ((!gdprConsentString || gdprConsentString === '') || !gdprConsent.vendorData) { + return false; + } + return checkTCFv2(gdprConsent.vendorData); + } + return true; +} + /** * function to load the iframes of the bidder to load the topics data */ function loadTopicsForBidders() { + HAS_GDPR_CONSENT = hasGDPRConsent(); + if (!HAS_GDPR_CONSENT) { + logInfo('Topics Module : Consent string is required to fetch the topics from third party domains.'); + return; + } const topics = config.getConfig('userSync.topics') || bidderIframeList; if (topics) { listenMessagesFromTopicIframe(); From 642e4b1c81732f6896ae62acd49023cf2161306f Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Tue, 29 Nov 2022 11:35:22 +0530 Subject: [PATCH 10/13] Topics Module: Added Device Access check --- modules/topicsFpdModule.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index f929b3f2cf8..342d00c6a3c 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -1,4 +1,4 @@ -import {logError, logWarn, mergeDeep, isEmpty, safeJSONParse, logInfo} from '../src/utils.js'; +import {logError, logWarn, mergeDeep, isEmpty, safeJSONParse, logInfo, hasDeviceAccess} from '../src/utils.js'; import {getRefererInfo} from '../src/refererDetection.js'; import {submodule} from '../src/hook.js'; import {GreedyPromise} from '../src/utils/promise.js'; @@ -12,6 +12,7 @@ const DEFAULT_EXPIRATION_DAYS = 21; const TCF_REQUIRED_PURPOSES = ['1', '2', '3', '4']; let HAS_GDPR_CONSENT = true; let LOAD_TOPICS_INITIALISE = false; +const HAS_DEVICE_ACCESS = hasDeviceAccess(); const bidderIframeList = { maxTopicCaller: 1, @@ -119,7 +120,7 @@ export function processFpd(config, {global}, {data = topicsData} = {}) { */ function getCachedTopics() { let cachedTopicData = []; - if (!HAS_GDPR_CONSENT) { + if (!HAS_GDPR_CONSENT && !HAS_DEVICE_ACCESS) { return cachedTopicData; } const topics = config.getConfig('userSync.topics') || bidderIframeList; @@ -230,7 +231,7 @@ function hasGDPRConsent() { */ function loadTopicsForBidders() { HAS_GDPR_CONSENT = hasGDPRConsent(); - if (!HAS_GDPR_CONSENT) { + if (!HAS_GDPR_CONSENT && !HAS_DEVICE_ACCESS) { logInfo('Topics Module : Consent string is required to fetch the topics from third party domains.'); return; } From 9915442e7b85634a2d1ad5b7540b78de55c799e8 Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Wed, 30 Nov 2022 01:04:29 +0530 Subject: [PATCH 11/13] Topics Module: Unit test cases added and minor changes --- modules/topicsFpdModule.js | 16 +- test/spec/modules/topicsFpdModule_spec.js | 178 +++++++++++++++++++++- 2 files changed, 184 insertions(+), 10 deletions(-) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index 342d00c6a3c..4c35b5ce541 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -118,9 +118,9 @@ export function processFpd(config, {global}, {data = topicsData} = {}) { /** * function to fetch the cached topic data from storage for bidders and return it */ -function getCachedTopics() { +export function getCachedTopics() { let cachedTopicData = []; - if (!HAS_GDPR_CONSENT && !HAS_DEVICE_ACCESS) { + if (!HAS_GDPR_CONSENT || !HAS_DEVICE_ACCESS) { return cachedTopicData; } const topics = config.getConfig('userSync.topics') || bidderIframeList; @@ -132,7 +132,7 @@ function getCachedTopics() { if (isBidderConfigured) { if (!isCachedDataExpired(value[lastUpdated], isBidderConfigured?.expiry || DEFAULT_EXPIRATION_DAYS)) { Object.keys(value).forEach((segData) => { - value != lastUpdated && cachedTopicData.push(value[segData]); + segData != lastUpdated && cachedTopicData.push(value[segData]); }) } else { // delete the specific bidder map from the store and store the updated maps @@ -148,7 +148,7 @@ function getCachedTopics() { * Recieve messages from iframe loaded for bidders to fetch topic * @param {MessageEvent} evt */ -function receiveMessage(evt) { +export function receiveMessage(evt) { if (evt && evt.data) { try { let data = safeJSONParse(evt.data); @@ -179,8 +179,8 @@ export function storeInLocalStorage(bidder, topics) { function isCachedDataExpired(storedTime, cacheTime) { const _MS_PER_DAY = 1000 * 60 * 60 * 24; - const now = new Date().getTime(); - const daysDifference = Math.floor((storedTime - now) / _MS_PER_DAY); + const currentTime = new Date().getTime(); + const daysDifference = Math.ceil((currentTime - storedTime) / _MS_PER_DAY); return daysDifference > cacheTime; } @@ -212,7 +212,7 @@ function checkTCFv2(vendorData, requiredPurposes = TCF_REQUIRED_PURPOSES) { }).reduce((a, b) => a && b, true); } -function hasGDPRConsent() { +export function hasGDPRConsent() { // Check for GDPR consent for purpose 1,2,3,4 and return false if consent has not been given const gdprConsent = gdprDataHandler.getConsentData(); const hasGdpr = (gdprConsent && typeof gdprConsent.gdprApplies === 'boolean' && gdprConsent.gdprApplies) ? 1 : 0; @@ -231,7 +231,7 @@ function hasGDPRConsent() { */ function loadTopicsForBidders() { HAS_GDPR_CONSENT = hasGDPRConsent(); - if (!HAS_GDPR_CONSENT && !HAS_DEVICE_ACCESS) { + if (!HAS_GDPR_CONSENT || !HAS_DEVICE_ACCESS) { logInfo('Topics Module : Consent string is required to fetch the topics from third party domains.'); return; } diff --git a/test/spec/modules/topicsFpdModule_spec.js b/test/spec/modules/topicsFpdModule_spec.js index 3781768497b..69e7cc16ca8 100644 --- a/test/spec/modules/topicsFpdModule_spec.js +++ b/test/spec/modules/topicsFpdModule_spec.js @@ -1,5 +1,7 @@ -import {getTopics, getTopicsData, processFpd} from '../../../modules/topicsFpdModule.js'; -import {deepClone} from '../../../src/utils.js'; +import {getTopics, getTopicsData, processFpd, hasGDPRConsent, getCachedTopics, receiveMessage, topicStorageName} from '../../../modules/topicsFpdModule.js'; +import {deepClone, safeJSONParse} from '../../../src/utils.js'; +import {gdprDataHandler} from 'src/adapterManager.js'; +import {getCoreStorageManager} from 'src/storageManager.js'; describe('getTopicsData', () => { function makeTopic(topic, modelv, taxv = '1') { @@ -237,3 +239,175 @@ describe('processFpd', () => { }); }); }); + +describe('Topics Module GDPR consent check', () => { + let gdprDataHdlrStub; + beforeEach(() => { + gdprDataHdlrStub = sinon.stub(gdprDataHandler, 'getConsentData'); + }) + + afterEach(() => { + gdprDataHdlrStub.restore(); + }); + + it('should return false when GDPR is applied but consent string is not present', () => { + const consentString = ''; + const consentConfig = { + consentString: consentString, + gdprApplies: true, + vendorData: {} + }; + gdprDataHdlrStub.returns(consentConfig); + expect(hasGDPRConsent()).to.equal(false); + }); + + it("should return true when GDPR doesn't apply", () => { + const consentString = 'CPi8wgAPi8wgAADABBENCrCsAP_AAH_AAAAAISNB7D=='; + const consentConfig = { + consentString: consentString, + gdprApplies: false, + vendorData: {} + }; + + gdprDataHdlrStub.returns(consentConfig); + expect(hasGDPRConsent()).to.equal(true); + }); + + it('should return true when GDPR is applied and purpose consent is true for all purpose[1,2,3,4]', () => { + const consentString = 'CPi8wgAPi8wgAADABBENCrCsAP_AAH_AAAAAISNB7D=='; + const consentConfig = { + consentString: consentString, + gdprApplies: true, + vendorData: { + metadata: consentString, + gdprApplies: true, + purpose: { + consents: { + 1: true, + 2: true, + 3: true, + 4: true + } + } + } + }; + + gdprDataHdlrStub.returns(consentConfig); + expect(hasGDPRConsent()).to.equal(true); + }); + + it('should return false when GDPR is applied and purpose consent is false for one of the purpose[1,2,3,4]', () => { + const consentString = 'CPi8wgAPi8wgAADABBENCrCsAP_AAH_AAAAAISNB7D=='; + const consentConfig = { + consentString: consentString, + gdprApplies: true, + vendorData: { + metadata: consentString, + gdprApplies: true, + purpose: { + consents: { + 1: true, + 2: true, + 3: true, + 4: false + } + } + } + }; + + gdprDataHdlrStub.returns(consentConfig); + expect(hasGDPRConsent()).to.equal(false); + }); +}); + +describe('getCachedTopics()', () => { + const storage = getCoreStorageManager('topicsFpd'); + const expected = [{ + ext: { + segtax: 600, + segclass: '2206021246' + }, + segment: [{ + 'id': '243' + }, { + 'id': '265' + }], + name: 'ads.pubmatic.com' + }]; + const consentString = 'CPi8wgAPi8wgAADABBENCrCsAP_AAH_AAAAAISNB7D=='; + const consentConfig = { + consentString: consentString, + gdprApplies: true, + vendorData: { + metadata: consentString, + gdprApplies: true, + purpose: { + consents: { + 1: true, + 2: true, + 3: true, + 4: true + } + } + } + }; + const mockData = [ + { + name: 'domain', + segment: [{id: 123}] + }, + { + name: 'domain', + segment: [{id: 321}], + } + ]; + + const evt = { + data: '{"segment":{"domain":"ads.pubmatic.com","topics":[{"configVersion":"chrome.1","modelVersion":"2206021246","taxonomyVersion":"1","topic":165,"version":"chrome.1:1:2206021246"}],"bidder":"pubmatic"},"date":1669743901858}', + origin: 'https://ads.pubmatic.com' + } + + let gdprDataHdlrStub; + beforeEach(() => { + gdprDataHdlrStub = sinon.stub(gdprDataHandler, 'getConsentData'); + }); + + afterEach(() => { + storage.removeDataFromLocalStorage(topicStorageName); + gdprDataHdlrStub.restore(); + }); + + it('should return segments for bidder if GDPR consent is true and there is cached segments stored which is not expired', () => { + let storedSegments = '[["pubmatic",{"2206021246":{"ext":{"segtax":600,"segclass":"2206021246"},"segment":[{"id":"243"},{"id":"265"}],"name":"ads.pubmatic.com"},"lastUpdated":1669719242027}]]'; + storage.setDataInLocalStorage(topicStorageName, storedSegments); + gdprDataHdlrStub.returns(consentConfig); + assert.deepEqual(getCachedTopics(), expected); + }); + + it('should return empty segments for bidder if GDPR consent is true and there is cached segments stored which is expired', () => { + let storedSegments = '[["pubmatic",{"2206021246":{"ext":{"segtax":600,"segclass":"2206021246"},"segment":[{"id":"243"},{"id":"265"}],"name":"ads.pubmatic.com"},"lastUpdated":1659719242027}]]'; + storage.setDataInLocalStorage(topicStorageName, storedSegments); + gdprDataHdlrStub.returns(consentConfig); + assert.deepEqual(getCachedTopics(), []); + }); + + it('should stored segments if receiveMessage event is triggerred with segment data', () => { + return processFpd({}, {global: {}}, {data: Promise.resolve(mockData)}) + .then(({global}) => { + receiveMessage(evt) + let segments = new Map(safeJSONParse(storage.getDataFromLocalStorage(topicStorageName))); + expect(segments.has('pubmatic')).to.equal(true); + }); + }); + + it('should update stored segments if receiveMessage event is triggerred with segment data', () => { + let storedSegments = '[["pubmatic",{"2206021246":{"ext":{"segtax":600,"segclass":"2206021246"},"segment":[{"id":"243"},{"id":"265"}],"name":"ads.pubmatic.com"},"lastUpdated":1669719242027}]]'; + storage.setDataInLocalStorage(topicStorageName, storedSegments); + return processFpd({}, {global: {}}, {data: Promise.resolve(mockData)}) + .then(({global}) => { + receiveMessage(evt); + let segments = new Map(safeJSONParse(storage.getDataFromLocalStorage(topicStorageName))); + expect(segments.get('pubmatic')[2206021246].segment.length).to.equal(1); + }); + }); +}) From 033cf1a1d6dfa5824a1adef6f8e850764af3b52e Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Fri, 2 Dec 2022 17:51:28 +0530 Subject: [PATCH 12/13] Topics Module: Array.find used instead of array.some and variable name changed --- modules/topicsFpdModule.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index 4c35b5ce541..1c2cd1cc5bf 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -128,9 +128,9 @@ export function getCachedTopics() { let storedSegments = new Map(safeJSONParse(coreStorage.getDataFromLocalStorage(topicStorageName))); storedSegments && storedSegments.forEach((value, cachedBidder) => { // Check bidder exist in config for cached bidder data and then only retrieve the cached data - let isBidderConfigured = bidderList.some(({bidder}) => cachedBidder == bidder) - if (isBidderConfigured) { - if (!isCachedDataExpired(value[lastUpdated], isBidderConfigured?.expiry || DEFAULT_EXPIRATION_DAYS)) { + let bidderConfigObj = bidderList.find(({bidder}) => cachedBidder == bidder) + if (bidderConfigObj) { + if (!isCachedDataExpired(value[lastUpdated], bidderConfigObj?.expiry || DEFAULT_EXPIRATION_DAYS)) { Object.keys(value).forEach((segData) => { segData != lastUpdated && cachedTopicData.push(value[segData]); }) From 8f5404932fc41ecc12b59a6b45086951b38c3f08 Mon Sep 17 00:00:00 2001 From: Nitin Nimbalkar Date: Wed, 14 Dec 2022 12:22:15 +0530 Subject: [PATCH 13/13] Topics IFrame Implementation: Purpose present check is handled --- modules/topicsFpdModule.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/topicsFpdModule.js b/modules/topicsFpdModule.js index 1c2cd1cc5bf..f5a3c0a2c70 100644 --- a/modules/topicsFpdModule.js +++ b/modules/topicsFpdModule.js @@ -200,7 +200,7 @@ function listenMessagesFromTopicIframe() { function checkTCFv2(vendorData, requiredPurposes = TCF_REQUIRED_PURPOSES) { const {gdprApplies, purpose} = vendorData; - if (!gdprApplies) { + if (!gdprApplies || !purpose) { return true; } return requiredPurposes.map((purposeNo) => {