From 6f47071b52f2a65fa717a6d606c721ccdcd15dce Mon Sep 17 00:00:00 2001 From: Demetrio Girardi Date: Thu, 27 Jul 2023 12:59:38 -0700 Subject: [PATCH] userId: check any consent (not just GDPR) for changes when deciding if a stored ID needs to be refreshed --- modules/userId/index.js | 135 ++++++--------------- src/consentHandler.js | 7 +- test/spec/modules/id5IdSystem_spec.js | 4 +- test/spec/modules/userId_spec.js | 58 ++++----- test/spec/unit/core/consentHandler_spec.js | 14 ++- 5 files changed, 82 insertions(+), 136 deletions(-) diff --git a/modules/userId/index.js b/modules/userId/index.js index 784ad7a7db1..b20b38f0e40 100644 --- a/modules/userId/index.js +++ b/modules/userId/index.js @@ -130,7 +130,7 @@ import {find, includes} from '../../src/polyfill.js'; import {config} from '../../src/config.js'; import * as events from '../../src/events.js'; import {getGlobal} from '../../src/prebidGlobal.js'; -import adapterManager, {gdprDataHandler, gppDataHandler} from '../../src/adapterManager.js'; +import adapterManager, {gdprDataHandler} from '../../src/adapterManager.js'; import CONSTANTS from '../../src/constants.json'; import {module, ready as hooksReady} from '../../src/hook.js'; import {buildEidPermissions, createEidsArray, EID_CONFIG} from './eids.js'; @@ -141,7 +141,6 @@ import { STORAGE_TYPE_LOCALSTORAGE } from '../../src/storageManager.js'; import { - cyrb53Hash, deepAccess, deepSetValue, delayExecution, @@ -162,7 +161,7 @@ import {defer, GreedyPromise} from '../../src/utils/promise.js'; import {registerOrtbProcessor, REQUEST} from '../../src/pbjsORTB.js'; import {newMetrics, timedAuctionHook, useMetrics} from '../../src/utils/perfMetrics.js'; import {findRootDomain} from '../../src/fpd/rootDomain.js'; -import {GDPR_GVLIDS} from '../../src/consentHandler.js'; +import {allConsent, GDPR_GVLIDS} from '../../src/consentHandler.js'; import {MODULE_TYPE_UID} from '../../src/activities/modules.js'; import {isActivityAllowed} from '../../src/activities/rules.js'; import {ACTIVITY_ENRICH_EIDS} from '../../src/activities/activities.js'; @@ -173,10 +172,6 @@ const COOKIE = STORAGE_TYPE_COOKIES; const LOCAL_STORAGE = STORAGE_TYPE_LOCALSTORAGE; const DEFAULT_SYNC_DELAY = 500; const NO_AUCTION_DELAY = 0; -const CONSENT_DATA_COOKIE_STORAGE_CONFIG = { - name: '_pbjs_userid_consent_data', - expires: 30 // 30 days expiration, which should match how often consent is refreshed by CMPs -}; export const PBJS_USER_ID_OPTOUT_NAME = '_pbjs_id_optout'; export const coreStorage = getCoreStorageManager('userId'); export const dep = { @@ -261,11 +256,13 @@ export function setStoredValue(submodule, value) { if (storage.type === COOKIE) { const setCookie = cookieSetter(submodule); setCookie(null, valueStr, expiresStr); + setCookie('_cst', getConsentHash(), expiresStr); if (typeof storage.refreshInSeconds === 'number') { setCookie('_last', new Date().toUTCString(), expiresStr); } } else if (storage.type === LOCAL_STORAGE) { mgr.setDataInLocalStorage(`${storage.name}_exp`, expiresStr); + mgr.setDataInLocalStorage(`${storage.name}_cst`, getConsentHash()); mgr.setDataInLocalStorage(storage.name, encodeURIComponent(valueStr)); if (typeof storage.refreshInSeconds === 'number') { mgr.setDataInLocalStorage(`${storage.name}_last`, new Date().toUTCString()); @@ -283,11 +280,11 @@ export function deleteStoredValue(submodule) { const setCookie = cookieSetter(submodule, coreStorage); const expiry = (new Date(Date.now() - 1000 * 60 * 60 * 24)).toUTCString(); deleter = (suffix) => setCookie(suffix, '', expiry) - suffixes = ['', '_last']; + suffixes = ['', '_last', '_cst']; break; case LOCAL_STORAGE: deleter = (suffix) => coreStorage.removeDataFromLocalStorage(submodule.config.storage.name + suffix) - suffixes = ['', '_last', '_exp']; + suffixes = ['', '_last', '_exp', '_cst']; break; } if (deleter) { @@ -342,70 +339,6 @@ function getStoredValue(submodule, key = undefined) { return storedValue; } -/** - * makes an object that can be stored with only the keys we need to check. - * excluding the vendorConsents object since the consentString is enough to know - * if consent has changed without needing to have all the details in an object - * @param consentData - * @returns {{apiVersion: number, gdprApplies: boolean, consentString: string}} - */ -function makeStoredConsentDataHash(consentData) { - const storedConsentData = { - consentString: '', - gdprApplies: false, - apiVersion: 0 - }; - - if (consentData) { - storedConsentData.consentString = consentData.consentString; - storedConsentData.gdprApplies = consentData.gdprApplies; - storedConsentData.apiVersion = consentData.apiVersion; - } - - return cyrb53Hash(JSON.stringify(storedConsentData)); -} - -/** - * puts the current consent data into cookie storage - * @param consentData - */ -export function setStoredConsentData(consentData) { - try { - const expiresStr = (new Date(Date.now() + (CONSENT_DATA_COOKIE_STORAGE_CONFIG.expires * (60 * 60 * 24 * 1000)))).toUTCString(); - coreStorage.setCookie(CONSENT_DATA_COOKIE_STORAGE_CONFIG.name, makeStoredConsentDataHash(consentData), expiresStr, 'Lax'); - } catch (error) { - logError(error); - } -} - -/** - * get the stored consent data from local storage, if any - * @returns {string} - */ -function getStoredConsentData() { - try { - return coreStorage.getCookie(CONSENT_DATA_COOKIE_STORAGE_CONFIG.name); - } catch (e) { - logError(e); - } -} - -/** - * test if the consent object stored locally matches the current consent data. if they - * don't match or there is nothing stored locally, it means a refresh of the user id - * submodule is needed - * @param storedConsentData - * @param consentData - * @returns {boolean} - */ -function storedConsentDataMatchesConsentData(storedConsentData, consentData) { - return ( - typeof storedConsentData !== 'undefined' && - storedConsentData !== null && - storedConsentData === makeStoredConsentDataHash(consentData) - ); -} - /** * @param {SubmoduleContainer[]} submodules * @param {function} cb - callback for after processing is done. @@ -573,18 +506,15 @@ function idSystemInitializer({delay = GreedyPromise.timeout} = {}) { } } - function timeGdpr() { - return gdprDataHandler.promise.finally(initMetrics.startTiming('userId.init.gdpr')); - } - function timeGpp() { - return gppDataHandler.promise.finally(initMetrics.startTiming('userId.init.gpp')) + function timeConsent() { + return allConsent.promise.finally(initMetrics.startTiming('userId.init.consent')) } let done = cancelAndTry( GreedyPromise.all([hooksReady, startInit.promise]) - .then(() => GreedyPromise.all([timeGdpr(), timeGpp()]).then(([gdpr]) => gdpr)) - .then(checkRefs((consentData) => { - initSubmodules(initModules, allModules, consentData); + .then(timeConsent) + .then(checkRefs(() => { + initSubmodules(initModules, allModules); })) .then(() => startCallbacks.promise.finally(initMetrics.startTiming('userId.callbacks.pending'))) .then(checkRefs(() => { @@ -618,12 +548,11 @@ function idSystemInitializer({delay = GreedyPromise.timeout} = {}) { done = cancelAndTry( done .catch(() => null) - .then(timeGdpr) // fetch again in case a refresh was forced before this was resolved - .then(checkRefs((consentData) => { + .then(timeConsent) // fetch again in case a refresh was forced before this was resolved + .then(checkRefs(() => { const cbModules = initSubmodules( initModules, allModules.filter((sm) => submoduleNames == null || submoduleNames.includes(sm.submodule.name)), - consentData, true ).filter((sm) => { return sm.callback != null; @@ -812,7 +741,27 @@ function getUserIdsAsync() { ); } -function populateSubmoduleId(submodule, consentData, storedConsentData, forceRefresh, allSubmodules) { +export function getConsentHash() { + // transform decimal string into base64 to save some space on cookies + let hash = Number(allConsent.hash); + const bytes = []; + while (hash > 0) { + bytes.push(String.fromCharCode(hash & 255)); + hash = hash >>> 8; + } + return btoa(bytes.join()); +} + +function consentChanged(submodule) { + const storedConsent = getStoredValue(submodule, 'cst'); + return !storedConsent || storedConsent !== getConsentHash(); +} + +function populateSubmoduleId(submodule, forceRefresh, allSubmodules) { + // TODO: the ID submodule API only takes GDPR consent; it should be updated now that GDPR + // is only a tiny fraction of a vast consent universe + const gdprConsent = gdprDataHandler.getConsentData(); + // There are two submodule configuration types to handle: storage or value // 1. storage: retrieve user id data from cookie/html storage or with the submodule's getId method // 2. value: pass directly to bids @@ -826,12 +775,12 @@ function populateSubmoduleId(submodule, consentData, storedConsentData, forceRef refreshNeeded = storedDate && (Date.now() - storedDate.getTime() > submodule.config.storage.refreshInSeconds * 1000); } - if (!storedId || refreshNeeded || forceRefresh || !storedConsentDataMatchesConsentData(storedConsentData, consentData)) { + if (!storedId || refreshNeeded || forceRefresh || consentChanged(submodule)) { // No id previously saved, or a refresh is needed, or consent has changed. Request a new id from the submodule. - response = submodule.submodule.getId(submodule.config, consentData, storedId); + response = submodule.submodule.getId(submodule.config, gdprConsent, storedId); } else if (typeof submodule.submodule.extendId === 'function') { // If the id exists already, give submodule a chance to decide additional actions that need to be taken - response = submodule.submodule.extendId(submodule.config, consentData, storedId); + response = submodule.submodule.extendId(submodule.config, gdprConsent, storedId); } if (isPlainObject(response)) { @@ -855,7 +804,7 @@ function populateSubmoduleId(submodule, consentData, storedConsentData, forceRef // cache decoded value (this is copied to every adUnit bid) submodule.idObj = submodule.config.value; } else { - const response = submodule.submodule.getId(submodule.config, consentData, undefined); + const response = submodule.submodule.getId(submodule.config, gdprConsent, undefined); if (isPlainObject(response)) { if (typeof response.callback === 'function') { submodule.callback = response.callback; } if (response.id) { submodule.idObj = submodule.submodule.decode(response.id, submodule.config); } @@ -881,7 +830,7 @@ function updatePPID(userIds = getUserIds()) { } } -function initSubmodules(dest, submodules, consentData, forceRefresh = false) { +function initSubmodules(dest, submodules, forceRefresh = false) { return uidMetrics().fork().measureTime('userId.init.modules', function () { if (!submodules.length) return []; // to simplify log messages from here on @@ -901,14 +850,10 @@ function initSubmodules(dest, submodules, consentData, forceRefresh = false) { return []; } - // we always want the latest consentData stored, even if we don't execute any submodules - const storedConsentData = getStoredConsentData(); - setStoredConsentData(consentData); - const initialized = submodules.reduce((carry, submodule) => { return submoduleMetrics(submodule.submodule.name).measureTime('init', () => { try { - populateSubmoduleId(submodule, consentData, storedConsentData, forceRefresh, submodules); + populateSubmoduleId(submodule, forceRefresh, submodules); carry.push(submodule); } catch (e) { logError(`Error in userID module '${submodule.submodule.name}':`, e); diff --git a/src/consentHandler.js b/src/consentHandler.js index 2fd8f0baf35..9e3ee5b4c40 100644 --- a/src/consentHandler.js +++ b/src/consentHandler.js @@ -185,6 +185,7 @@ export const coppaDataHandler = (() => { getCoppa, getConsentData: getCoppa, getConsentMeta: getCoppa, + reset() {}, get promise() { return GreedyPromise.resolve(getCoppa()) }, @@ -213,14 +214,14 @@ export function multiHandler(handlers = ALL_HANDLERS) { return Object.assign( { get promise() { - return Promise.all(handlers.map(([name, handler]) => handler.promise.then(val => [name, val]))) + return GreedyPromise.all(handlers.map(([name, handler]) => handler.promise.then(val => [name, val]))) .then(entries => Object.fromEntries(entries)); }, get hash() { - return handlers.map(([_, handler]) => handler.hash).join(':'); + return cyrb53Hash(handlers.map(([_, handler]) => handler.hash).join(':')); } }, - Object.fromEntries(['getConsentData', 'getConsentMeta'].map(n => [n, collector(n)])), + Object.fromEntries(['getConsentData', 'getConsentMeta', 'reset'].map(n => [n, collector(n)])), ) } diff --git a/test/spec/modules/id5IdSystem_spec.js b/test/spec/modules/id5IdSystem_spec.js index 2d81c9b7b8d..b8d4554c03e 100644 --- a/test/spec/modules/id5IdSystem_spec.js +++ b/test/spec/modules/id5IdSystem_spec.js @@ -10,7 +10,7 @@ import { storeInLocalStorage, storeNbInCache, } from 'modules/id5IdSystem.js'; -import {coreStorage, init, requestBidsHook, setSubmoduleRegistry} from 'modules/userId/index.js'; +import {coreStorage, getConsentHash, init, requestBidsHook, setSubmoduleRegistry} from 'modules/userId/index.js'; import {config} from 'src/config.js'; import * as events from 'src/events.js'; import CONSTANTS from 'src/constants.json'; @@ -791,6 +791,7 @@ describe('ID5 ID System', function () { coreStorage.removeDataFromLocalStorage(ID5_STORAGE_NAME); coreStorage.removeDataFromLocalStorage(`${ID5_STORAGE_NAME}_last`); coreStorage.removeDataFromLocalStorage(ID5_NB_STORAGE_NAME); + coreStorage.setDataInLocalStorage(ID5_STORAGE_NAME + '_cst', getConsentHash()) adUnits = [getAdUnitMock()]; }); afterEach(function () { @@ -798,6 +799,7 @@ describe('ID5 ID System', function () { coreStorage.removeDataFromLocalStorage(ID5_STORAGE_NAME); coreStorage.removeDataFromLocalStorage(`${ID5_STORAGE_NAME}_last`); coreStorage.removeDataFromLocalStorage(ID5_NB_STORAGE_NAME); + coreStorage.removeDataFromLocalStorage(ID5_STORAGE_NAME + '_cst') sandbox.restore(); }); diff --git a/test/spec/modules/userId_spec.js b/test/spec/modules/userId_spec.js index acc016a903d..2cab8f9df64 100644 --- a/test/spec/modules/userId_spec.js +++ b/test/spec/modules/userId_spec.js @@ -2,7 +2,7 @@ import { attachIdSystem, auctionDelay, coreStorage, dep, - findRootDomain, + findRootDomain, getConsentHash, init, PBJS_USER_ID_OPTOUT_NAME, requestBidsHook, @@ -21,7 +21,6 @@ import CONSTANTS from 'src/constants.json'; import {getGlobal} from 'src/prebidGlobal.js'; import {resetConsentData, } from 'modules/consentManagement.js'; import {server} from 'test/mocks/xhr.js'; -import {find} from 'src/polyfill.js'; import {unifiedIdSubmodule} from 'modules/unifiedIdSystem.js'; import {britepoolIdSubmodule} from 'modules/britepoolIdSystem.js'; import {id5IdSubmodule} from 'modules/id5IdSystem.js'; @@ -56,7 +55,7 @@ import {hook} from '../../../src/hook.js'; import {mockGdprConsent} from '../../helpers/consentData.js'; import {getPPID} from '../../../src/adserver.js'; import {uninstall as uninstallGdprEnforcement} from 'modules/gdprEnforcement.js'; -import {GDPR_GVLIDS} from '../../../src/consentHandler.js'; +import {allConsent, GDPR_GVLIDS, gdprDataHandler} from '../../../src/consentHandler.js'; import {MODULE_TYPE_UID} from '../../../src/activities/modules.js'; import {ACTIVITY_ENRICH_EIDS} from '../../../src/activities/activities.js'; import {ACTIVITY_PARAM_COMPONENT_NAME, ACTIVITY_PARAM_COMPONENT_TYPE} from '../../../src/activities/params.js'; @@ -380,8 +379,6 @@ describe('User ID', function () { expect(bid).to.not.have.deep.nested.property('userIdAsEids'); }); }); - // setCookie is called once in order to store consentData - expect(coreStorage.setCookie.callCount).to.equal(1); }); }); @@ -929,6 +926,7 @@ describe('User ID', function () { beforeEach(() => { mockIdCallback = sinon.stub(); + coreStorage.setCookie('MOCKID', '', EXPIRED_COOKIE_DATE); let mockIdSystem = { name: 'mockId', decode: function(value) { @@ -1155,7 +1153,7 @@ describe('User ID', function () { it('pbjs.refreshUserIds refreshes single', function() { coreStorage.setCookie('MOCKID', '', EXPIRED_COOKIE_DATE); - coreStorage.setCookie('REFRESH', '', EXPIRED_COOKIE_DATE); + coreStorage.setCookie('refreshedid', '', EXPIRED_COOKIE_DATE); let sandbox = sinon.createSandbox(); let mockIdCallback = sandbox.stub().returns({id: {'MOCKID': '1111'}}); @@ -3119,17 +3117,12 @@ describe('User ID', function () { } }; - consentData = { - gdprApplies: true, - consentString: 'mockString', - apiVersion: 1, - hasValidated: true // mock presence of GPDR enforcement module - } // clear cookies expStr = (new Date(Date.now() + 25000).toUTCString()); coreStorage.setCookie(mockIdCookieName, '', EXPIRED_COOKIE_DATE); coreStorage.setCookie(`${mockIdCookieName}_last`, '', EXPIRED_COOKIE_DATE); - coreStorage.setCookie(CONSENT_LOCAL_STORAGE_NAME, '', EXPIRED_COOKIE_DATE); + coreStorage.setCookie(`${mockIdCookieName}_cst`, '', EXPIRED_COOKIE_DATE); + allConsent.reset(); // init adUnits = [getAdUnitMock()]; @@ -3143,10 +3136,20 @@ describe('User ID', function () { config.resetConfig(); }); - it('calls getId if no stored consent data and refresh is not needed', function () { - coreStorage.setCookie(mockIdCookieName, JSON.stringify({id: '1234'}), expStr); - coreStorage.setCookie(`${mockIdCookieName}_last`, (new Date(Date.now() - 1 * 1000).toUTCString()), expStr); + function setStorage({ + val = JSON.stringify({id: '1234'}), + lastDelta = 60 * 1000, + cst = null + } = {}) { + coreStorage.setCookie(mockIdCookieName, val, expStr); + coreStorage.setCookie(`${mockIdCookieName}_last`, (new Date(Date.now() - lastDelta).toUTCString()), expStr); + if (cst != null) { + coreStorage.setCookie(`${mockIdCookieName}_cst`, cst, expStr); + } + } + it('calls getId if no stored consent data and refresh is not needed', function () { + setStorage({lastDelta: 1000}); config.setConfig(userIdConfig); let innerAdUnits; @@ -3160,9 +3163,7 @@ describe('User ID', function () { }); it('calls getId if no stored consent data but refresh is needed', function () { - coreStorage.setCookie(mockIdCookieName, JSON.stringify({id: '1234'}), expStr); - coreStorage.setCookie(`${mockIdCookieName}_last`, (new Date(Date.now() - 60 * 1000).toUTCString()), expStr); - + setStorage(); config.setConfig(userIdConfig); let innerAdUnits; @@ -3176,11 +3177,7 @@ describe('User ID', function () { }); it('calls getId if empty stored consent and refresh not needed', function () { - coreStorage.setCookie(mockIdCookieName, JSON.stringify({id: '1234'}), expStr); - coreStorage.setCookie(`${mockIdCookieName}_last`, (new Date(Date.now() - 1 * 1000).toUTCString()), expStr); - - setStoredConsentData(); - + setStorage({cst: ''}); config.setConfig(userIdConfig); let innerAdUnits; @@ -3194,10 +3191,10 @@ describe('User ID', function () { }); it('calls getId if stored consent does not match current consent and refresh not needed', function () { - coreStorage.setCookie(mockIdCookieName, JSON.stringify({id: '1234'}), expStr); - coreStorage.setCookie(`${mockIdCookieName}_last`, (new Date(Date.now() - 1 * 1000).toUTCString()), expStr); - - setStoredConsentData({...consentData, consentString: 'different'}); + setStorage({cst: getConsentHash()}); + gdprDataHandler.setConsentData({ + consentString: 'different' + }); config.setConfig(userIdConfig); @@ -3212,10 +3209,7 @@ describe('User ID', function () { }); it('does not call getId if stored consent matches current consent and refresh not needed', function () { - coreStorage.setCookie(mockIdCookieName, JSON.stringify({id: '1234'}), expStr); - coreStorage.setCookie(`${mockIdCookieName}_last`, (new Date(Date.now() - 1 * 1000).toUTCString()), expStr); - - setStoredConsentData({...consentData}); + setStorage({lastDelta: 1000, cst: getConsentHash()}); config.setConfig(userIdConfig); diff --git a/test/spec/unit/core/consentHandler_spec.js b/test/spec/unit/core/consentHandler_spec.js index a8e9a9662df..1bcad3216ce 100644 --- a/test/spec/unit/core/consentHandler_spec.js +++ b/test/spec/unit/core/consentHandler_spec.js @@ -1,4 +1,4 @@ -import {multiHandler, ConsentHandler, gvlidRegistry} from '../../../../src/consentHandler.js'; +import {ConsentHandler, gvlidRegistry, multiHandler} from '../../../../src/consentHandler.js'; describe('Consent data handler', () => { let handler; @@ -128,10 +128,14 @@ describe('multiHandler', () => { }); describe('.hash', () => { - it('concats underlying hashses', () => { - handlers.h1.hash = 'one'; - handlers.h2.hash = 'two'; - expect(multi.hash).to.eql('one:two'); + ['h1', 'h2'].forEach((handler, i) => { + it(`changes when handler #${i + 1} changes hash`, () => { + handlers.h1.hash = 'one'; + handlers.h2.hash = 'two' + const first = multi.hash; + handlers[handler].hash = 'new'; + expect(multi.hash).to.not.eql(first); + }) }) }) })