From 50adafd7e6799e5f0e4581eab6402b0de28749f2 Mon Sep 17 00:00:00 2001 From: Scott Menzer Date: Thu, 20 Aug 2020 12:21:31 +0200 Subject: [PATCH 1/5] updated userid module to stop caching the entire consent object but rather just a hash of it, since all we need it for is comparison purposes. --- modules/userId/index.js | 12 +++++------ src/utils.js | 20 ++++++++++++++++++ test/spec/modules/userId_spec.js | 35 -------------------------------- test/spec/utils_spec.js | 16 +++++++++++++++ 4 files changed, 41 insertions(+), 42 deletions(-) diff --git a/modules/userId/index.js b/modules/userId/index.js index afdd93a57ba..26cef3b4d96 100644 --- a/modules/userId/index.js +++ b/modules/userId/index.js @@ -232,7 +232,7 @@ function getStoredValue(storage, key = undefined) { * @param consentData * @returns {{apiVersion: number, gdprApplies: boolean, consentString: string}} */ -function makeStoredConsentDataObject(consentData) { +function makeStoredConsentDataHash(consentData) { const storedConsentData = { consentString: '', gdprApplies: false, @@ -245,7 +245,7 @@ function makeStoredConsentDataObject(consentData) { storedConsentData.apiVersion = consentData.apiVersion; } - return storedConsentData; + return utils.simpleHash(JSON.stringify(storedConsentData)); } /** @@ -255,7 +255,7 @@ function makeStoredConsentDataObject(consentData) { export function setStoredConsentData(consentData) { try { const expiresStr = (new Date(Date.now() + (CONSENT_DATA_COOKIE_STORAGE_CONFIG.expires * (60 * 60 * 24 * 1000)))).toUTCString(); - coreStorage.setCookie(CONSENT_DATA_COOKIE_STORAGE_CONFIG.name, JSON.stringify(makeStoredConsentDataObject(consentData)), expiresStr, 'Lax'); + coreStorage.setCookie(CONSENT_DATA_COOKIE_STORAGE_CONFIG.name, makeStoredConsentDataHash(consentData), expiresStr, 'Lax'); } catch (error) { utils.logError(error); } @@ -266,13 +266,11 @@ export function setStoredConsentData(consentData) { * @returns {string} */ function getStoredConsentData() { - let storedValue; try { - storedValue = JSON.parse(coreStorage.getCookie(CONSENT_DATA_COOKIE_STORAGE_CONFIG.name)); + return coreStorage.getCookie(CONSENT_DATA_COOKIE_STORAGE_CONFIG.name); } catch (e) { utils.logError(e); } - return storedValue; } /** @@ -287,7 +285,7 @@ function storedConsentDataMatchesConsentData(storedConsentData, consentData) { return ( typeof storedConsentData === 'undefined' || storedConsentData === null || - utils.deepEqual(storedConsentData, makeStoredConsentDataObject(consentData)) + storedConsentData == makeStoredConsentDataHash(consentData) ); } diff --git a/src/utils.js b/src/utils.js index f0fa57c4cff..8a3c444a862 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1215,3 +1215,23 @@ export function mergeDeep(target, ...sources) { return mergeDeep(target, ...sources); } + +/** + * returns a hash of a string using a fast algorithm + * source: https://stackoverflow.com/a/52171480/845390 + * @param str + * @param seed (optional) + * @returns {number} + */ +export function simpleHash(str, seed = 0) { + let h1 = 0xdeadbeef ^ seed; + let h2 = 0x41c6ce57 ^ seed; + for (let i = 0, ch; i < str.length; i++) { + ch = str.charCodeAt(i); + h1 = Math.imul(h1 ^ ch, 2654435761); + h2 = Math.imul(h2 ^ ch, 1597334677); + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909); + return 4294967296 * (2097151 & h2) + (h1 >>> 0); +} diff --git a/test/spec/modules/userId_spec.js b/test/spec/modules/userId_spec.js index d9671aabc84..fd7e8a76972 100644 --- a/test/spec/modules/userId_spec.js +++ b/test/spec/modules/userId_spec.js @@ -1591,13 +1591,6 @@ describe('User ID', function() { sinon.assert.notCalled(mockGetId); sinon.assert.calledOnce(mockDecode); sinon.assert.calledOnce(mockExtendId); - - let consent = gdprDataHandler.getConsentData(); - let userIdStoredConsent = JSON.parse(coreStorage.getCookie(CONSENT_LOCAL_STORAGE_NAME)); - expect(userIdStoredConsent.gdprApplies).to.equal(consent.gdprApplies); - expect(userIdStoredConsent.gdprApplies).to.equal(testConsentData.gdprApplies); - expect(userIdStoredConsent.consentString).to.equal(consent.consentString); - expect(userIdStoredConsent.consentString).to.equal(testConsentData.consentData); }); it('calls getId if no stored consent data but refresh is needed', function () { @@ -1611,13 +1604,6 @@ describe('User ID', function() { sinon.assert.calledOnce(mockGetId); sinon.assert.calledOnce(mockDecode); sinon.assert.notCalled(mockExtendId); - - let consent = gdprDataHandler.getConsentData(); - let userIdStoredConsent = JSON.parse(coreStorage.getCookie(CONSENT_LOCAL_STORAGE_NAME)); - expect(userIdStoredConsent.gdprApplies).to.equal(consent.gdprApplies); - expect(userIdStoredConsent.gdprApplies).to.equal(testConsentData.gdprApplies); - expect(userIdStoredConsent.consentString).to.equal(consent.consentString); - expect(userIdStoredConsent.consentString).to.equal(testConsentData.consentData); }); it('calls getId if empty stored consent and refresh not needed', function () { @@ -1633,13 +1619,6 @@ describe('User ID', function() { sinon.assert.calledOnce(mockGetId); sinon.assert.calledOnce(mockDecode); sinon.assert.notCalled(mockExtendId); - - let consent = gdprDataHandler.getConsentData(); - let userIdStoredConsent = JSON.parse(coreStorage.getCookie(CONSENT_LOCAL_STORAGE_NAME)); - expect(userIdStoredConsent.gdprApplies).to.equal(consent.gdprApplies); - expect(userIdStoredConsent.gdprApplies).to.equal(testConsentData.gdprApplies); - expect(userIdStoredConsent.consentString).to.equal(consent.consentString); - expect(userIdStoredConsent.consentString).to.equal(testConsentData.consentData); }); it('calls getId if stored consent does not match current consent and refresh not needed', function () { @@ -1659,13 +1638,6 @@ describe('User ID', function() { sinon.assert.calledOnce(mockGetId); sinon.assert.calledOnce(mockDecode); sinon.assert.notCalled(mockExtendId); - - let consent = gdprDataHandler.getConsentData(); - let userIdStoredConsent = JSON.parse(coreStorage.getCookie(CONSENT_LOCAL_STORAGE_NAME)); - expect(userIdStoredConsent.gdprApplies).to.equal(consent.gdprApplies); - expect(userIdStoredConsent.gdprApplies).to.equal(testConsentData.gdprApplies); - expect(userIdStoredConsent.consentString).to.equal(consent.consentString); - expect(userIdStoredConsent.consentString).to.equal(testConsentData.consentData); }); it('does not call getId if stored consent matches current consent and refresh not needed', function () { @@ -1685,13 +1657,6 @@ describe('User ID', function() { sinon.assert.notCalled(mockGetId); sinon.assert.calledOnce(mockDecode); sinon.assert.calledOnce(mockExtendId); - - let consent = gdprDataHandler.getConsentData(); - let userIdStoredConsent = JSON.parse(coreStorage.getCookie(CONSENT_LOCAL_STORAGE_NAME)); - expect(userIdStoredConsent.gdprApplies).to.equal(consent.gdprApplies); - expect(userIdStoredConsent.gdprApplies).to.equal(testConsentData.gdprApplies); - expect(userIdStoredConsent.consentString).to.equal(consent.consentString); - expect(userIdStoredConsent.consentString).to.equal(testConsentData.consentData); }); }); }); diff --git a/test/spec/utils_spec.js b/test/spec/utils_spec.js index fca59633ebe..387561340f7 100644 --- a/test/spec/utils_spec.js +++ b/test/spec/utils_spec.js @@ -1177,5 +1177,21 @@ describe('Utils', function () { } expect(utils.deepEqual(obj1, obj2)).to.equal(false); }); + + describe('simpleHash', function() { + it('should return the same hash for the same string', function() { + const stringOne = 'string1'; + expect(utils.simpleHash(stringOne)).to.equal(utils.simpleHash(stringOne)); + }); + it('should return a different hash for the same string with different seeds', function() { + const stringOne = 'string1'; + expect(utils.simpleHash(stringOne, 1)).to.not.equal(utils.simpleHash(stringOne, 2)); + }); + it('should return a different hash for different strings with the same seed', function() { + const stringOne = 'string1'; + const stringTwo = 'string2'; + expect(utils.simpleHash(stringOne)).to.not.equal(utils.simpleHash(stringTwo)); + }); + }); }); }); From 1cd1301d10852c606bf0aba73045f08a684ec70a Mon Sep 17 00:00:00 2001 From: Scott Menzer Date: Thu, 20 Aug 2020 12:55:26 +0200 Subject: [PATCH 2/5] IE doesn't support Math.imul, so providing a polyfill for it when necessary --- src/utils.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/utils.js b/src/utils.js index 8a3c444a862..0b5769f0d81 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1224,6 +1224,24 @@ export function mergeDeep(target, ...sources) { * @returns {number} */ export function simpleHash(str, seed = 0) { + // IE doesn't support imul + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/imul#Polyfill + if (!Math.imul) { + Math.imul = function(opA, opB) { + opB |= 0; // ensure that opB is an integer. opA will automatically be coerced. + // floating points give us 53 bits of precision to work with plus 1 sign bit + // automatically handled for our convienence: + // 1. 0x003fffff /*opA & 0x000fffff*/ * 0x7fffffff /*opB*/ = 0x1fffff7fc00001 + // 0x1fffff7fc00001 < Number.MAX_SAFE_INTEGER /*0x1fffffffffffff*/ + var result = (opA & 0x003fffff) * opB; + // 2. We can remove an integer coersion from the statement above because: + // 0x1fffff7fc00001 + 0xffc00000 = 0x1fffffff800001 + // 0x1fffffff800001 < Number.MAX_SAFE_INTEGER /*0x1fffffffffffff*/ + if (opA & 0xffc00000) result += (opA & 0xffc00000) * opB | 0; + return result | 0; + }; + } + let h1 = 0xdeadbeef ^ seed; let h2 = 0x41c6ce57 ^ seed; for (let i = 0, ch; i < str.length; i++) { From 9b3f0027a0aab665616560987ad57b28187a8fe0 Mon Sep 17 00:00:00 2001 From: Scott Menzer Date: Thu, 20 Aug 2020 15:54:29 +0200 Subject: [PATCH 3/5] use `===` to compare consent values; convert hashes to a string when returning them --- modules/userId/index.js | 3 +-- src/utils.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/userId/index.js b/modules/userId/index.js index 26cef3b4d96..bcc0f68b2b0 100644 --- a/modules/userId/index.js +++ b/modules/userId/index.js @@ -244,7 +244,6 @@ function makeStoredConsentDataHash(consentData) { storedConsentData.gdprApplies = consentData.gdprApplies; storedConsentData.apiVersion = consentData.apiVersion; } - return utils.simpleHash(JSON.stringify(storedConsentData)); } @@ -285,7 +284,7 @@ function storedConsentDataMatchesConsentData(storedConsentData, consentData) { return ( typeof storedConsentData === 'undefined' || storedConsentData === null || - storedConsentData == makeStoredConsentDataHash(consentData) + storedConsentData === makeStoredConsentDataHash(consentData) ); } diff --git a/src/utils.js b/src/utils.js index 0b5769f0d81..71631611644 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1251,5 +1251,5 @@ export function simpleHash(str, seed = 0) { } h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909); h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909); - return 4294967296 * (2097151 & h2) + (h1 >>> 0); + return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(); } From 3d099233ecdc3e3108116d5eb9a723676c029c23 Mon Sep 17 00:00:00 2001 From: Scott Menzer Date: Thu, 20 Aug 2020 16:00:41 +0200 Subject: [PATCH 4/5] add test for string response fix @returns doc --- src/utils.js | 2 +- test/spec/utils_spec.js | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils.js b/src/utils.js index 71631611644..7925dc75999 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1221,7 +1221,7 @@ export function mergeDeep(target, ...sources) { * source: https://stackoverflow.com/a/52171480/845390 * @param str * @param seed (optional) - * @returns {number} + * @returns {string} */ export function simpleHash(str, seed = 0) { // IE doesn't support imul diff --git a/test/spec/utils_spec.js b/test/spec/utils_spec.js index 387561340f7..dbaad919bd3 100644 --- a/test/spec/utils_spec.js +++ b/test/spec/utils_spec.js @@ -1192,6 +1192,10 @@ describe('Utils', function () { const stringTwo = 'string2'; expect(utils.simpleHash(stringOne)).to.not.equal(utils.simpleHash(stringTwo)); }); + it('should return a string value, not a number', function() { + const stringOne = 'string1'; + expect(typeof utils.simpleHash(stringOne)).to.equal('string'); + }); }); }); }); From 44b5dee5b54e9643627236fb02351d22146d97c8 Mon Sep 17 00:00:00 2001 From: Scott Menzer Date: Thu, 20 Aug 2020 18:15:05 +0200 Subject: [PATCH 5/5] don't use polyfills! --- src/utils.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/utils.js b/src/utils.js index 7925dc75999..6801a7dc1d1 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1226,8 +1226,10 @@ export function mergeDeep(target, ...sources) { export function simpleHash(str, seed = 0) { // IE doesn't support imul // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/imul#Polyfill - if (!Math.imul) { - Math.imul = function(opA, opB) { + let imul = function(opA, opB) { + if (isFn(Math.imul)) { + return Math.imul(opA, opB); + } else { opB |= 0; // ensure that opB is an integer. opA will automatically be coerced. // floating points give us 53 bits of precision to work with plus 1 sign bit // automatically handled for our convienence: @@ -1239,17 +1241,17 @@ export function simpleHash(str, seed = 0) { // 0x1fffffff800001 < Number.MAX_SAFE_INTEGER /*0x1fffffffffffff*/ if (opA & 0xffc00000) result += (opA & 0xffc00000) * opB | 0; return result | 0; - }; - } + } + }; let h1 = 0xdeadbeef ^ seed; let h2 = 0x41c6ce57 ^ seed; for (let i = 0, ch; i < str.length; i++) { ch = str.charCodeAt(i); - h1 = Math.imul(h1 ^ ch, 2654435761); - h2 = Math.imul(h2 ^ ch, 1597334677); + h1 = imul(h1 ^ ch, 2654435761); + h2 = imul(h2 ^ ch, 1597334677); } - h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909); - h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909); + h1 = imul(h1 ^ (h1 >>> 16), 2246822507) ^ imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = imul(h2 ^ (h2 >>> 16), 2246822507) ^ imul(h1 ^ (h1 >>> 13), 3266489909); return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(); }