ID5 analytics adapter: initial release (#6871)

* Adding Markdown for the new id5 analytics module

* #24 Adding a first untested implementation

* #24 Adding some unit tess and refactoring

* #24 Adding cleanup transformations, improvements and tests

* #24 Improving on specs and implementation of cleanup

* #24 Adding standard tracking of bidWon and cleanup of native creative

* #24 More cleanup rules

* #24 Using real URL instad of mock

* #24 Typo

* #24 Code review improvements

Co-authored-by: Marco Cosentino <mcosentino@id5.io>
Co-authored-by: Scott Menzer <scott@id5.io>

modules/id5AnalyticsAdapter.js

@@ -0,0 +1,304 @@
+import buildAdapter from '../src/AnalyticsAdapter.js';
+import CONSTANTS from '../src/constants.json';
+import adapterManager from '../src/adapterManager.js';
+import { ajax } from '../src/ajax.js';
+import { logInfo, logError } from '../src/utils.js';
+import events from '../src/events.js';
+
+const {
+  EVENTS: {
+    AUCTION_END,
+    TCF2_ENFORCEMENT,
+    BID_WON,
+    BID_VIEWABLE,
+    AD_RENDER_FAILED
+  }
+} = CONSTANTS
+
+const GVLID = 131;
+
+const STANDARD_EVENTS_TO_TRACK = [
+  AUCTION_END,
+  TCF2_ENFORCEMENT,
+  BID_WON,
+];
+
+// These events cause the buffered events to be sent over
+const FLUSH_EVENTS = [
+  TCF2_ENFORCEMENT,
+  AUCTION_END,
+  BID_WON,
+  BID_VIEWABLE,
+  AD_RENDER_FAILED
+];
+
+const CONFIG_URL_PREFIX = 'https://api.id5-sync.com/analytics'
+const TZ = new Date().getTimezoneOffset();
+const PBJS_VERSION = $$PREBID_GLOBAL$$.version;
+const ID5_REDACTED = '__ID5_REDACTED__';
+const isArray = Array.isArray;
+
+let id5Analytics = Object.assign(buildAdapter({analyticsType: 'endpoint'}), {
+  // Keeps an array of events for each auction
+  eventBuffer: {},
+
+  eventsToTrack: STANDARD_EVENTS_TO_TRACK,
+
+  track: (event) => {
+    const _this = id5Analytics;
+
+    if (!event || !event.args) {
+      return;
+    }
+
+    try {
+      const auctionId = event.args.auctionId;
+      _this.eventBuffer[auctionId] = _this.eventBuffer[auctionId] || [];
+
+      // Collect events and send them in a batch when the auction ends
+      const que = _this.eventBuffer[auctionId];
+      que.push(_this.makeEvent(event.eventType, event.args));
+
+      if (FLUSH_EVENTS.indexOf(event.eventType) >= 0) {
+        // Auction ended. Send the batch of collected events
+        _this.sendEvents(que);
+
+        // From now on just send events to server side as they come
+        que.push = (pushedEvent) => _this.sendEvents([pushedEvent]);
+      }
+    } catch (error) {
+      logError('id5Analytics: ERROR', error);
+      _this.sendErrorEvent(error);
+    }
+  },
+
+  sendEvents: (eventsToSend) => {
+    const _this = id5Analytics;
+    // By giving some content this will be automatically a POST
+    eventsToSend.forEach((event) =>
+      ajax(_this.options.ingestUrl, null, JSON.stringify(event)));
+  },
+
+  makeEvent: (event, payload) => {
+    const _this = id5Analytics;
+    const filteredPayload = deepTransformingClone(payload,
+      transformFnFromCleanupRules(event));
+    return {
+      source: 'pbjs',
+      event,
+      payload: filteredPayload,
+      partnerId: _this.options.partnerId,
+      meta: {
+        sampling: _this.options.id5Sampling,
+        pbjs: PBJS_VERSION,
+        tz: TZ,
+      }
+    };
+  },
+
+  sendErrorEvent: (error) => {
+    const _this = id5Analytics;
+    _this.sendEvents([
+      _this.makeEvent('analyticsError', {
+        message: error.message,
+        stack: error.stack,
+      })
+    ]);
+  },
+
+  random: () => Math.random(),
+});
+
+const ENABLE_FUNCTION = (config) => {
+  const _this = id5Analytics;
+  _this.options = (config && config.options) || {};
+
+  const partnerId = _this.options.partnerId;
+  if (typeof partnerId !== 'number') {
+    logError('id5Analytics: partnerId in config.options must be a number representing the id5 partner ID');
+    return;
+  }
+
+  ajax(`${CONFIG_URL_PREFIX}/${partnerId}/pbjs`, (result) => {
+    logInfo('id5Analytics: Received from configuration endpoint', result);
+
+    const configFromServer = JSON.parse(result);
+
+    const sampling = _this.options.id5Sampling =
+      typeof configFromServer.sampling === 'number' ? configFromServer.sampling : 0;
+
+    if (typeof configFromServer.ingestUrl !== 'string') {
+      logError('id5Analytics: cannot find ingestUrl in config endpoint response; no analytics will be available');
+      return;
+    }
+    _this.options.ingestUrl = configFromServer.ingestUrl;
+
+    // 3-way fallback for which events to track: server > config > standard
+    _this.eventsToTrack = configFromServer.eventsToTrack || _this.options.eventsToTrack || STANDARD_EVENTS_TO_TRACK;
+    _this.eventsToTrack = isArray(_this.eventsToTrack) ? _this.eventsToTrack : STANDARD_EVENTS_TO_TRACK;
+
+    logInfo('id5Analytics: Configuration is', _this.options);
+    logInfo('id5Analytics: Tracking events', _this.eventsToTrack);
+    if (sampling > 0 && _this.random() < (1 / sampling)) {
+      // Init the module only if we got lucky
+      logInfo('id5Analytics: Selected by sampling. Starting up!')
+
+      // Clean start
+      _this.eventBuffer = {};
+
+      // Replay all events until now
+      if (!config.disablePastEventsProcessing) {
+        events.getEvents().forEach((event) => {
+          if (event && _this.eventsToTrack.indexOf(event.eventType) >= 0) {
+            _this.track(event);
+          }
+        });
+      }
+
+      // Merge in additional cleanup rules
+      if (configFromServer.additionalCleanupRules) {
+        const newRules = configFromServer.additionalCleanupRules;
+        _this.eventsToTrack.forEach((key) => {
+          // Some protective checks in case we mess up server side
+          if (
+            isArray(newRules[key]) &&
+            newRules[key].every((eventRules) =>
+              isArray(eventRules.match) &&
+              (eventRules.apply in TRANSFORM_FUNCTIONS))
+          ) {
+            logInfo('id5Analytics: merging additional cleanup rules for event ' + key);
+            CLEANUP_RULES[key].push(...newRules[key]);
+          }
+        });
+      }
+
+      // Register to the events of interest
+      _this.handlers = {};
+      _this.eventsToTrack.forEach((eventType) => {
+        const handler = _this.handlers[eventType] = (args) =>
+          _this.track({ eventType, args });
+        events.on(eventType, handler);
+      });
+    }
+  });
+
+  // Make only one init possible within a lifecycle
+  _this.enableAnalytics = () => {};
+};
+
+id5Analytics.enableAnalytics = ENABLE_FUNCTION;
+id5Analytics.disableAnalytics = () => {
+  const _this = id5Analytics;
+  // Un-register to the events of interest
+  _this.eventsToTrack.forEach((eventType) => {
+    if (_this.handlers && _this.handlers[eventType]) {
+      events.off(eventType, _this.handlers[eventType]);
+    }
+  });
+
+  // Make re-init possible. Work around the fact that past events cannot be forgotten
+  _this.enableAnalytics = (config) => {
+    config.disablePastEventsProcessing = true;
+    ENABLE_FUNCTION(config);
+  };
+};
+
+adapterManager.registerAnalyticsAdapter({
+  adapter: id5Analytics,
+  code: 'id5Analytics',
+  gvlid: GVLID
+});
+
+export default id5Analytics;
+
+function redact(obj, key) {
+  obj[key] = ID5_REDACTED;
+}
+
+function erase(obj, key) {
+  delete obj[key];
+}
+
+// The transform function matches against a path and applies
+// required transformation if match is found.
+function deepTransformingClone(obj, transform, currentPath = []) {
+  const result = isArray(obj) ? [] : {};
+  const recursable = typeof obj === 'object' && obj !== null;
+  if (recursable) {
+    const keys = Object.keys(obj);
+    if (keys.length > 0) {
+      keys.forEach((key) => {
+        const newPath = currentPath.concat(key);
+        result[key] = deepTransformingClone(obj[key], transform, newPath);
+        transform(newPath, result, key);
+      });
+      return result;
+    }
+  }
+  return obj;
+}
+
+// Every set of rules is an object where "match" is an array and
+// "apply" is the function to apply in case of match. The function to apply
+// takes (obj, prop) and transforms property "prop" in object "obj".
+// The "match" is an array of path parts. Each part is either a string or an array.
+// In case of array, it represents alternatives which all would match.
+// Special path part '*' matches any subproperty
+const CLEANUP_RULES = {};
+CLEANUP_RULES[AUCTION_END] = [{
+  match: [['adUnits', 'bidderRequests'], '*', 'bids', '*', ['userId', 'crumbs'], '*'],
+  apply: 'redact'
+}, {
+  match: [['adUnits', 'bidderRequests'], '*', 'bids', '*', 'userIdAsEids', '*', 'uids', '*', ['id', 'ext']],
+  apply: 'redact'
+}, {
+  match: ['bidderRequests', '*', 'gdprConsent', 'vendorData'],
+  apply: 'erase'
+}, {
+  match: ['bidsReceived', '*', ['ad', 'native']],
+  apply: 'erase'
+}, {
+  match: ['noBids', '*', ['userId', 'crumbs'], '*'],
+  apply: 'redact'
+}, {
+  match: ['noBids', '*', 'userIdAsEids', '*', 'uids', '*', ['id', 'ext']],
+  apply: 'redact'
+}];
+
+CLEANUP_RULES[BID_WON] = [{
+  match: [['ad', 'native']],
+  apply: 'erase'
+}];
+
+const TRANSFORM_FUNCTIONS = {
+  'redact': redact,
+  'erase': erase,
+};
+
+// Builds a rule function depending on the event type
+function transformFnFromCleanupRules(eventType) {
+  const rules = CLEANUP_RULES[eventType] || [];
+  return (path, obj, key) => {
+    for (let i = 0; i < rules.length; i++) {
+      let match = true;
+      const ruleMatcher = rules[i].match;
+      const transformation = rules[i].apply;
+      if (ruleMatcher.length !== path.length) {
+        continue;
+      }
+      for (let fragment = 0; fragment < ruleMatcher.length && match; fragment++) {
+        const choices = makeSureArray(ruleMatcher[fragment]);
+        match = !choices.every((choice) => choice !== '*' && path[fragment] !== choice);
+      }
+      if (match) {
+        const transformfn = TRANSFORM_FUNCTIONS[transformation];
+        transformfn(obj, key);
+        break;
+      }
+    }
+  };
+}
+
+function makeSureArray(object) {
+  return isArray(object) ? object : [object];
+}

modules/id5AnalyticsAdapter.md

@@ -0,0 +1,42 @@
+# Overview
+Module Name: ID5 Analytics Adapter
+
+Module Type: Analytics Adapter
+
+Maintainer: [id5.io](https://id5.io)
+
+# ID5 Universal ID
+
+The ID5 Universal ID is a shared, neutral identifier that publishers and ad tech platforms can use to recognise users even in environments where 3rd party cookies are not available. The ID5 Universal ID is designed to respect users' privacy choices and publishers’ preferences throughout the advertising value chain. For more information about the ID5 Universal ID and detailed integration docs, please visit [our documentation](https://support.id5.io/portal/en/kb/articles/prebid-js-user-id-module).
+
+# ID5 Analytics Registration
+
+The ID5 Analytics Adapter is free to use during our Beta period, but requires a simple registration with ID5. Please visit [id5.io/universal-id](https://id5.io/universal-id) to sign up and request your ID5 Partner Number to get started. If you're already using the ID5 Universal ID, you may use your existing Partner Number with the analytics adapter.
+
+The ID5 privacy policy is at [https://www.id5.io/platform-privacy-policy](https://www.id5.io/platform-privacy-policy).
+
+## ID5 Analytics Configuration
+
+First, make sure to add the ID5 Analytics submodule to your Prebid.js package with:
+
+```
+gulp build --modules=...,id5AnalyticsAdapter
+```
+
+The following configuration parameters are available:
+
+```javascript
+pbjs.enableAnalytics({
+    provider: 'id5Analytics',
+    options: {
+        partnerId: 1234, // change to the Partner Number you received from ID5
+        eventsToTrack: ['auctionEnd','bidWon']
+    }
+});
+```
+
+| Parameter | Scope | Type | Description | Example |
+| --- | --- | --- | --- | --- |
+| provider | Required | String | The name of this module: `id5Analytics` | `id5Analytics` |
+| options.partnerId | Required | Number | This is the ID5 Partner Number obtained from registering with ID5. | `1234` |
+| options.eventsToTrack | Optional | Array of strings | Overrides the set of tracked events | `['auctionEnd','bidWon']` |