diff --git a/CHANGELOG.md b/CHANGELOG.md index 4258fdfcc56..9c04835cd4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.67.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.66.1...v1.67.0) (2024-05-23) + + +### Features + +* sre 456 ut move high cardinality histogram metrics to summaries cp ([#3409](https://github.com/rudderlabs/rudder-transformer/issues/3409)) ([be20dc2](https://github.com/rudderlabs/rudder-transformer/commit/be20dc26ade2fa0212dc91126cf42087a84a07c9)) + ### [1.66.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.66.0...v1.66.1) (2024-05-20) diff --git a/package-lock.json b/package-lock.json index f51f3ccd8ef..bfeb00963a1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.66.1", + "version": "1.67.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.66.1", + "version": "1.67.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 50a276ce42e..7fa3a7330c8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.66.1", + "version": "1.67.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { diff --git a/src/cdk/v2/destinations/koddi/config.js b/src/cdk/v2/destinations/koddi/config.js new file mode 100644 index 00000000000..927e1858fcd --- /dev/null +++ b/src/cdk/v2/destinations/koddi/config.js @@ -0,0 +1,39 @@ +const { getMappingConfig } = require('../../../../v0/util'); + +/** + * ref :- https://developers.koddi.com/reference/winning-ads + * impressions - https://developers.koddi.com/reference/impressions-1 + * clicks - https://developers.koddi.com/reference/clicks-1 + * conversions - https://developers.koddi.com/reference/conversions-1 + */ +const EVENT_TYPES = { + IMPRESSIONS: 'impressions', + CLICKS: 'clicks', + CONVERSIONS: 'conversions', +}; + +const CONFIG_CATEGORIES = { + IMPRESSIONS: { + type: 'track', + name: 'ImpressionsConfig', + }, + CLICKS: { + type: 'track', + name: 'ClicksConfig', + }, + CONVERSIONS: { + type: 'track', + name: 'ConversionsConfig', + }, +}; + +const MAPPING_CONFIG = getMappingConfig(CONFIG_CATEGORIES, __dirname); + +module.exports = { + EVENT_TYPES, + CONFIG_CATEGORIES, + MAPPING_CONFIG, + IMPRESSIONS_CONFIG: MAPPING_CONFIG[CONFIG_CATEGORIES.IMPRESSIONS.name], + CLICKS_CONFIG: MAPPING_CONFIG[CONFIG_CATEGORIES.CLICKS.name], + CONVERSIONS_CONFIG: MAPPING_CONFIG[CONFIG_CATEGORIES.CONVERSIONS.name], +}; diff --git a/src/cdk/v2/destinations/koddi/data/ClicksConfig.json b/src/cdk/v2/destinations/koddi/data/ClicksConfig.json new file mode 100644 index 00000000000..96ab27b2ae0 --- /dev/null +++ b/src/cdk/v2/destinations/koddi/data/ClicksConfig.json @@ -0,0 +1,35 @@ +[ + { + "sourceKeys": "properties.tracking_data", + "required": true, + "destKey": "trackingData" + }, + { + "sourceKeys": "properties.rank", + "required": true, + "destKey": "rank" + }, + { + "sourceKeys": "properties.beacon_issued", + "required": true, + "destKey": "beaconIssued" + }, + { + "sourceKeys": "userId", + "sourceFromGenericMap": true, + "required": true, + "destKey": "userGuid" + }, + { + "sourceKeys": "properties.test_version_override", + "destKey": "testVersionOverride" + }, + { + "sourceKeys": "properties.destination_url", + "destKey": "destinationUrl" + }, + { + "sourceKeys": "properties.overrides", + "destKey": "overrides" + } +] diff --git a/src/cdk/v2/destinations/koddi/data/ConversionsConfig.json b/src/cdk/v2/destinations/koddi/data/ConversionsConfig.json new file mode 100644 index 00000000000..495574f1983 --- /dev/null +++ b/src/cdk/v2/destinations/koddi/data/ConversionsConfig.json @@ -0,0 +1,53 @@ +[ + { + "sourceKeys": "context.page.referring_domain", + "destKey": "domain" + }, + { + "sourceKeys": "context.locale", + "required": true, + "destKey": "culture" + }, + { + "sourceKeys": "properties.currency", + "required": true, + "destKey": "currency" + }, + { + "sourceKeys": ["context.ip", "request_ip"], + "destKey": "user_ip" + }, + { + "sourceKeys": "context.userAgent", + "destKey": "user_agent" + }, + { + "sourceKeys": "userId", + "sourceFromGenericMap": true, + "required": true, + "destKey": "user_guid" + }, + { + "sourceKeys": "context.device.type", + "destKey": "device_type" + }, + { + "sourceKeys": ["properties.order_id", "properties.transaction_id"], + "required": true, + "destKey": "transaction_id" + }, + { + "sourceKeys": "properties.conversion_source", + "destKey": "conversion_source" + }, + { + "sourceKeys": "timestamp", + "sourceFromGenericMap": true, + "destKey": "unixtime" + }, + { + "sourceKeys": "properties.bidders", + "required": true, + "destKey": "bidders" + } +] diff --git a/src/cdk/v2/destinations/koddi/data/ImpressionsConfig.json b/src/cdk/v2/destinations/koddi/data/ImpressionsConfig.json new file mode 100644 index 00000000000..de53703b32d --- /dev/null +++ b/src/cdk/v2/destinations/koddi/data/ImpressionsConfig.json @@ -0,0 +1,22 @@ +[ + { + "sourceKeys": "properties.tracking_data", + "required": true, + "destKey": "trackingData" + }, + { + "sourceKeys": "properties.rank", + "required": true, + "destKey": "rank" + }, + { + "sourceKeys": "properties.beacon_issued", + "required": true, + "destKey": "beaconIssued" + }, + { + "sourceKeys": "timestamp", + "sourceFromGenericMap": true, + "destKey": "ts" + } +] diff --git a/src/cdk/v2/destinations/koddi/procWorkflow.yaml b/src/cdk/v2/destinations/koddi/procWorkflow.yaml new file mode 100644 index 00000000000..cc3f0166dc1 --- /dev/null +++ b/src/cdk/v2/destinations/koddi/procWorkflow.yaml @@ -0,0 +1,33 @@ +bindings: + - name: EventType + path: ../../../../constants + - path: ../../bindings/jsontemplate + - name: removeUndefinedAndNullValues + path: ../../../../v0/util + - path: ./utils + - path: ./config + +steps: + - name: messageType + template: | + .message.type.toLowerCase(); + - name: eventType + template: | + .message.integrations.koddi.eventType.toLowerCase(); + - name: validateInput + template: | + let messageType = $.outputs.messageType; + let eventType = $.outputs.eventType; + $.assert(messageType, "message Type is not present. Aborting message."); + $.assert(messageType in {{$.EventType.([.TRACK])}}, "message type " + messageType + " is not supported"); + $.assert(eventType in {{$.EVENT_TYPES.([.IMPRESSIONS, .CLICKS, .CONVERSIONS])}}, "event type " + eventType + " is not supported"); + $.assertConfig(.destination.Config.apiBaseUrl, "API Base URL is not present. Aborting"); + $.assertConfig(.destination.Config.clientName, "Client Name is not present. Aborting"); + - name: preparePayload + template: | + const payload = $.constructFullPayload($.outputs.eventType, .message, .destination.Config); + $.context.payload = $.removeUndefinedAndNullValues(payload); + - name: buildResponse + template: | + const response = $.constructResponse($.outputs.eventType, .destination.Config, $.context.payload); + response diff --git a/src/cdk/v2/destinations/koddi/rtWorkflow.yaml b/src/cdk/v2/destinations/koddi/rtWorkflow.yaml new file mode 100644 index 00000000000..dd438a911ce --- /dev/null +++ b/src/cdk/v2/destinations/koddi/rtWorkflow.yaml @@ -0,0 +1,31 @@ +bindings: + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "batchedRequest": ., + "batched": false, + "destination": ^[idx].destination, + "metadata": ^[idx].metadata[], + "statusCode": 200 + })[] + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + - name: finalPayload + template: | + [...$.outputs.successfulEvents, ...$.outputs.failedEvents] diff --git a/src/cdk/v2/destinations/koddi/utils.js b/src/cdk/v2/destinations/koddi/utils.js new file mode 100644 index 00000000000..13014e2e7ce --- /dev/null +++ b/src/cdk/v2/destinations/koddi/utils.js @@ -0,0 +1,116 @@ +const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { EVENT_TYPES, IMPRESSIONS_CONFIG, CLICKS_CONFIG, CONVERSIONS_CONFIG } = require('./config'); +const { + constructPayload, + defaultRequestConfig, + toUnixTimestamp, + stripTrailingSlash, +} = require('../../../../v0/util'); + +const validateBidders = (bidders) => { + if (!Array.isArray(bidders)) { + throw new InstrumentationError('properties.bidders should be an array of objects. Aborting.'); + } + if (bidders.length === 0) { + throw new InstrumentationError( + 'properties.bidders should contains at least one bidder. Aborting.', + ); + } + bidders.forEach((bidder) => { + if (!(bidder.bidder || bidder.alternate_bidder)) { + throw new InstrumentationError('bidder or alternate_bidder is not present. Aborting.'); + } + if (!bidder.count) { + throw new InstrumentationError('count is not present. Aborting.'); + } + if (!bidder.base_price) { + throw new InstrumentationError('base_price is not present. Aborting.'); + } + }); +}; + +/** + * This function constructs payloads based upon mappingConfig for all calls. + * @param {*} eventType + * @param {*} message + * @param {*} Config + * @returns + */ +const constructFullPayload = (eventType, message, Config) => { + let payload; + switch (eventType) { + case EVENT_TYPES.IMPRESSIONS: + payload = constructPayload(message, IMPRESSIONS_CONFIG); + payload.clientName = Config.clientName; + break; + case EVENT_TYPES.CLICKS: + payload = constructPayload(message, CLICKS_CONFIG); + payload.clientName = Config.clientName; + if (!Config.testVersionOverride) { + payload.testVersionOverride = null; + } + if (!Config.overrides) { + payload.overrides = null; + } + break; + case EVENT_TYPES.CONVERSIONS: + payload = constructPayload(message, CONVERSIONS_CONFIG); + payload.client_name = Config.clientName; + payload.unixtime = toUnixTimestamp(payload.unixtime); + validateBidders(payload.bidders); + break; + default: + throw new InstrumentationError(`event type ${eventType} is not supported.`); + } + return payload; +}; + +const getEndpoint = (eventType, Config) => { + let endpoint = stripTrailingSlash(Config.apiBaseUrl); + switch (eventType) { + case EVENT_TYPES.IMPRESSIONS: + endpoint += '?action=impression'; + break; + case EVENT_TYPES.CLICKS: + endpoint += '?action=click'; + break; + case EVENT_TYPES.CONVERSIONS: + endpoint += '/conversion'; + break; + default: + throw new InstrumentationError(`event type ${eventType} is not supported.`); + } + return endpoint; +}; + +/** + * This function constructs response based upon event. + * @param {*} eventType + * @param {*} Config + * @param {*} payload + * @returns + */ +const constructResponse = (eventType, Config, payload) => { + if (!Object.values(EVENT_TYPES).includes(eventType)) { + throw new InstrumentationError(`event type ${eventType} is not supported.`); + } + const response = defaultRequestConfig(); + response.endpoint = getEndpoint(eventType, Config); + response.headers = { + accept: 'application/json', + }; + if (eventType === EVENT_TYPES.CONVERSIONS) { + response.body.JSON = payload; + response.method = 'POST'; + response.headers = { + ...response.headers, + 'content-type': 'application/json', + }; + } else { + response.params = payload; + response.method = 'GET'; + } + return response; +}; + +module.exports = { getEndpoint, validateBidders, constructFullPayload, constructResponse }; diff --git a/src/cdk/v2/destinations/koddi/utils.test.js b/src/cdk/v2/destinations/koddi/utils.test.js new file mode 100644 index 00000000000..2c1f660f701 --- /dev/null +++ b/src/cdk/v2/destinations/koddi/utils.test.js @@ -0,0 +1,421 @@ +const { + getEndpoint, + validateBidders, + constructFullPayload, + constructResponse, +} = require('./utils'); +const { InstrumentationError } = require('@rudderstack/integrations-lib'); + +describe('getEndpoint', () => { + it('returns the correct endpoint for IMPRESSIONS event', () => { + const eventType = 'impressions'; + const Config = { + apiBaseUrl: 'https://www.test-client.com/', + clientName: 'test-client', + }; + const result = getEndpoint(eventType, Config); + expect(result).toEqual('https://www.test-client.com?action=impression'); + }); + + it('returns the correct endpoint for CLICKS event', () => { + const eventType = 'clicks'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const result = getEndpoint(eventType, Config); + expect(result).toEqual('https://www.test-client.com?action=click'); + }); + + it('returns the correct endpoint for IMPRESSIONS event', () => { + const eventType = 'conversions'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const result = getEndpoint(eventType, Config); + expect(result).toEqual('https://www.test-client.com/conversion'); + }); + + it('should throw error for unsupported event', () => { + const eventType = 'test'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + expect(() => getEndpoint(eventType, Config)).toThrow(InstrumentationError); + expect(() => getEndpoint(eventType, Config)).toThrow('event type test is not supported.'); + }); +}); + +describe('validateBidders', () => { + it('should throw error if bidders is not an array', () => { + const bidders = {}; + expect(() => validateBidders(bidders)).toThrow(InstrumentationError); + expect(() => validateBidders(bidders)).toThrow( + 'properties.bidders should be an array of objects. Aborting.', + ); + }); + + it('should throw error if bidders is an empty array', () => { + const bidders = []; + expect(() => validateBidders(bidders)).toThrow(InstrumentationError); + expect(() => validateBidders(bidders)).toThrow( + 'properties.bidders should contains at least one bidder. Aborting.', + ); + }); + + it('should throw error if bidder or alternate_bidder is not present', () => { + const bidders = [ + { count: 1, base_price: 100 }, + { bidder: 'bidder1', count: 2, base_price: 200 }, + { alternate_bidder: 'alternate1', count: 3, base_price: 300 }, + ]; + expect(() => validateBidders(bidders)).toThrow(InstrumentationError); + expect(() => validateBidders(bidders)).toThrow( + 'bidder or alternate_bidder is not present. Aborting.', + ); + }); + + it('should throw error if count is not present', () => { + const bidders = [{ bidder: 'bidder1', alternate_bidder: 'alternate1', base_price: 100 }]; + expect(() => validateBidders(bidders)).toThrow(InstrumentationError); + expect(() => validateBidders(bidders)).toThrow('count is not present. Aborting.'); + }); + + it('should throw error if base_price is not present', () => { + const bidders = [{ bidder: 'bidder1', alternate_bidder: 'alternate1', count: 1 }]; + expect(() => validateBidders(bidders)).toThrow(InstrumentationError); + expect(() => validateBidders(bidders)).toThrow('base_price is not present. Aborting.'); + }); + + it('should not throw error if all required fields are present for all bidders', () => { + const bidders = [ + { bidder: 'bidder1', alternate_bidder: 'alternate1', count: 1, base_price: 100 }, + { bidder: 'bidder2', alternate_bidder: 'alternate2', count: 2, base_price: 200 }, + ]; + expect(() => validateBidders(bidders)).not.toThrow(); + }); +}); + +describe('constructFullPayload', () => { + it('should construct payload for IMPRESSIONS event', () => { + const eventType = 'impressions'; + const message = { + type: 'track', + event: 'Impressions Event', + properties: { + tracking_data: 'dummy-tracking-data', + rank: 1, + beacon_issued: '2024-03-04T15:32:56.409Z', + }, + timestamp: '2024-03-03T00:29:12.117+05:30', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const expectedPayload = { + beaconIssued: '2024-03-04T15:32:56.409Z', + clientName: 'test-client', + rank: 1, + trackingData: 'dummy-tracking-data', + ts: '2024-03-03T00:29:12.117+05:30', + }; + const payload = constructFullPayload(eventType, message, Config); + expect(payload).toEqual(expectedPayload); + }); + it('should throw error if required value is missing for IMPRESSIONS event', () => { + const eventType = 'impressions'; + const message = { + type: 'track', + event: 'Impressions Event', + properties: { + tracking_data: '', + rank: 1, + beacon_issued: '2024-03-04T15:32:56.409Z', + }, + timestamp: '2024-03-03T00:29:12.117+05:30', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + try { + const payload = constructFullPayload(eventType, message, Config); + } catch (error) { + expect(error.message).toEqual('Missing required value from "properties.tracking_data"'); + } + }); + + it('should construct payload for CLICKS event', () => { + const eventType = 'clicks'; + const message = { + type: 'track', + event: 'Clicks Event', + properties: { + tracking_data: 'dummy-tracking-data', + rank: 1, + beacon_issued: '2024-03-04T15:32:56.409Z', + }, + anonymousId: '1234', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const expectedPayload = { + beaconIssued: '2024-03-04T15:32:56.409Z', + clientName: 'test-client', + rank: 1, + trackingData: 'dummy-tracking-data', + userGuid: '1234', + overrides: null, + testVersionOverride: null, + }; + const payload = constructFullPayload(eventType, message, Config); + expect(payload).toEqual(expectedPayload); + }); + it('should construct payload with non-null value if overrides and testVersionOverride are enable and values for these are provided for CLICKS event ', () => { + const eventType = 'clicks'; + const message = { + type: 'track', + event: 'Clicks Event', + properties: { + tracking_data: 'dummy-tracking-data', + rank: 1, + beacon_issued: '2024-03-04T15:32:56.409Z', + overrides: 'overridden-value', + testVersionOverride: 1, + }, + anonymousId: '1234', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + overrides: true, + testVersionOverride: false, + }; + const expectedPayload = { + beaconIssued: '2024-03-04T15:32:56.409Z', + clientName: 'test-client', + rank: 1, + trackingData: 'dummy-tracking-data', + userGuid: '1234', + overrides: 'overridden-value', + testVersionOverride: null, + }; + const payload = constructFullPayload(eventType, message, Config); + expect(payload).toEqual(expectedPayload); + }); + it('should throw error if required value is missing for CLICKS event', () => { + const eventType = 'clicks'; + const message = { + type: 'track', + event: 'Clicks Event', + properties: { + tracking_data: 'dummy-tracking-data', + rank: 1, + beacon_issued: '2024-03-04T15:32:56.409Z', + }, + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + try { + const payload = constructFullPayload(eventType, message, Config); + } catch (error) { + expect(error.message).toEqual('Missing required value from "userId"'); + } + }); + + it('should construct payload for CONVERSIONS event', () => { + const eventType = 'conversions'; + const message = { + type: 'track', + event: 'Conversions Event', + properties: { + currency: 'USD', + order_id: '123', + bidders: [ + { + bidder: 'dummy-bidder-id', + count: 1, + base_price: 100.1, + }, + ], + }, + context: { + locale: 'en-US', + ip: '127.0.0.1', + }, + timestamp: '2024-03-03T00:29:12.117+05:30', + anonymousId: '1234', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const expectedPayload = { + client_name: 'test-client', + culture: 'en-US', + currency: 'USD', + transaction_id: '123', + unixtime: 1709405952, + user_guid: '1234', + user_ip: '127.0.0.1', + bidders: [ + { + bidder: 'dummy-bidder-id', + count: 1, + base_price: 100.1, + }, + ], + }; + const payload = constructFullPayload(eventType, message, Config); + expect(payload).toEqual(expectedPayload); + }); + it('should throw error if required value is missing for CONVERSIONS event', () => { + const eventType = 'conversions'; + const message = { + type: 'track', + event: 'Conversions Event', + properties: { + currency: 'USD', + order_id: '123', + bidders: [ + { + bidder: 'dummy-bidder-id', + count: 1, + base_price: 100.1, + }, + ], + }, + context: { + ip: '127.0.0.1', + }, + timestamp: '2024-03-03T00:29:12.117+05:30', + anonymousId: '1234', + }; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + try { + const payload = constructFullPayload(eventType, message, Config); + } catch (error) { + expect(error.message).toEqual('Missing required value from "context.locale"'); + } + }); + + it('should throw error for unsupported event', () => { + const eventType = 'test'; + const message = {}; + const Config = {}; + expect(() => constructFullPayload(eventType, message, Config)).toThrow(InstrumentationError); + expect(() => constructFullPayload(eventType, message, Config)).toThrow( + 'event type test is not supported.', + ); + }); +}); + +describe('constructResponse', () => { + it('should construct response for IMPRESSIONS event', () => { + const eventType = 'impressions'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const payload = { + beaconIssued: '2024-03-04T15:32:56.409Z', + clientName: 'test-client', + rank: 1, + trackingData: 'dummy-tracking-data', + ts: '2024-03-03T00:29:12.117+05:30', + }; + const expectedResponse = { + endpoint: 'https://www.test-client.com?action=impression', + headers: { + accept: 'application/json', + }, + method: 'GET', + params: payload, + }; + const response = constructResponse(eventType, Config, payload); + expect(response).toMatchObject(expectedResponse); + }); + + it('should construct response for CLICKS event', () => { + const eventType = 'clicks'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const payload = { + beaconIssued: '2024-03-04T15:32:56.409Z', + clientName: 'test-client', + rank: 1, + trackingData: 'dummy-tracking-data', + userGuid: '1234', + }; + const expectedResponse = { + endpoint: 'https://www.test-client.com?action=click', + headers: { + accept: 'application/json', + }, + method: 'GET', + params: payload, + }; + const response = constructResponse(eventType, Config, payload); + expect(response).toMatchObject(expectedResponse); + }); + + it('should construct response for CONVERSIONS event', () => { + const eventType = 'conversions'; + const Config = { + apiBaseUrl: 'https://www.test-client.com', + clientName: 'test-client', + }; + const payload = { + client_name: 'test-client', + culture: 'en-US', + currency: 'USD', + transaction_id: '123', + unixtime: 1709405952, + userGuid: '1234', + user_ip: '127.0.0.1', + bidders: [ + { + bidder: 'dummy-bidder-id', + count: 1, + base_price: 100.1, + }, + ], + }; + + const expectedResponse = { + endpoint: 'https://www.test-client.com/conversion', + headers: { + accept: 'application/json', + 'content-type': 'application/json', + }, + method: 'POST', + body: { + JSON: payload, + }, + }; + const response = constructResponse(eventType, Config, payload); + expect(response).toMatchObject(expectedResponse); + }); + + it('should throw error for unsupported event', () => { + const eventType = 'test'; + const Config = {}; + const payload = {}; + expect(() => constructResponse(eventType, Config, payload)).toThrow(InstrumentationError); + expect(() => constructResponse(eventType, Config, payload)).toThrow( + 'event type test is not supported.', + ); + }); +}); diff --git a/src/cdk/v2/destinations/reddit/procWorkflow.yaml b/src/cdk/v2/destinations/reddit/procWorkflow.yaml index 7b989f15e44..06d2c95f258 100644 --- a/src/cdk/v2/destinations/reddit/procWorkflow.yaml +++ b/src/cdk/v2/destinations/reddit/procWorkflow.yaml @@ -36,13 +36,13 @@ steps: const os = (.message.context.os.name)? .message.context.os.name.toLowerCase(): null; const hashData = .destination.Config.hashData; let user = .message.().({ - "email": hashData ? $.SHA256({{{{$.getGenericPaths("email")}}}}) : ({{{{$.getGenericPaths("email")}}}}), - "external_id": hashData ? $.SHA256({{{{$.getGenericPaths("userId")}}}}) : ({{{{$.getGenericPaths("userId")}}}}), - "ip_address": hashData? $.SHA256(.context.ip || .request_ip) : (.context.ip || .request_ip), + "email": hashData ? $.SHA256({{{{$.getGenericPaths("email")}}}}.trim()) : ({{{{$.getGenericPaths("email")}}}}), + "external_id": hashData ? $.SHA256({{{{$.getGenericPaths("userId")}}}}.trim()) : ({{{{$.getGenericPaths("userId")}}}}), + "ip_address": hashData? $.SHA256(.context.ip.trim() || .request_ip.trim()) : (.context.ip || .request_ip), "uuid": .properties.uuid, "user_agent": .context.userAgent, - "idfa": $.isAppleFamily(os)? (hashData? $.SHA256(.context.device.advertisingId): .context.device.advertisingId): null, - "aaid": os === "android" && .context.device ? (hashData? $.SHA256(.context.device.advertisingId): .context.device.advertisingId): null, + "idfa": $.isAppleFamily(os)? (hashData? $.SHA256(.context.device.advertisingId.trim()): .context.device.advertisingId): null, + "aaid": os === "android" && .context.device ? (hashData? $.SHA256(.context.device.advertisingId.trim()): .context.device.advertisingId): null, "opt_out": .properties.optOut, "screen_dimensions": {"width": .context.screen.width, "height": .context.screen.height}, }); diff --git a/src/features.json b/src/features.json index a7e4b701093..58af795a772 100644 --- a/src/features.json +++ b/src/features.json @@ -71,7 +71,8 @@ "LINKEDIN_ADS": true, "BLOOMREACH": true, "MOVABLE_INK": true, - "EMARSYS": true + "EMARSYS": true, + "KODDI": true }, "regulations": [ "BRAZE", diff --git a/src/legacy/router.js b/src/legacy/router.js index 9dd83b5988f..afc8c1a7970 100644 --- a/src/legacy/router.js +++ b/src/legacy/router.js @@ -649,6 +649,9 @@ if (startDestTransformer) { stats.timing('user_transform_request_latency', startTime, { processSessions, }); + stats.timingSummary('user_transform_request_latency_summary', startTime, { + processSessions, + }); stats.increment('user_transform_requests', { processSessions }); stats.histogram('user_transform_output_events', transformedEvents.length, { processSessions, diff --git a/src/services/userTransform.ts b/src/services/userTransform.ts index 18c47ddc83a..62980a935a6 100644 --- a/src/services/userTransform.ts +++ b/src/services/userTransform.ts @@ -173,7 +173,17 @@ export class UserTransformService { ...getTransformationMetadata(eventsToProcess[0]?.metadata), }); - stats.histogram('user_transform_batch_size', requestSize, { + stats.timing('user_transform_batch_size', requestSize, { + ...metaTags, + ...getTransformationMetadata(eventsToProcess[0]?.metadata), + }); + + stats.timingSummary('user_transform_request_latency_summary', userFuncStartTime, { + ...metaTags, + ...getTransformationMetadata(eventsToProcess[0]?.metadata), + }); + + stats.timingSummary('user_transform_batch_size_summary', requestSize, { ...metaTags, ...getTransformationMetadata(eventsToProcess[0]?.metadata), }); diff --git a/src/util/customTransformer-v1.js b/src/util/customTransformer-v1.js index 7e854a3714b..e9877a614df 100644 --- a/src/util/customTransformer-v1.js +++ b/src/util/customTransformer-v1.js @@ -93,6 +93,7 @@ async function userTransformHandlerV1( }; stats.counter('user_transform_function_input_events', events.length, tags); stats.timing('user_transform_function_latency', invokeTime, tags); + stats.timingSummary('user_transform_function_latency_summary', invokeTime, tags); } return { transformedEvents, logs }; diff --git a/src/util/customTransformer.js b/src/util/customTransformer.js index a87c12dd6e2..37364ef5d01 100644 --- a/src/util/customTransformer.js +++ b/src/util/customTransformer.js @@ -260,6 +260,7 @@ async function runUserTransform( stats.counter('user_transform_function_input_events', events.length, tags); stats.timing('user_transform_function_latency', invokeTime, tags); + stats.timingSummary('user_transform_function_latency_summary', invokeTime, tags); } return { diff --git a/src/util/customTransforrmationsStore-v1.js b/src/util/customTransforrmationsStore-v1.js index 3263049b6f9..6e2d799f3a5 100644 --- a/src/util/customTransforrmationsStore-v1.js +++ b/src/util/customTransforrmationsStore-v1.js @@ -31,6 +31,7 @@ async function getTransformationCodeV1(versionId) { responseStatusHandler(response.status, 'Transformation', versionId, url); stats.increment('get_transformation_code', { success: 'true', ...tags }); stats.timing('get_transformation_code_time', startTime, tags); + stats.timingSummary('get_transformation_code_time_summary', startTime, tags); const myJson = await response.json(); transformationCache[versionId] = myJson; return myJson; @@ -56,6 +57,7 @@ async function getLibraryCodeV1(versionId) { responseStatusHandler(response.status, 'Transformation Library', versionId, url); stats.increment('get_libraries_code', { success: 'true', ...tags }); stats.timing('get_libraries_code_time', startTime, tags); + stats.timingSummary('get_libraries_code_time_summary', startTime, tags); const myJson = await response.json(); libraryCache[versionId] = myJson; return myJson; @@ -83,6 +85,7 @@ async function getRudderLibByImportName(importName) { responseStatusHandler(response.status, 'Rudder Library', importName, url); stats.increment('get_libraries_code', { success: 'true', ...tags }); stats.timing('get_libraries_code_time', startTime, tags); + stats.timingSummary('get_libraries_code_time_summary', startTime, tags); const myJson = await response.json(); rudderLibraryCache[importName] = myJson; return myJson; diff --git a/src/util/customTransforrmationsStore.js b/src/util/customTransforrmationsStore.js index 08d417c07cd..2c5a7b446d2 100644 --- a/src/util/customTransforrmationsStore.js +++ b/src/util/customTransforrmationsStore.js @@ -24,6 +24,7 @@ async function getTransformationCode(versionId) { responseStatusHandler(response.status, 'Transformation', versionId, url); stats.increment('get_transformation_code', { versionId, success: 'true' }); stats.timing('get_transformation_code_time', startTime, { versionId }); + stats.timingSummary('get_transformation_code_time_summary', startTime, { versionId }); const myJson = await response.json(); myCache.set(versionId, myJson); return myJson; diff --git a/src/util/openfaas/index.js b/src/util/openfaas/index.js index 7a1fce3cfa4..3cf3525e6fb 100644 --- a/src/util/openfaas/index.js +++ b/src/util/openfaas/index.js @@ -305,6 +305,7 @@ const executeFaasFunction = async ( stats.counter('user_transform_function_input_events', events.length, tags); stats.timing('user_transform_function_latency', startTime, tags); + stats.timingSummary('user_transform_function_latency_summary', startTime, tags); } }; diff --git a/src/util/prometheus.js b/src/util/prometheus.js index 882dff9e757..78d32c9cb94 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -11,7 +11,7 @@ function appendPrefix(name) { } class Prometheus { - constructor() { + constructor(enableSummaryMetrics = true) { this.prometheusRegistry = new prometheusClient.Registry(); this.prometheusRegistry.setDefaultLabels(defaultLabels); prometheusClient.collectDefaultMetrics({ @@ -21,7 +21,7 @@ class Prometheus { prometheusClient.AggregatorRegistry.setRegistries(this.prometheusRegistry); this.aggregatorRegistry = new prometheusClient.AggregatorRegistry(); - this.createMetrics(); + this.createMetrics(enableSummaryMetrics); } async metricsController(ctx) { @@ -56,11 +56,22 @@ class Prometheus { return gauge; } - newSummaryStat(name, help, labelNames) { + newSummaryStat( + name, + help, + labelNames, + percentiles = [0.5, 0.9, 0.99], + maxAgeSeconds = 300, + ageBuckets = 5, + ) { + // we enable a 5 minute sliding window and calculate the 50th, 90th, and 99th percentiles by default const summary = new prometheusClient.Summary({ name, help, labelNames, + percentiles, + maxAgeSeconds, + ageBuckets, }); this.prometheusRegistry.registerMetric(summary); return summary; @@ -117,6 +128,21 @@ class Prometheus { } } + timingSummary(name, start, tags = {}) { + try { + let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name)); + if (!metric) { + logger.warn( + `Prometheus: summary metric ${name} not found in the registry. Creating a new one`, + ); + metric = this.newSummaryStat(name, name, Object.keys(tags)); + } + metric.observe(tags, (new Date() - start) / 1000); + } catch (e) { + logger.error(`Prometheus: Summary metric ${name} failed with error ${e}`); + } + } + histogram(name, value, tags = {}) { try { let metric = this.prometheusRegistry.getSingleMetric(appendPrefix(name)); @@ -166,7 +192,7 @@ class Prometheus { } } - createMetrics() { + createMetrics(enableSummaryMetrics) { const metrics = [ // Counters { @@ -587,12 +613,6 @@ class Prometheus { type: 'gauge', labelNames: ['destination_id'], }, - { - name: 'mixpanel_batch_track_pack_size', - help: 'mixpanel_batch_track_pack_size', - type: 'gauge', - labelNames: ['destination_id'], - }, { name: 'mixpanel_batch_import_pack_size', help: 'mixpanel_batch_import_pack_size', @@ -698,6 +718,18 @@ class Prometheus { 'k8_namespace', ], }, + { + name: 'user_transform_request_latency_summary', + help: 'user_transform_request_latency_summary', + type: 'summary', + labelNames: [ + 'workspaceId', + 'transformationId', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + }, { name: 'user_transform_batch_size', help: 'user_transform_batch_size', @@ -714,6 +746,18 @@ class Prometheus { 524288000, ], // 1KB, 100KB, 0.5MB, 1MB, 10MB, 20MB, 50MB, 100MB, 200MB, 500MB }, + { + name: 'user_transform_batch_size_summary', + help: 'user_transform_batch_size_summary', + type: 'summary', + labelNames: [ + 'workspaceId', + 'transformationId', + 'sourceType', + 'destinationType', + 'k8_namespace', + ], + }, { name: 'source_transform_request_latency', help: 'source_transform_request_latency', @@ -770,12 +814,24 @@ class Prometheus { type: 'histogram', labelNames: ['versionId', 'version'], }, + { + name: 'get_transformation_code_time_summary', + help: 'get_transformation_code_time_summary', + type: 'summary', + labelNames: ['versionId', 'version'], + }, { name: 'get_libraries_code_time', help: 'get_libraries_code_time', type: 'histogram', labelNames: ['libraryVersionId', 'versionId', 'type', 'version'], }, + { + name: 'get_libraries_code_time_summary', + help: 'get_libraries_code_time_summary', + type: 'summary', + labelNames: ['libraryVersionId', 'versionId', 'type', 'version'], + }, { name: 'isolate_cpu_time', help: 'isolate_cpu_time', @@ -1027,6 +1083,22 @@ class Prometheus { 'workspaceId', ], }, + { + name: 'user_transform_function_latency_summary', + help: 'user_transform_function_latency_summary', + type: 'summary', + labelNames: [ + 'identifier', + 'testMode', + 'sourceType', + 'destinationType', + 'k8_namespace', + 'errored', + 'statusCode', + 'transformationId', + 'workspaceId', + ], + }, ]; metrics.forEach((metric) => { @@ -1042,6 +1114,17 @@ class Prometheus { metric.labelNames, metric.buckets, ); + } else if (metric.type === 'summary') { + if (enableSummaryMetrics) { + this.newSummaryStat( + appendPrefix(metric.name), + metric.help, + metric.labelNames, + metric.percentiles, + metric.maxAge, + metric.ageBuckets, + ); + } } else { logger.error( `Prometheus: Metric creation failed. Name: ${metric.name}. Invalid type: ${metric.type}`, diff --git a/src/util/stats.js b/src/util/stats.js index 9a32fd1de3b..0aa13fc85cc 100644 --- a/src/util/stats.js +++ b/src/util/stats.js @@ -4,6 +4,8 @@ const logger = require('../logger'); const enableStats = process.env.ENABLE_STATS !== 'false'; const statsClientType = process.env.STATS_CLIENT || 'statsd'; +// summary metrics are enabled by default. To disable set ENABLE_SUMMARY_METRICS='false'. +const enableSummaryMetrics = process.env.ENABLE_SUMMARY_METRICS !== 'false'; let statsClient; function init() { @@ -19,7 +21,7 @@ function init() { case 'prometheus': logger.info('setting up prometheus client'); - statsClient = new prometheus.Prometheus(); + statsClient = new prometheus.Prometheus(enableSummaryMetrics); break; default: @@ -38,6 +40,15 @@ const timing = (name, start, tags = {}) => { statsClient.timing(name, start, tags); }; +// timingSummary is used to record observations for a summary metric +const timingSummary = (name, start, tags = {}) => { + if (!enableStats || !statsClient || !enableSummaryMetrics) { + return; + } + + statsClient.timingSummary(name, start, tags); +}; + const increment = (name, tags = {}) => { if (!enableStats || !statsClient) { return; @@ -88,4 +99,13 @@ async function metricsController(ctx) { init(); -module.exports = { init, timing, increment, counter, gauge, histogram, metricsController }; +module.exports = { + init, + timing, + timingSummary, + increment, + counter, + gauge, + histogram, + metricsController, +}; diff --git a/src/util/statsd.js b/src/util/statsd.js index a32a6f6f302..7613de7975d 100644 --- a/src/util/statsd.js +++ b/src/util/statsd.js @@ -21,6 +21,11 @@ class Statsd { this.statsdClient.timing(name, start, tags); } + // timingSummary is just a wrapper around timing for statsd.For prometheus, we will have to implement a different function. + timingSummary(name, start, tags = {}) { + this.statsdClient.timing(name, start, tags); + } + increment(name, tags = {}) { this.statsdClient.increment(name, 1, tags); } diff --git a/src/v0/destinations/awin/transform.js b/src/v0/destinations/awin/transform.js index 68dd9d62e12..0e1e2205487 100644 --- a/src/v0/destinations/awin/transform.js +++ b/src/v0/destinations/awin/transform.js @@ -3,6 +3,7 @@ const { BASE_URL, ConfigCategory, mappingConfig } = require('./config'); const { defaultRequestConfig, constructPayload, simpleProcessRouterDest } = require('../../util'); const { getParams, trackProduct, populateCustomTransactionProperties } = require('./utils'); +const { FilteredEventsError } = require('../../util/errorTypes'); const responseBuilder = (message, { Config }) => { const { advertiserId, eventsToTrack, customFieldMap } = Config; @@ -33,9 +34,9 @@ const responseBuilder = (message, { Config }) => { ...customTransactionProperties, }; } else { - throw new InstrumentationError( - "Event is not present in 'Events to Track' list. Aborting message.", - 400, + throw new FilteredEventsError( + "Event is not present in 'Events to Track' list. Dropping the event.", + 298, ); } } diff --git a/src/v0/destinations/facebook_offline_conversions/utils.js b/src/v0/destinations/facebook_offline_conversions/utils.js index c48de4e0b99..460ef711768 100644 --- a/src/v0/destinations/facebook_offline_conversions/utils.js +++ b/src/v0/destinations/facebook_offline_conversions/utils.js @@ -396,7 +396,7 @@ const preparePayload = (facebookOfflineConversionsPayload, destination) => { const keys = Object.keys(facebookOfflineConversionsPayload); keys.forEach((key) => { if (isHashRequired && HASHING_REQUIRED_KEYS.includes(key)) { - payload[key] = sha256(facebookOfflineConversionsPayload[key]); + payload[key] = sha256(facebookOfflineConversionsPayload[key].trim()); } else { payload[key] = facebookOfflineConversionsPayload[key]; } @@ -407,8 +407,8 @@ const preparePayload = (facebookOfflineConversionsPayload, destination) => { ? facebookOfflineConversionsPayload.name.split(' ') : null; if (split !== null && Array.isArray(split) && split.length === 2) { - payload.fn = isHashRequired ? sha256(split[0]) : split[0]; - payload.ln = isHashRequired ? sha256(split[1]) : split[1]; + payload.fn = isHashRequired ? sha256(split[0].trim()) : split[0]; + payload.ln = isHashRequired ? sha256(split[1].trim()) : split[1]; } delete payload.name; } diff --git a/src/v0/destinations/fb/transform.js b/src/v0/destinations/fb/transform.js index e6f8e986cf2..1160cef407f 100644 --- a/src/v0/destinations/fb/transform.js +++ b/src/v0/destinations/fb/transform.js @@ -90,7 +90,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) { clonedUpdatedEvent[prop] !== '' ) { isUDSet = true; - clonedUpdatedEvent[prop] = sha256(clonedUpdatedEvent[prop].toLowerCase()); + clonedUpdatedEvent[prop] = sha256(clonedUpdatedEvent[prop].trim().toLowerCase()); } break; case 'ud[zp]': @@ -113,7 +113,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) { } else { isUDSet = true; clonedUpdatedEvent[prop] = sha256( - clonedUpdatedEvent[prop].toLowerCase() === 'female' ? 'f' : 'm', + clonedUpdatedEvent[prop].trim().toLowerCase() === 'female' ? 'f' : 'm', ); } } @@ -128,7 +128,7 @@ function sanityCheckPayloadForTypesAndModifications(updatedEvent) { if (clonedUpdatedEvent[prop] && clonedUpdatedEvent[prop] !== '') { isUDSet = true; clonedUpdatedEvent[prop] = sha256( - clonedUpdatedEvent[prop].toLowerCase().replace(/ /g, ''), + clonedUpdatedEvent[prop].trim().toLowerCase().replace(/ /g, ''), ); } break; diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json b/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json index bf5485270b8..c38b24598d3 100644 --- a/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json +++ b/src/v0/destinations/google_adwords_enhanced_conversions/data/trackConfig.json @@ -55,7 +55,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -64,7 +64,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -73,7 +73,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -82,7 +82,7 @@ "sourceFromGenericMap": true, "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } }, { @@ -127,7 +127,7 @@ "sourceKeys": ["context.traits.streetAddress", "context.traits.address"], "required": false, "metadata": { - "type": "hashToSha256" + "type": ["trim", "hashToSha256"] } } ] diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js index 0be7c3f0ee5..55d0c16c8cc 100644 --- a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js +++ b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js @@ -24,7 +24,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); const updateMappingJson = (mapping) => { const newMapping = []; mapping.forEach((element) => { - if (get(element, 'metadata.type') && element.metadata.type === 'hashToSha256') { + if (get(element, 'metadata.type') && element.metadata.type.includes('hashToSha256')) { element.metadata.type = 'toString'; } newMapping.push(element); diff --git a/src/v0/destinations/google_adwords_offline_conversions/utils.js b/src/v0/destinations/google_adwords_offline_conversions/utils.js index 70b42e2157a..dfa892a769e 100644 --- a/src/v0/destinations/google_adwords_offline_conversions/utils.js +++ b/src/v0/destinations/google_adwords_offline_conversions/utils.js @@ -140,17 +140,17 @@ const buildAndGetAddress = (message, hashUserIdentifier) => { const address = constructPayload(message, trackAddStoreAddressConversionsMapping); if (address.hashed_last_name) { address.hashed_last_name = hashUserIdentifier - ? sha256(address.hashed_last_name).toString() + ? sha256(address.hashed_last_name.trim()).toString() : address.hashed_last_name; } if (address.hashed_first_name) { address.hashed_first_name = hashUserIdentifier - ? sha256(address.hashed_first_name).toString() + ? sha256(address.hashed_first_name.trim()).toString() : address.hashed_first_name; } if (address.hashed_street_address) { address.hashed_street_address = hashUserIdentifier - ? sha256(address.hashed_street_address).toString() + ? sha256(address.hashed_street_address.trim()).toString() : address.hashed_street_address; } return Object.keys(address).length > 0 ? address : null; @@ -269,8 +269,10 @@ const getAddConversionPayload = (message, Config) => { const phone = getFieldValueFromMessage(message, 'phone'); const userIdentifierInfo = { - email: hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email).toString() : email, - phone: hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone).toString() : phone, + email: + hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email.trim()).toString() : email, + phone: + hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone.trim()).toString() : phone, address: buildAndGetAddress(message, hashUserIdentifier), }; @@ -363,8 +365,10 @@ const getClickConversionPayloadAndEndpoint = ( // Ref - https://developers.google.com/google-ads/api/rest/reference/rest/v11/customers/uploadClickConversions#ClickConversion const userIdentifierInfo = { - email: hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email).toString() : email, - phone: hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone).toString() : phone, + email: + hashUserIdentifier && isDefinedAndNotNull(email) ? sha256(email.trim()).toString() : email, + phone: + hashUserIdentifier && isDefinedAndNotNull(phone) ? sha256(phone.trim()).toString() : phone, }; const keyName = getExisitingUserIdentifier(userIdentifierInfo, defaultUserIdentifier); diff --git a/src/v0/destinations/impact/transform.js b/src/v0/destinations/impact/transform.js index 2eefdf7992d..729f9889380 100644 --- a/src/v0/destinations/impact/transform.js +++ b/src/v0/destinations/impact/transform.js @@ -59,7 +59,7 @@ const buildPageLoadPayload = (message, campaignId, impactAppId, enableEmailHashi let payload = constructPayload(message, MAPPING_CONFIG[CONFIG_CATEGORIES.PAGELOAD.name]); if (isDefinedAndNotNull(payload.CustomerEmail)) { payload.CustomerEmail = enableEmailHashing - ? sha1(payload?.CustomerEmail) + ? sha1(payload?.CustomerEmail.trim()) : payload?.CustomerEmail; } payload.CampaignId = campaignId; @@ -155,7 +155,7 @@ const processTrackEvent = (message, Config) => { payload.ImpactAppId = impactAppId; if (isDefinedAndNotNull(payload.CustomerEmail)) { payload.CustomerEmail = enableEmailHashing - ? sha1(payload?.CustomerEmail) + ? sha1(payload?.CustomerEmail.trim()) : payload?.CustomerEmail; } diff --git a/src/v0/destinations/mp/config.js b/src/v0/destinations/mp/config.js index 35b40294f55..3abdf2eebb4 100644 --- a/src/v0/destinations/mp/config.js +++ b/src/v0/destinations/mp/config.js @@ -49,7 +49,6 @@ const MP_IDENTIFY_EXCLUSION_LIST = [ ]; const GEO_SOURCE_ALLOWED_VALUES = [null, 'reverse_geocoding']; -const TRACK_MAX_BATCH_SIZE = 50; const IMPORT_MAX_BATCH_SIZE = 2000; const ENGAGE_MAX_BATCH_SIZE = 2000; const GROUPS_MAX_BATCH_SIZE = 200; @@ -68,7 +67,6 @@ module.exports = { MP_IDENTIFY_EXCLUSION_LIST, getCreateDeletionTaskEndpoint, DISTINCT_ID_MAX_BATCH_SIZE, - TRACK_MAX_BATCH_SIZE, IMPORT_MAX_BATCH_SIZE, ENGAGE_MAX_BATCH_SIZE, GROUPS_MAX_BATCH_SIZE, diff --git a/src/v0/destinations/mp/transform.js b/src/v0/destinations/mp/transform.js index 09a7862f9a3..2065764b989 100644 --- a/src/v0/destinations/mp/transform.js +++ b/src/v0/destinations/mp/transform.js @@ -24,7 +24,6 @@ const { mappingConfig, BASE_ENDPOINT, BASE_ENDPOINT_EU, - TRACK_MAX_BATCH_SIZE, IMPORT_MAX_BATCH_SIZE, ENGAGE_MAX_BATCH_SIZE, GROUPS_MAX_BATCH_SIZE, @@ -47,21 +46,19 @@ const mPEventPropertiesConfigJson = mappingConfig[ConfigCategory.EVENT_PROPERTIE const setImportCredentials = (destConfig) => { const endpoint = destConfig.dataResidency === 'eu' ? `${BASE_ENDPOINT_EU}/import/` : `${BASE_ENDPOINT}/import/`; - const headers = { 'Content-Type': 'application/json' }; const params = { strict: destConfig.strictMode ? 1 : 0 }; - const { apiSecret, serviceAccountUserName, serviceAccountSecret, projectId } = destConfig; - if (apiSecret) { - headers.Authorization = `Basic ${base64Convertor(`${apiSecret}:`)}`; + const { serviceAccountUserName, serviceAccountSecret, projectId, token } = destConfig; + let credentials; + if (token) { + credentials = `${token}:`; } else if (serviceAccountUserName && serviceAccountSecret && projectId) { - headers.Authorization = `Basic ${base64Convertor( - `${serviceAccountUserName}:${serviceAccountSecret}`, - )}`; + credentials = `${serviceAccountUserName}:${serviceAccountSecret}`; params.projectId = projectId; - } else { - throw new InstrumentationError( - 'Event timestamp is older than 5 days and no API secret or service account credentials (i.e. username, secret and projectId) are provided in destination configuration', - ); } + const headers = { + 'Content-Type': 'application/json', + Authorization: `Basic ${base64Convertor(credentials)}`, + }; return { endpoint, headers, params }; }; @@ -70,46 +67,34 @@ const responseBuilderSimple = (payload, message, eventType, destConfig) => { response.method = defaultPostRequestConfig.requestMethod; response.userId = message.userId || message.anonymousId; response.body.JSON_ARRAY = { batch: JSON.stringify([removeUndefinedValues(payload)]) }; - const { apiSecret, serviceAccountUserName, serviceAccountSecret, projectId, dataResidency } = - destConfig; + const { dataResidency } = destConfig; const duration = getTimeDifference(message.timestamp); + + const setCredentials = () => { + const credentials = setImportCredentials(destConfig); + response.endpoint = credentials.endpoint; + response.headers = credentials.headers; + response.params = { + project_id: credentials.params?.projectId, + strict: credentials.params.strict, + }; + }; + switch (eventType) { case EventType.ALIAS: case EventType.TRACK: case EventType.SCREEN: - case EventType.PAGE: - if ( - !apiSecret && - !(serviceAccountUserName && serviceAccountSecret && projectId) && - duration.days <= 5 - ) { - response.endpoint = - dataResidency === 'eu' ? `${BASE_ENDPOINT_EU}/track/` : `${BASE_ENDPOINT}/track/`; - response.headers = {}; - } else if (duration.years > 5) { + case EventType.PAGE: { + if (duration.years > 5) { throw new InstrumentationError('Event timestamp should be within last 5 years'); - } else { - const credentials = setImportCredentials(destConfig); - response.endpoint = credentials.endpoint; - response.headers = credentials.headers; - response.params = { - project_id: credentials.params?.projectId, - strict: credentials.params.strict, - }; - break; } + setCredentials(); break; - case 'merge': - // eslint-disable-next-line no-case-declarations - const credentials = setImportCredentials(destConfig); - response.endpoint = credentials.endpoint; - response.headers = credentials.headers; - response.params = { - project_id: credentials.params?.projectId, - strict: credentials.params.strict, - }; + } + case 'merge': { + setCredentials(); break; - + } default: response.endpoint = dataResidency === 'eu' ? `${BASE_ENDPOINT_EU}/engage/` : `${BASE_ENDPOINT}/engage/`; @@ -484,7 +469,6 @@ const processRouterDest = async (inputs, reqMetadata) => { const batchSize = { engage: 0, groups: 0, - track: 0, import: 0, }; @@ -516,23 +500,16 @@ const processRouterDest = async (inputs, reqMetadata) => { ); transformedPayloads = lodash.flatMap(transformedPayloads); - const { engageEvents, groupsEvents, trackEvents, importEvents, batchErrorRespList } = + const { engageEvents, groupsEvents, importEvents, batchErrorRespList } = groupEventsByEndpoint(transformedPayloads); const engageRespList = batchEvents(engageEvents, ENGAGE_MAX_BATCH_SIZE, reqMetadata); const groupsRespList = batchEvents(groupsEvents, GROUPS_MAX_BATCH_SIZE, reqMetadata); - const trackRespList = batchEvents(trackEvents, TRACK_MAX_BATCH_SIZE, reqMetadata); const importRespList = batchEvents(importEvents, IMPORT_MAX_BATCH_SIZE, reqMetadata); - const batchSuccessRespList = [ - ...engageRespList, - ...groupsRespList, - ...trackRespList, - ...importRespList, - ]; + const batchSuccessRespList = [...engageRespList, ...groupsRespList, ...importRespList]; batchSize.engage += engageRespList.length; batchSize.groups += groupsRespList.length; - batchSize.track += trackRespList.length; batchSize.import += importRespList.length; return [...batchSuccessRespList, ...batchErrorRespList]; diff --git a/src/v0/destinations/mp/util.js b/src/v0/destinations/mp/util.js index d564e805ad2..b2807d6e119 100644 --- a/src/v0/destinations/mp/util.js +++ b/src/v0/destinations/mp/util.js @@ -136,7 +136,7 @@ const createIdentifyResponse = (message, type, destination, responseBuilderSimpl * @returns */ const isImportAuthCredentialsAvailable = (destination) => - destination.Config.apiSecret || + destination.Config.token || (destination.Config.serviceAccountSecret && destination.Config.serviceAccountUserName && destination.Config.projectId); @@ -179,7 +179,6 @@ const groupEventsByEndpoint = (events) => { const eventMap = { engage: [], groups: [], - track: [], import: [], }; const batchErrorRespList = []; @@ -204,7 +203,6 @@ const groupEventsByEndpoint = (events) => { return { engageEvents: eventMap.engage, groupsEvents: eventMap.groups, - trackEvents: eventMap.track, importEvents: eventMap.import, batchErrorRespList, }; @@ -349,7 +347,6 @@ const generatePageOrScreenCustomEventName = (message, userDefinedEventTemplate) * @param {Object} batchSize - The object containing the batch size for different endpoints. * @param {number} batchSize.engage - The batch size for engage endpoint. * @param {number} batchSize.groups - The batch size for group endpoint. - * @param {number} batchSize.track - The batch size for track endpoint. * @param {number} batchSize.import - The batch size for import endpoint. * @param {string} destinationId - The ID of the destination. * @returns {void} @@ -361,9 +358,6 @@ const recordBatchSizeMetrics = (batchSize, destinationId) => { stats.gauge('mixpanel_batch_group_pack_size', batchSize.groups, { destination_id: destinationId, }); - stats.gauge('mixpanel_batch_track_pack_size', batchSize.track, { - destination_id: destinationId, - }); stats.gauge('mixpanel_batch_import_pack_size', batchSize.import, { destination_id: destinationId, }); diff --git a/src/v0/destinations/mp/util.test.js b/src/v0/destinations/mp/util.test.js index 40cdb346492..3666081f593 100644 --- a/src/v0/destinations/mp/util.test.js +++ b/src/v0/destinations/mp/util.test.js @@ -18,7 +18,6 @@ describe('Unit test cases for groupEventsByEndpoint', () => { expect(result).toEqual({ engageEvents: [], groupsEvents: [], - trackEvents: [], importEvents: [], batchErrorRespList: [], }); @@ -122,19 +121,6 @@ describe('Unit test cases for groupEventsByEndpoint', () => { }, }, ], - trackEvents: [ - { - message: { - endpoint: '/track', - body: { - JSON_ARRAY: { - batch: '[{prop:4}]', - }, - }, - userId: 'user1', - }, - }, - ], importEvents: [ { message: { diff --git a/src/v0/destinations/pinterest_tag/utils.js b/src/v0/destinations/pinterest_tag/utils.js index 340fba498ed..57d595571fd 100644 --- a/src/v0/destinations/pinterest_tag/utils.js +++ b/src/v0/destinations/pinterest_tag/utils.js @@ -41,8 +41,8 @@ const getHashedValue = (key, value) => { case 'fn': case 'ge': value = Array.isArray(value) - ? value.map((val) => val.toString().toLowerCase()) - : value.toString().toLowerCase(); + ? value.map((val) => val.toString().trim().toLowerCase()) + : value.toString().trim().toLowerCase(); break; case 'ph': // phone numbers should only contain digits & should not contain leading zeros @@ -53,7 +53,7 @@ const getHashedValue = (key, value) => { case 'zp': // zip fields should only contain digits value = Array.isArray(value) - ? value.map((val) => val.toString().replace(/\D/g, '')) + ? value.map((val) => val.toString().trim().replace(/\D/g, '')) : value.toString().replace(/\D/g, ''); break; case 'hashed_maids': diff --git a/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json b/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json index 530d6e392ad..2910f1b44ce 100644 --- a/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json +++ b/src/v0/destinations/tiktok_ads/data/TikTokTrackV2.json @@ -25,10 +25,7 @@ }, { "destKey": "properties.content_type", - "sourceKeys": ["properties.contentType", "properties.content_type"], - "metadata": { - "defaultValue": "product" - } + "sourceKeys": ["properties.contentType", "properties.content_type"] }, { "destKey": "properties.shop_id", diff --git a/src/v0/destinations/tiktok_ads/transformV2.js b/src/v0/destinations/tiktok_ads/transformV2.js index 3bd8699e3ad..4624ec91819 100644 --- a/src/v0/destinations/tiktok_ads/transformV2.js +++ b/src/v0/destinations/tiktok_ads/transformV2.js @@ -31,7 +31,7 @@ const { JSON_MIME_TYPE } = require('../../util/constant'); * @param {*} event * @returns track payload */ -const getTrackResponsePayload = (message, destConfig, event) => { +const getTrackResponsePayload = (message, destConfig, event, setDefaultForContentType = true) => { const payload = constructPayload(message, trackMappingV2); // if contents is not an array converting it into array @@ -53,6 +53,10 @@ const getTrackResponsePayload = (message, destConfig, event) => { if (destConfig.hashUserProperties && isDefinedAndNotNullAndNotEmpty(payload.user)) { payload.user = hashUserField(payload.user); } + // setting content-type default value in case of all standard event except `page-view` + if (!payload.properties?.content_type && setDefaultForContentType) { + payload.properties.content_type = 'product'; + } payload.event = event; // add partner name and return payload return removeUndefinedAndNullValues(payload); @@ -90,13 +94,17 @@ const trackResponseBuilder = async (message, { Config }) => { }); } }); - } else { + } else if (!eventNameMapping[event]) { /* + Custom Event Case -> if there exists no event mapping we will build payload with custom event recieved For custom event we do not want to lower case the event or trim it we just want to send those as it is Doc https://ads.tiktok.com/help/article/standard-events-parameters?lang=en */ - event = eventNameMapping[event] || message.event; - // if there exists no event mapping we will build payload with custom event recieved + event = message.event; + responseList.push(getTrackResponsePayload(message, Config, event, false)); + } else { + // incoming event name is already a standard event name + event = eventNameMapping[event]; responseList.push(getTrackResponsePayload(message, Config, event)); } // set event source and event_source_id diff --git a/src/v0/destinations/yahoo_dsp/util.js b/src/v0/destinations/yahoo_dsp/util.js index 255f84d1c91..54002a3bce9 100644 --- a/src/v0/destinations/yahoo_dsp/util.js +++ b/src/v0/destinations/yahoo_dsp/util.js @@ -51,7 +51,7 @@ const populateIdentifiers = (audienceList, Config) => { } // here, hashing the data if is not hashed and pushing in the seedList array. if (hashRequired) { - seedList.push(sha256(userTraits[audienceAttribute])); + seedList.push(sha256(userTraits[audienceAttribute].trim())); } else { seedList.push(userTraits[audienceAttribute]); } diff --git a/src/v0/sources/adjust/config.ts b/src/v0/sources/adjust/config.ts new file mode 100644 index 00000000000..d1c6ab8242b --- /dev/null +++ b/src/v0/sources/adjust/config.ts @@ -0,0 +1,16 @@ +export const excludedFieldList = [ + 'activity_kind', + 'event', + 'event_name', + 'gps_adid', + 'idfa', + 'idfv', + 'adid', + 'tracker', + 'tracker_name', + 'app_name', + 'ip_address', + 'tracking_enabled', + 'tracker_token', + 'created_at', +]; diff --git a/src/v0/sources/adjust/mapping.json b/src/v0/sources/adjust/mapping.json new file mode 100644 index 00000000000..60ea66281e4 --- /dev/null +++ b/src/v0/sources/adjust/mapping.json @@ -0,0 +1,52 @@ +[ + { + "sourceKeys": "activity_kind", + "destKeys": "properties.activity_kind" + }, + { + "sourceKeys": "event", + "destKeys": "properties.event_token" + }, + { + "sourceKeys": "event_name", + "destKeys": "event" + }, + { + "sourceKeys": "gps_adid", + "destKeys": "properties.gps_adid" + }, + { + "sourceKeys": "idfa", + "destKeys": "context.device.advertisingId" + }, + { + "sourceKeys": "idfv", + "destKeys": "context.device.id" + }, + { + "sourceKeys": "adid", + "destKeys": "context.device.id " + }, + { + "sourceKeys": "tracker", + "destKeys": "properties.tracker" + }, + { + "sourceKeys": "tracker_name", + "destKeys": "properties.tracker_name" + }, + { "sourceKeys": "tracker_token", "destKeys": "properties.tracker_token" }, + + { + "sourceKeys": "app_name", + "destKeys": "context.app.name" + }, + { + "sourceKeys": "ip_address", + "destKeys": ["context.ip", "request_ip"] + }, + { + "sourceKeys": "tracking_enabled", + "destKeys": "properties.tracking_enabled" + } +] diff --git a/src/v0/sources/adjust/transform.js b/src/v0/sources/adjust/transform.js new file mode 100644 index 00000000000..8568622aeba --- /dev/null +++ b/src/v0/sources/adjust/transform.js @@ -0,0 +1,61 @@ +const lodash = require('lodash'); +const path = require('path'); +const fs = require('fs'); +const { TransformationError, structuredLogger: logger } = require('@rudderstack/integrations-lib'); +const Message = require('../message'); +const { CommonUtils } = require('../../../util/common'); +const { excludedFieldList } = require('./config'); +const { extractCustomFields, generateUUID } = require('../../util'); + +// ref : https://help.adjust.com/en/article/global-callbacks#general-recommended-placeholders +// import mapping json using JSON.parse to preserve object key order +const mapping = JSON.parse(fs.readFileSync(path.resolve(__dirname, './mapping.json'), 'utf-8')); + +const formatProperties = (input) => { + const { query_parameters: qParams } = input; + logger.debug(`[Adjust] Input event: query_params: ${JSON.stringify(qParams)}`); + if (!qParams) { + throw new TransformationError('Query_parameters is missing'); + } + const formattedOutput = {}; + Object.entries(qParams).forEach(([key, [value]]) => { + formattedOutput[key] = value; + }); + return formattedOutput; +}; + +const processEvent = (inputEvent) => { + const message = new Message(`Adjust`); + const event = lodash.cloneDeep(inputEvent); + const formattedPayload = formatProperties(event); + // event type is always track + const eventType = 'track'; + message.setEventType(eventType); + message.setPropertiesV2(formattedPayload, mapping); + let customProperties = {}; + customProperties = extractCustomFields( + formattedPayload, + customProperties, + 'root', + excludedFieldList, + ); + message.properties = { ...message.properties, ...customProperties }; + + if (formattedPayload.created_at) { + const ts = new Date(formattedPayload.created_at * 1000).toISOString(); + message.setProperty('originalTimestamp', ts); + message.setProperty('timestamp', ts); + } + + // adjust does not has the concept of user but we need to set some random anonymousId in order to make the server accept the message + message.anonymousId = generateUUID(); + return message; +}; + +// This fucntion just converts the incoming payload to array of already not and sends it to processEvent +const process = (events) => { + const eventsArray = CommonUtils.toArray(events); + return eventsArray.map(processEvent); +}; + +module.exports = { process }; diff --git a/src/v0/util/facebookUtils/index.js b/src/v0/util/facebookUtils/index.js index c7753d255fb..7462320ccad 100644 --- a/src/v0/util/facebookUtils/index.js +++ b/src/v0/util/facebookUtils/index.js @@ -292,7 +292,13 @@ const formingFinalResponse = ( throw new TransformationError('Payload could not be constructed'); }; +const isHtmlFormat = (string) => { + const htmlTags = /<(?!(!doctype\s*html|html))\b[^>]*>[\S\s]*?<\/[^>]*>/i; + return htmlTags.test(string); +}; + module.exports = { + isHtmlFormat, getContentType, getContentCategory, transformedPayloadData, diff --git a/src/v0/util/facebookUtils/index.test.js b/src/v0/util/facebookUtils/index.test.js index 20c4ee59f25..1a2de4ed129 100644 --- a/src/v0/util/facebookUtils/index.test.js +++ b/src/v0/util/facebookUtils/index.test.js @@ -3,6 +3,7 @@ const { fetchUserData, deduceFbcParam, getContentType, + isHtmlFormat, } = require('./index'); const sha256 = require('sha256'); const { MAPPING_CONFIG, CONFIG_CATEGORIES } = require('../../destinations/facebook_pixel/config'); @@ -639,3 +640,53 @@ describe('getContentType', () => { expect(result).toBe(defaultValue); }); }); + +describe('isHtmlFormat', () => { + it('should return false for Json', () => { + expect(isHtmlFormat('{"a": 1, "b":2}')).toBe(false); + }); + + it('should return false for empty Json', () => { + expect(isHtmlFormat('{}')).toBe(false); + }); + + it('should return false for undefined', () => { + expect(isHtmlFormat(undefined)).toBe(false); + }); + + it('should return false for null', () => { + expect(isHtmlFormat(null)).toBe(false); + }); + + it('should return false for empty array', () => { + expect(isHtmlFormat([])).toBe(false); + }); + + it('should return true for html doctype', () => { + expect( + isHtmlFormat( + '
', + ), + ).toBe(true); + }); + + it('should return true for html', () => { + expect( + isHtmlFormat( + '