From 04c069486bdd3c101906fa6c621e983090fcab25 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Thu, 17 Oct 2024 18:25:20 +0530 Subject: [PATCH 01/32] feat: sources v2 spec support along with adapters --- src/controllers/__tests__/source.test.ts | 111 +++++++++++- src/controllers/util/index.test.ts | 163 ++++++++++++++++-- src/controllers/util/index.ts | 62 ++++++- src/interfaces/SourceService.ts | 8 +- src/middlewares/routeActivation.ts | 15 ++ .../__tests__/nativeIntegration.test.ts | 20 ++- src/services/source/nativeIntegration.ts | 18 +- src/types/index.ts | 22 +++ test/apitests/service.api.test.ts | 7 + 9 files changed, 390 insertions(+), 36 deletions(-) diff --git a/src/controllers/__tests__/source.test.ts b/src/controllers/__tests__/source.test.ts index 565f39d559..72bee83282 100644 --- a/src/controllers/__tests__/source.test.ts +++ b/src/controllers/__tests__/source.test.ts @@ -6,6 +6,7 @@ import { applicationRoutes } from '../../routes'; import { NativeIntegrationSourceService } from '../../services/source/nativeIntegration'; import { ServiceSelector } from '../../helpers/serviceSelector'; import { ControllerUtility } from '../util/index'; +import { SourceInputConversionResult } from '../../types'; let server: any; const OLD_ENV = process.env; @@ -38,6 +39,19 @@ const getData = () => { return [{ event: { a: 'b1' } }, { event: { a: 'b2' } }]; }; +const getV2Data = () => { + return [ + { request: { body: '{"a": "b"}' }, source: { id: 1 } }, + { request: { body: '{"a": "b"}' }, source: { id: 1 } }, + ]; +}; + +const getConvertedData = () => { + return getData().map((eventInstance) => { + return { output: eventInstance } as SourceInputConversionResult; + }); +}; + describe('Source controller tests', () => { describe('V0 Source transform tests', () => { test('successful source transform', async () => { @@ -49,7 +63,7 @@ describe('Source controller tests', () => { mockSourceService.sourceTransformRoutine = jest .fn() .mockImplementation((i, s, v, requestMetadata) => { - expect(i).toEqual(getData()); + expect(i).toEqual(getConvertedData()); expect(s).toEqual(sourceType); expect(v).toEqual(version); return testOutput; @@ -66,7 +80,7 @@ describe('Source controller tests', () => { expect(s).toEqual(sourceType); expect(v).toEqual(version); expect(e).toEqual(getData()); - return { implementationVersion: version, input: e }; + return { implementationVersion: version, input: getConvertedData() }; }); const response = await request(server) @@ -139,7 +153,7 @@ describe('Source controller tests', () => { mockSourceService.sourceTransformRoutine = jest .fn() .mockImplementation((i, s, v, requestMetadata) => { - expect(i).toEqual(getData()); + expect(i).toEqual(getConvertedData()); expect(s).toEqual(sourceType); expect(v).toEqual(version); return testOutput; @@ -156,7 +170,7 @@ describe('Source controller tests', () => { expect(s).toEqual(sourceType); expect(v).toEqual(version); expect(e).toEqual(getData()); - return { implementationVersion: version, input: e }; + return { implementationVersion: version, input: getConvertedData() }; }); const response = await request(server) @@ -217,4 +231,93 @@ describe('Source controller tests', () => { expect(adaptInputToVersionSpy).toHaveBeenCalledTimes(1); }); }); + + describe('V2 Source transform tests', () => { + test('successful source transform', async () => { + const sourceType = '__rudder_test__'; + const version = 'v2'; + const testOutput = [{ event: { a: 'b' }, source: { id: 'id' } }]; + + const mockSourceService = new NativeIntegrationSourceService(); + mockSourceService.sourceTransformRoutine = jest + .fn() + .mockImplementation((i, s, v, requestMetadata) => { + expect(i).toEqual(getConvertedData()); + expect(s).toEqual(sourceType); + expect(v).toEqual(version); + return testOutput; + }); + const getNativeSourceServiceSpy = jest + .spyOn(ServiceSelector, 'getNativeSourceService') + .mockImplementation(() => { + return mockSourceService; + }); + + const adaptInputToVersionSpy = jest + .spyOn(ControllerUtility, 'adaptInputToVersion') + .mockImplementation((s, v, e) => { + expect(s).toEqual(sourceType); + expect(v).toEqual(version); + expect(e).toEqual(getV2Data()); + return { implementationVersion: version, input: getConvertedData() }; + }); + + const response = await request(server) + .post('/v2/sources/__rudder_test__') + .set('Accept', 'application/json') + .send(getV2Data()); + + expect(response.status).toEqual(200); + expect(response.body).toEqual(testOutput); + + expect(response.header['apiversion']).toEqual('2'); + + expect(getNativeSourceServiceSpy).toHaveBeenCalledTimes(1); + expect(adaptInputToVersionSpy).toHaveBeenCalledTimes(1); + expect(mockSourceService.sourceTransformRoutine).toHaveBeenCalledTimes(1); + }); + + test('failing source transform', async () => { + const sourceType = '__rudder_test__'; + const version = 'v2'; + const mockSourceService = new NativeIntegrationSourceService(); + const getNativeSourceServiceSpy = jest + .spyOn(ServiceSelector, 'getNativeSourceService') + .mockImplementation(() => { + return mockSourceService; + }); + + const adaptInputToVersionSpy = jest + .spyOn(ControllerUtility, 'adaptInputToVersion') + .mockImplementation((s, v, e) => { + expect(s).toEqual(sourceType); + expect(v).toEqual(version); + expect(e).toEqual(getV2Data()); + throw new Error('test error'); + }); + + const response = await request(server) + .post('/v2/sources/__rudder_test__') + .set('Accept', 'application/json') + .send(getV2Data()); + + const expectedResp = [ + { + error: 'test error', + statTags: { + errorCategory: 'transformation', + }, + statusCode: 500, + }, + ]; + + expect(response.status).toEqual(200); + expect(response.body).toEqual(expectedResp); + + expect(response.header['apiversion']).toEqual('2'); + + expect(getNativeSourceServiceSpy).toHaveBeenCalledTimes(1); + expect(adaptInputToVersionSpy).toHaveBeenCalledTimes(1); + }); + }); }); diff --git a/src/controllers/util/index.test.ts b/src/controllers/util/index.test.ts index 6065920846..6ab2336b71 100644 --- a/src/controllers/util/index.test.ts +++ b/src/controllers/util/index.test.ts @@ -19,9 +19,9 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: undefined, input: [ - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, ], }; @@ -40,9 +40,9 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: 'v0', input: [ - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, ], }; @@ -71,16 +71,22 @@ describe('adaptInputToVersion', () => { implementationVersion: 'v1', input: [ { - event: { key1: 'val1', key2: 'val2' }, - source: { id: 'source_id', config: { configField1: 'configVal1' } }, + output: { + event: { key1: 'val1', key2: 'val2' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, }, { - event: { key1: 'val1', key2: 'val2' }, - source: { id: 'source_id', config: { configField1: 'configVal1' } }, + output: { + event: { key1: 'val1', key2: 'val2' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, }, { - event: { key1: 'val1', key2: 'val2' }, - source: { id: 'source_id', config: { configField1: 'configVal1' } }, + output: { + event: { key1: 'val1', key2: 'val2' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, }, ], }; @@ -100,9 +106,9 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: 'v1', input: [ - { event: { key1: 'val1', key2: 'val2' }, source: undefined }, - { event: { key1: 'val1', key2: 'val2' }, source: undefined }, - { event: { key1: 'val1', key2: 'val2' }, source: undefined }, + { output: { event: { key1: 'val1', key2: 'val2' }, source: undefined } }, + { output: { event: { key1: 'val1', key2: 'val2' }, source: undefined } }, + { output: { event: { key1: 'val1', key2: 'val2' }, source: undefined } }, ], }; @@ -131,9 +137,130 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: 'v0', input: [ - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, - { key1: 'val1', key2: 'val2' }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + { output: { key1: 'val1', key2: 'val2' } }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + + it('should convert input from v2 to v0 format when the request version is v2 and the implementation version is v0', () => { + const sourceType = 'pipedream'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v0', + input: [ + { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, + { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, + { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + + it('should convert input from v2 to v1 format when the request version is v2 and the implementation version is v1', () => { + const sourceType = 'webhook'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v1', + input: [ + { + output: { + event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, ], }; diff --git a/src/controllers/util/index.ts b/src/controllers/util/index.ts index c5bf7ab358..b562381ed6 100644 --- a/src/controllers/util/index.ts +++ b/src/controllers/util/index.ts @@ -10,6 +10,8 @@ import { RouterTransformationRequestData, RudderMessage, SourceInput, + SourceInputConversionResult, + SourceInputV2, } from '../../types'; import { getValueFromMessage } from '../../v0/util'; import genericFieldMap from '../../v0/util/data/GenericFieldMapping.json'; @@ -45,28 +47,72 @@ export class ControllerUtility { return this.sourceVersionMap; } - private static convertSourceInputv1Tov0(sourceEvents: SourceInput[]): NonNullable[] { - return sourceEvents.map((sourceEvent) => sourceEvent.event); + private static convertSourceInputv1Tov0( + sourceEvents: SourceInput[], + ): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => ({ + output: sourceEvent.event as NonNullable, + })); } - private static convertSourceInputv0Tov1(sourceEvents: unknown[]): SourceInput[] { - return sourceEvents.map( - (sourceEvent) => ({ event: sourceEvent, source: undefined }) as SourceInput, - ); + private static convertSourceInputv0Tov1( + sourceEvents: unknown[], + ): SourceInputConversionResult[] { + return sourceEvents.map((sourceEvent) => ({ + output: { event: sourceEvent, source: undefined } as SourceInput, + })); + } + + private static convertSourceInputv2Tov0( + sourceEvents: SourceInputV2[], + ): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => { + try { + const v0Event = JSON.parse(sourceEvent.request.body); + v0Event.query_parameters = sourceEvent.request.query_parameters; + return { output: v0Event }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v2 to v0 spec'); + return { output: {} as NonNullable, conversionError }; + } + }); + } + + private static convertSourceInputv2Tov1( + sourceEvents: SourceInputV2[], + ): SourceInputConversionResult[] { + return sourceEvents.map((sourceEvent) => { + try { + const v1Event = { event: JSON.parse(sourceEvent.request.body), source: sourceEvent.source }; + v1Event.event.query_parameters = sourceEvent.request.query_parameters; + return { output: v1Event }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v2 to v1 spec'); + return { output: {} as SourceInput, conversionError }; + } + }); } public static adaptInputToVersion( sourceType: string, requestVersion: string, input: NonNullable[], - ): { implementationVersion: string; input: NonNullable[] } { + ): { implementationVersion: string; input: SourceInputConversionResult>[] } { const sourceToVersionMap = this.getSourceVersionsMap(); const implementationVersion = sourceToVersionMap.get(sourceType); - let updatedInput: NonNullable[] = input; + let updatedInput: SourceInputConversionResult>[] = input.map((event) => ({ + output: event, + })); if (requestVersion === 'v0' && implementationVersion === 'v1') { updatedInput = this.convertSourceInputv0Tov1(input); } else if (requestVersion === 'v1' && implementationVersion === 'v0') { updatedInput = this.convertSourceInputv1Tov0(input as SourceInput[]); + } else if (requestVersion === 'v2' && implementationVersion === 'v0') { + updatedInput = this.convertSourceInputv2Tov0(input as SourceInputV2[]); + } else if (requestVersion === 'v2' && implementationVersion === 'v1') { + updatedInput = this.convertSourceInputv2Tov1(input as SourceInputV2[]); } return { implementationVersion, input: updatedInput }; } diff --git a/src/interfaces/SourceService.ts b/src/interfaces/SourceService.ts index c7de8cfe8b..32a7125e7a 100644 --- a/src/interfaces/SourceService.ts +++ b/src/interfaces/SourceService.ts @@ -1,10 +1,14 @@ -import { MetaTransferObject, SourceTransformationResponse } from '../types/index'; +import { + MetaTransferObject, + SourceInputConversionResult, + SourceTransformationResponse, +} from '../types/index'; export interface SourceService { getTags(): MetaTransferObject; sourceTransformRoutine( - sourceEvents: NonNullable[], + sourceEvents: SourceInputConversionResult>[], sourceType: string, version: string, requestMetadata: NonNullable, diff --git a/src/middlewares/routeActivation.ts b/src/middlewares/routeActivation.ts index ffb1e15e80..126749b083 100644 --- a/src/middlewares/routeActivation.ts +++ b/src/middlewares/routeActivation.ts @@ -106,4 +106,19 @@ export class RouteActivationMiddleware { RouteActivationMiddleware.shouldActivateRoute(destination, deliveryFilterList), ); } + + // This middleware will be used by source endpoint when we completely deprecate v0, v1 versions. + public static isVersionAllowed(ctx: Context, next: Next) { + const { version } = ctx.params; + if (version === 'v0' || version === 'v1') { + ctx.status = 500; + ctx.body = + '/v0, /v1 versioned endpoints are deprecated. Use /v2 version endpoint. This is probably caused because of source transformation call from an outdated rudder-server version. Please upgrade rudder-server to a minimum of 1.xx.xx version.'; + } else if (version === 'v2') { + next(); + } else { + ctx.status = 404; + ctx.body = 'Path not found. Verify the version of your api call.'; + } + } } diff --git a/src/services/source/__tests__/nativeIntegration.test.ts b/src/services/source/__tests__/nativeIntegration.test.ts index 2ef8129cdc..51bb37f5f1 100644 --- a/src/services/source/__tests__/nativeIntegration.test.ts +++ b/src/services/source/__tests__/nativeIntegration.test.ts @@ -44,7 +44,15 @@ describe('NativeIntegration Source Service', () => { }); const service = new NativeIntegrationSourceService(); - const resp = await service.sourceTransformRoutine(events, sourceType, version, requestMetadata); + const adapterConvertedEvents = events.map((eventInstance) => { + return { output: eventInstance }; + }); + const resp = await service.sourceTransformRoutine( + adapterConvertedEvents, + sourceType, + version, + requestMetadata, + ); expect(resp).toEqual(tresponse); @@ -81,7 +89,15 @@ describe('NativeIntegration Source Service', () => { jest.spyOn(stats, 'increment').mockImplementation(() => {}); const service = new NativeIntegrationSourceService(); - const resp = await service.sourceTransformRoutine(events, sourceType, version, requestMetadata); + const adapterConvertedEvents = events.map((eventInstance) => { + return { output: eventInstance }; + }); + const resp = await service.sourceTransformRoutine( + adapterConvertedEvents, + sourceType, + version, + requestMetadata, + ); expect(resp).toEqual(tresponse); diff --git a/src/services/source/nativeIntegration.ts b/src/services/source/nativeIntegration.ts index 5c89de7b92..58a6a19649 100644 --- a/src/services/source/nativeIntegration.ts +++ b/src/services/source/nativeIntegration.ts @@ -4,6 +4,7 @@ import { ErrorDetailer, MetaTransferObject, RudderMessage, + SourceInputConversionResult, SourceTransformationEvent, SourceTransformationResponse, } from '../../types/index'; @@ -28,7 +29,7 @@ export class NativeIntegrationSourceService implements SourceService { } public async sourceTransformRoutine( - sourceEvents: NonNullable[], + sourceEvents: SourceInputConversionResult>[], sourceType: string, version: string, // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -39,7 +40,20 @@ export class NativeIntegrationSourceService implements SourceService { const respList: SourceTransformationResponse[] = await Promise.all( sourceEvents.map(async (sourceEvent) => { try { - const newSourceEvent = sourceEvent; + if (sourceEvent.conversionError) { + stats.increment('source_transform_errors', { + source: sourceType, + version, + }); + logger.debug(`Error during source Transform: ${sourceEvent.conversionError}`, { + ...logger.getLogMetadata(metaTO.errorDetails), + }); + return SourcePostTransformationService.handleFailureEventsSource( + sourceEvent.conversionError, + metaTO, + ); + } + const newSourceEvent = sourceEvent.output; const { headers } = newSourceEvent; delete newSourceEvent.headers; const respEvents: RudderMessage | RudderMessage[] | SourceTransformationResponse = diff --git a/src/types/index.ts b/src/types/index.ts index 45ec7445c3..0bc2cbc33b 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -355,6 +355,26 @@ type SourceInput = { event: NonNullable[]; source?: Source; }; + +type SourceRequestV2 = { + method: string; + url: string; + proto: string; + body: string; + headers: NonNullable; + query_parameters: NonNullable; +}; + +type SourceInputV2 = { + request: SourceRequestV2; + source?: Source; +}; + +type SourceInputConversionResult = { + output: T; + conversionError?: Error; +}; + export { ComparatorInput, DeliveryJobState, @@ -382,7 +402,9 @@ export { UserDeletionRequest, UserDeletionResponse, SourceInput, + SourceInputV2, Source, + SourceInputConversionResult, UserTransformationLibrary, UserTransformationResponse, UserTransformationServiceResponse, diff --git a/test/apitests/service.api.test.ts b/test/apitests/service.api.test.ts index 9c1d96e7fe..2ad1f323ac 100644 --- a/test/apitests/service.api.test.ts +++ b/test/apitests/service.api.test.ts @@ -78,6 +78,13 @@ describe('features tests', () => { const supportTransformerProxyV1 = JSON.parse(response.text).supportTransformerProxyV1; expect(typeof supportTransformerProxyV1).toBe('boolean'); }); + + test('features upgradedToSourceTransformV2 to be boolean', async () => { + const response = await request(server).get('/features'); + expect(response.status).toEqual(200); + const upgradedToSourceTransformV2 = JSON.parse(response.text).upgradedToSourceTransformV2; + expect(typeof upgradedToSourceTransformV2).toBe('boolean'); + }); }); describe('Api tests with a mock source/destination', () => { From 778b028cb0ba0f9a3b5feefbc11bfb72901fe01b Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Thu, 17 Oct 2024 18:48:48 +0530 Subject: [PATCH 02/32] chore: unwanted file .python-version removed, updated .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 09c536ebb8..624a40d751 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,7 @@ dist # Stores VSCode versions used for testing VSCode extensions .vscode-test +.vscode # yarn v2 .yarn/cache @@ -133,7 +134,7 @@ dist # Others **/.DS_Store .dccache - +.python-version .idea # component test report From de8faba7ed4f908f69485aa2776c71e806fbcc44 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Mon, 21 Oct 2024 13:38:24 +0530 Subject: [PATCH 03/32] chore: lint check github workflow issue for non js files fixed --- .github/workflows/verify.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 115cad4248..e8b1920b87 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -32,11 +32,18 @@ jobs: uses: Ana06/get-changed-files@v1.2 with: token: ${{ secrets.GITHUB_TOKEN }} + + - name: Filter JS/TS Files + run: | + echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' > changed_files.txt + if [ ! -s changed_files.txt ]; then + echo "No JS/TS files to format or lint." + exit 0 + fi - name: Run format Checks run: | - npx prettier ${{steps.files.outputs.added_modified}} --write - + npx prettier --write $(cat changed_files.txt) - run: git diff --exit-code - name: Formatting Error message From 3f8e75d984258bb88b9458ccfbc101ddcbb98c05 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Fri, 25 Oct 2024 11:46:47 +0530 Subject: [PATCH 04/32] chore: refactoring version conversion adapter to readable format --- .github/workflows/verify.yml | 3 +- src/controllers/source.ts | 1 + .../util/conversionStrategies/abstractions.ts | 5 ++ .../conversionStrategies/strategyDefault.ts | 15 +++++ .../conversionStrategies/strategyV0ToV1.ts | 11 +++ .../conversionStrategies/strategyV1ToV0.ts | 10 +++ .../conversionStrategies/strategyV1ToV2.ts | 37 ++++++++++ .../conversionStrategies/strategyV2ToV0.ts | 18 +++++ .../conversionStrategies/strategyV2ToV1.ts | 18 +++++ src/controllers/util/index.ts | 67 +++++++++++++++---- src/controllers/util/versionConversion.ts | 65 ++++++++++++++++++ src/types/index.ts | 1 + 12 files changed, 236 insertions(+), 15 deletions(-) create mode 100644 src/controllers/util/conversionStrategies/abstractions.ts create mode 100644 src/controllers/util/conversionStrategies/strategyDefault.ts create mode 100644 src/controllers/util/conversionStrategies/strategyV0ToV1.ts create mode 100644 src/controllers/util/conversionStrategies/strategyV1ToV0.ts create mode 100644 src/controllers/util/conversionStrategies/strategyV1ToV2.ts create mode 100644 src/controllers/util/conversionStrategies/strategyV2ToV0.ts create mode 100644 src/controllers/util/conversionStrategies/strategyV2ToV1.ts create mode 100644 src/controllers/util/versionConversion.ts diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index e8b1920b87..4fca34673a 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -32,7 +32,7 @@ jobs: uses: Ana06/get-changed-files@v1.2 with: token: ${{ secrets.GITHUB_TOKEN }} - + - name: Filter JS/TS Files run: | echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' > changed_files.txt @@ -45,7 +45,6 @@ jobs: run: | npx prettier --write $(cat changed_files.txt) - run: git diff --exit-code - - name: Formatting Error message if: ${{ failure() }} run: | diff --git a/src/controllers/source.ts b/src/controllers/source.ts index 230636f193..8b6d2d70f8 100644 --- a/src/controllers/source.ts +++ b/src/controllers/source.ts @@ -12,6 +12,7 @@ export class SourceController { const events = ctx.request.body as object[]; const { version, source }: { version: string; source: string } = ctx.params; const integrationService = ServiceSelector.getNativeSourceService(); + try { const { implementationVersion, input } = ControllerUtility.adaptInputToVersion( source, diff --git a/src/controllers/util/conversionStrategies/abstractions.ts b/src/controllers/util/conversionStrategies/abstractions.ts new file mode 100644 index 0000000000..f25bc374a2 --- /dev/null +++ b/src/controllers/util/conversionStrategies/abstractions.ts @@ -0,0 +1,5 @@ +import { SourceInputConversionResult } from '../../../types'; + +export abstract class VersionConversionStrategy { + abstract convert(sourceEvents: I[]): SourceInputConversionResult[]; +} diff --git a/src/controllers/util/conversionStrategies/strategyDefault.ts b/src/controllers/util/conversionStrategies/strategyDefault.ts new file mode 100644 index 0000000000..44b9fbf312 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyDefault.ts @@ -0,0 +1,15 @@ +import { SourceInputConversionResult } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyDefault extends VersionConversionStrategy< + NonNullable, + NonNullable +> { + convert( + sourceEvents: NonNullable[], + ): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => ({ + output: sourceEvent, + })); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV0ToV1.ts b/src/controllers/util/conversionStrategies/strategyV0ToV1.ts new file mode 100644 index 0000000000..28f170c4dd --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV0ToV1.ts @@ -0,0 +1,11 @@ +import { SourceInput, SourceInputConversionResult } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV0ToV1 extends VersionConversionStrategy, SourceInput> { + convert(sourceEvents: NonNullable[]): SourceInputConversionResult[] { + // This should be deprecated along with v0-webhook-rudder-server deprecation + return sourceEvents.map((sourceEvent) => ({ + output: { event: sourceEvent, source: undefined } as SourceInput, + })); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV1ToV0.ts b/src/controllers/util/conversionStrategies/strategyV1ToV0.ts new file mode 100644 index 0000000000..d0894099a5 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV1ToV0.ts @@ -0,0 +1,10 @@ +import { SourceInput, SourceInputConversionResult } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV1ToV0 extends VersionConversionStrategy> { + convert(sourceEvents: SourceInput[]): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => ({ + output: sourceEvent.event as NonNullable, + })); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV1ToV2.ts b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts new file mode 100644 index 0000000000..0db03cc811 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts @@ -0,0 +1,37 @@ +import { + SourceInput, + SourceInputConversionResult, + SourceInputV2, + SourceRequestV2, +} from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV1ToV2 extends VersionConversionStrategy { + convert(sourceEvents: SourceInput[]): SourceInputConversionResult[] { + // Currently this is not being used + // Hold off on testing this until atleast one v2 source has been implemented + return sourceEvents.map((sourceEvent) => { + try { + const sourceRequest: SourceRequestV2 = { + method: '', + url: '', + proto: '', + headers: {}, + query_parameters: {}, + body: JSON.stringify(sourceEvent.event), + }; + const sourceInputV2: SourceInputV2 = { + request: sourceRequest, + source: sourceEvent.source, + }; + return { + output: sourceInputV2, + }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v1 to v2 spec'); + return { output: {} as SourceInputV2, conversionError }; + } + }); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV0.ts b/src/controllers/util/conversionStrategies/strategyV2ToV0.ts new file mode 100644 index 0000000000..031039e538 --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV2ToV0.ts @@ -0,0 +1,18 @@ +import { SourceInputConversionResult, SourceInputV2 } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV2ToV0 extends VersionConversionStrategy> { + convert(sourceEvents: SourceInputV2[]): SourceInputConversionResult>[] { + return sourceEvents.map((sourceEvent) => { + try { + const v0Event = JSON.parse(sourceEvent.request.body); + v0Event.query_parameters = sourceEvent.request.query_parameters; + return { output: v0Event }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v2 to v0 spec'); + return { output: {} as NonNullable, conversionError }; + } + }); + } +} diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts new file mode 100644 index 0000000000..7ddafd782e --- /dev/null +++ b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts @@ -0,0 +1,18 @@ +import { SourceInput, SourceInputConversionResult, SourceInputV2 } from '../../../types'; +import { VersionConversionStrategy } from './abstractions'; + +export class StrategyV2ToV1 extends VersionConversionStrategy { + convert(sourceEvents: SourceInputV2[]): SourceInputConversionResult[] { + return sourceEvents.map((sourceEvent) => { + try { + const v1Event = { event: JSON.parse(sourceEvent.request.body), source: sourceEvent.source }; + v1Event.event.query_parameters = sourceEvent.request.query_parameters; + return { output: v1Event }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v2 to v1 spec'); + return { output: {} as SourceInput, conversionError }; + } + }); + } +} diff --git a/src/controllers/util/index.ts b/src/controllers/util/index.ts index b562381ed6..b6fa909d27 100644 --- a/src/controllers/util/index.ts +++ b/src/controllers/util/index.ts @@ -12,10 +12,12 @@ import { SourceInput, SourceInputConversionResult, SourceInputV2, + SourceRequestV2, } from '../../types'; import { getValueFromMessage } from '../../v0/util'; import genericFieldMap from '../../v0/util/data/GenericFieldMapping.json'; import { EventType, MappedToDestinationKey } from '../../constants'; +import { versionConversionFactory } from './versionConversion'; export class ControllerUtility { private static sourceVersionMap: Map = new Map(); @@ -55,6 +57,36 @@ export class ControllerUtility { })); } + private static convertSourceInputv1Tov2( + sourceEvents: SourceInput[], + ): SourceInputConversionResult[] { + // Currently this is not being used + // Hold off on testing this until atleast one v2 source has been implemented + return sourceEvents.map((sourceEvent) => { + try { + const sourceRequest: SourceRequestV2 = { + method: '', + url: '', + proto: '', + headers: {}, + query_parameters: {}, + body: JSON.stringify(sourceEvent.event), + }; + const sourceInputV2: SourceInputV2 = { + request: sourceRequest, + source: sourceEvent.source, + }; + return { + output: sourceInputV2, + }; + } catch (err) { + const conversionError = + err instanceof Error ? err : new Error('error converting v1 to v2 spec'); + return { output: {} as SourceInputV2, conversionError }; + } + }); + } + private static convertSourceInputv0Tov1( sourceEvents: unknown[], ): SourceInputConversionResult[] { @@ -102,19 +134,28 @@ export class ControllerUtility { ): { implementationVersion: string; input: SourceInputConversionResult>[] } { const sourceToVersionMap = this.getSourceVersionsMap(); const implementationVersion = sourceToVersionMap.get(sourceType); - let updatedInput: SourceInputConversionResult>[] = input.map((event) => ({ - output: event, - })); - if (requestVersion === 'v0' && implementationVersion === 'v1') { - updatedInput = this.convertSourceInputv0Tov1(input); - } else if (requestVersion === 'v1' && implementationVersion === 'v0') { - updatedInput = this.convertSourceInputv1Tov0(input as SourceInput[]); - } else if (requestVersion === 'v2' && implementationVersion === 'v0') { - updatedInput = this.convertSourceInputv2Tov0(input as SourceInputV2[]); - } else if (requestVersion === 'v2' && implementationVersion === 'v1') { - updatedInput = this.convertSourceInputv2Tov1(input as SourceInputV2[]); - } - return { implementationVersion, input: updatedInput }; + + const conversionStrategy = versionConversionFactory.getStrategy( + requestVersion, + implementationVersion, + ); + return { implementationVersion, input: conversionStrategy.convert(input) }; + + // let updatedInput: SourceInputConversionResult>[] = input.map((event) => ({ + // output: event, + // })); + // if (requestVersion === 'v0' && implementationVersion === 'v1') { + // updatedInput = this.convertSourceInputv0Tov1(input); + // } else if (requestVersion === 'v1' && implementationVersion === 'v0') { + // updatedInput = this.convertSourceInputv1Tov0(input as SourceInput[]); + // } else if (requestVersion === 'v1' && implementationVersion === 'v2') { + // updatedInput = this.convertSourceInputv1Tov2(input as SourceInput[]); + // } else if (requestVersion === 'v2' && implementationVersion === 'v0') { + // updatedInput = this.convertSourceInputv2Tov0(input as SourceInputV2[]); + // } else if (requestVersion === 'v2' && implementationVersion === 'v1') { + // updatedInput = this.convertSourceInputv2Tov1(input as SourceInputV2[]); + // } + // return { implementationVersion, input: updatedInput }; } private static getCompatibleStatusCode(status: number): number { diff --git a/src/controllers/util/versionConversion.ts b/src/controllers/util/versionConversion.ts new file mode 100644 index 0000000000..3058531f57 --- /dev/null +++ b/src/controllers/util/versionConversion.ts @@ -0,0 +1,65 @@ +import { VersionConversionStrategy } from './conversionStrategies/abstractions'; +import { StrategyDefault } from './conversionStrategies/strategyDefault'; +import { StrategyV0ToV1 } from './conversionStrategies/strategyV0ToV1'; +import { StrategyV1ToV0 } from './conversionStrategies/strategyV1ToV0'; +import { StrategyV1ToV2 } from './conversionStrategies/strategyV1ToV2'; +import { StrategyV2ToV0 } from './conversionStrategies/strategyV2ToV0'; +import { StrategyV2ToV1 } from './conversionStrategies/strategyV2ToV1'; + +export class VersionConversionFactory { + private strategyCache: Map> = new Map(); + + private getCase(requestVersion: string, implementationVersion: string) { + return `${String(requestVersion)}-to-${String(implementationVersion)}`; + } + + public getStrategy( + requestVersion: string, + implementationVersion: string, + ): VersionConversionStrategy { + const versionCase = this.getCase(requestVersion, implementationVersion); + + if (this.strategyCache.has(versionCase)) { + const cachedStrategy = this.strategyCache.get(versionCase); + if (cachedStrategy) { + return cachedStrategy; + } + } + + let strategy: VersionConversionStrategy; + + switch (versionCase) { + case 'v0-to-v1': + strategy = new StrategyV0ToV1(); + break; + + case 'v1-to-v0': + strategy = new StrategyV1ToV0(); + break; + + case 'v1-to-v2': + strategy = new StrategyV1ToV2(); + break; + + case 'v2-to-v0': + strategy = new StrategyV2ToV0(); + break; + + case 'v2-to-v1': + strategy = new StrategyV2ToV1(); + break; + + default: + strategy = new StrategyDefault(); + break; + } + + if (strategy) { + this.strategyCache[versionCase] = strategy; + } + + return strategy; + } +} + +export const versionConversionFactory = new VersionConversionFactory(); diff --git a/src/types/index.ts b/src/types/index.ts index 0bc2cbc33b..54ff3a994e 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -403,6 +403,7 @@ export { UserDeletionResponse, SourceInput, SourceInputV2, + SourceRequestV2, Source, SourceInputConversionResult, UserTransformationLibrary, From 5705f2e28a5c8e123a4018b2c6815f4143b4df67 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Mon, 28 Oct 2024 21:16:06 +0530 Subject: [PATCH 05/32] chore: stricter types, extra test cases for v2 --- .gitignore | 2 +- .../conversionStrategies/strategyV1ToV2.ts | 23 ++-- .../conversionStrategies/strategyV2ToV1.ts | 4 +- src/controllers/util/index.test.ts | 124 ++++++++++++++++++ src/controllers/util/index.ts | 97 -------------- src/middlewares/routeActivation.ts | 15 --- src/types/index.ts | 15 ++- 7 files changed, 151 insertions(+), 129 deletions(-) diff --git a/.gitignore b/.gitignore index 624a40d751..84421f49d9 100644 --- a/.gitignore +++ b/.gitignore @@ -139,4 +139,4 @@ dist # component test report test_reports/ -temp/ +temp/ \ No newline at end of file diff --git a/src/controllers/util/conversionStrategies/strategyV1ToV2.ts b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts index 0db03cc811..b4f04ef858 100644 --- a/src/controllers/util/conversionStrategies/strategyV1ToV2.ts +++ b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts @@ -8,21 +8,26 @@ import { VersionConversionStrategy } from './abstractions'; export class StrategyV1ToV2 extends VersionConversionStrategy { convert(sourceEvents: SourceInput[]): SourceInputConversionResult[] { - // Currently this is not being used - // Hold off on testing this until atleast one v2 source has been implemented return sourceEvents.map((sourceEvent) => { try { + const sourceEventParam = { ...sourceEvent }; + + let queryParameters: Record | undefined; + if (sourceEventParam.event && sourceEventParam.event.query_parameters) { + queryParameters = sourceEventParam.event.query_parameters; + delete sourceEventParam.event.query_parameters; + } + const sourceRequest: SourceRequestV2 = { - method: '', - url: '', - proto: '', - headers: {}, - query_parameters: {}, - body: JSON.stringify(sourceEvent.event), + body: JSON.stringify(sourceEventParam.event), }; + if (queryParameters) { + sourceRequest.query_parameters = queryParameters; + } + const sourceInputV2: SourceInputV2 = { request: sourceRequest, - source: sourceEvent.source, + source: sourceEventParam.source, }; return { output: sourceInputV2, diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts index 7ddafd782e..0872d549f0 100644 --- a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts +++ b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts @@ -6,7 +6,9 @@ export class StrategyV2ToV1 extends VersionConversionStrategy { try { const v1Event = { event: JSON.parse(sourceEvent.request.body), source: sourceEvent.source }; - v1Event.event.query_parameters = sourceEvent.request.query_parameters; + if (sourceEvent.request) { + v1Event.event.query_parameters = sourceEvent.request.query_parameters; + } return { output: v1Event }; } catch (err) { const conversionError = diff --git a/src/controllers/util/index.test.ts b/src/controllers/util/index.test.ts index 6ab2336b71..138572a8ea 100644 --- a/src/controllers/util/index.test.ts +++ b/src/controllers/util/index.test.ts @@ -201,6 +201,38 @@ describe('adaptInputToVersion', () => { expect(result).toEqual(expected); }); + it('should fail trying to convert input from v2 to v0 format when the request version is v2 and the implementation version is v0', () => { + const sourceType = 'pipedream'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v0', + input: [ + { + output: {}, + conversionError: new SyntaxError('Unexpected end of JSON input'), + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + it('should convert input from v2 to v1 format when the request version is v2 and the implementation version is v1', () => { const sourceType = 'webhook'; const requestVersion = 'v2'; @@ -269,6 +301,38 @@ describe('adaptInputToVersion', () => { expect(result).toEqual(expected); }); + it('should fail trying to convert input from v2 to v1 format when the request version is v2 and the implementation version is v1', () => { + const sourceType = 'webhook'; + const requestVersion = 'v2'; + + const input = [ + { + request: { + method: 'POST', + url: 'http://example.com', + proto: 'HTTP/2', + headers: { headerkey: ['headervalue'] }, + body: '{"key": "value"', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + const expected = { + implementationVersion: 'v1', + input: [ + { + output: {}, + conversionError: new SyntaxError('Unexpected end of JSON input'), + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); + // Should return an empty array when the input is an empty array it('should return an empty array when the input is an empty array', () => { const sourceType = 'pipedream'; @@ -280,6 +344,66 @@ describe('adaptInputToVersion', () => { expect(result).toEqual(expected); }); + + it('should convert input from v1 to v2 format when the request version is v1 and the implementation version is v2', () => { + const sourceType = 'someSourceType'; + const requestVersion = 'v1'; + + // Mock return value for getSourceVersionsMap + jest + .spyOn(ControllerUtility as any, 'getSourceVersionsMap') + .mockReturnValue(new Map([['someSourceType', 'v2']])); + + const input = [ + { + event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + event: { key: 'value' }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + event: {}, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + + const expected = { + implementationVersion: 'v2', + input: [ + { + output: { + request: { + body: '{"key":"value"}', + query_parameters: { paramkey: ['paramvalue'] }, + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + request: { + body: '{"key":"value"}', + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + { + output: { + request: { + body: '{}', + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); }); type timestampTestCases = { diff --git a/src/controllers/util/index.ts b/src/controllers/util/index.ts index b6fa909d27..ab2a0f5dc3 100644 --- a/src/controllers/util/index.ts +++ b/src/controllers/util/index.ts @@ -9,10 +9,7 @@ import { ProcessorTransformationRequest, RouterTransformationRequestData, RudderMessage, - SourceInput, SourceInputConversionResult, - SourceInputV2, - SourceRequestV2, } from '../../types'; import { getValueFromMessage } from '../../v0/util'; import genericFieldMap from '../../v0/util/data/GenericFieldMapping.json'; @@ -49,84 +46,6 @@ export class ControllerUtility { return this.sourceVersionMap; } - private static convertSourceInputv1Tov0( - sourceEvents: SourceInput[], - ): SourceInputConversionResult>[] { - return sourceEvents.map((sourceEvent) => ({ - output: sourceEvent.event as NonNullable, - })); - } - - private static convertSourceInputv1Tov2( - sourceEvents: SourceInput[], - ): SourceInputConversionResult[] { - // Currently this is not being used - // Hold off on testing this until atleast one v2 source has been implemented - return sourceEvents.map((sourceEvent) => { - try { - const sourceRequest: SourceRequestV2 = { - method: '', - url: '', - proto: '', - headers: {}, - query_parameters: {}, - body: JSON.stringify(sourceEvent.event), - }; - const sourceInputV2: SourceInputV2 = { - request: sourceRequest, - source: sourceEvent.source, - }; - return { - output: sourceInputV2, - }; - } catch (err) { - const conversionError = - err instanceof Error ? err : new Error('error converting v1 to v2 spec'); - return { output: {} as SourceInputV2, conversionError }; - } - }); - } - - private static convertSourceInputv0Tov1( - sourceEvents: unknown[], - ): SourceInputConversionResult[] { - return sourceEvents.map((sourceEvent) => ({ - output: { event: sourceEvent, source: undefined } as SourceInput, - })); - } - - private static convertSourceInputv2Tov0( - sourceEvents: SourceInputV2[], - ): SourceInputConversionResult>[] { - return sourceEvents.map((sourceEvent) => { - try { - const v0Event = JSON.parse(sourceEvent.request.body); - v0Event.query_parameters = sourceEvent.request.query_parameters; - return { output: v0Event }; - } catch (err) { - const conversionError = - err instanceof Error ? err : new Error('error converting v2 to v0 spec'); - return { output: {} as NonNullable, conversionError }; - } - }); - } - - private static convertSourceInputv2Tov1( - sourceEvents: SourceInputV2[], - ): SourceInputConversionResult[] { - return sourceEvents.map((sourceEvent) => { - try { - const v1Event = { event: JSON.parse(sourceEvent.request.body), source: sourceEvent.source }; - v1Event.event.query_parameters = sourceEvent.request.query_parameters; - return { output: v1Event }; - } catch (err) { - const conversionError = - err instanceof Error ? err : new Error('error converting v2 to v1 spec'); - return { output: {} as SourceInput, conversionError }; - } - }); - } - public static adaptInputToVersion( sourceType: string, requestVersion: string, @@ -140,22 +59,6 @@ export class ControllerUtility { implementationVersion, ); return { implementationVersion, input: conversionStrategy.convert(input) }; - - // let updatedInput: SourceInputConversionResult>[] = input.map((event) => ({ - // output: event, - // })); - // if (requestVersion === 'v0' && implementationVersion === 'v1') { - // updatedInput = this.convertSourceInputv0Tov1(input); - // } else if (requestVersion === 'v1' && implementationVersion === 'v0') { - // updatedInput = this.convertSourceInputv1Tov0(input as SourceInput[]); - // } else if (requestVersion === 'v1' && implementationVersion === 'v2') { - // updatedInput = this.convertSourceInputv1Tov2(input as SourceInput[]); - // } else if (requestVersion === 'v2' && implementationVersion === 'v0') { - // updatedInput = this.convertSourceInputv2Tov0(input as SourceInputV2[]); - // } else if (requestVersion === 'v2' && implementationVersion === 'v1') { - // updatedInput = this.convertSourceInputv2Tov1(input as SourceInputV2[]); - // } - // return { implementationVersion, input: updatedInput }; } private static getCompatibleStatusCode(status: number): number { diff --git a/src/middlewares/routeActivation.ts b/src/middlewares/routeActivation.ts index 126749b083..ffb1e15e80 100644 --- a/src/middlewares/routeActivation.ts +++ b/src/middlewares/routeActivation.ts @@ -106,19 +106,4 @@ export class RouteActivationMiddleware { RouteActivationMiddleware.shouldActivateRoute(destination, deliveryFilterList), ); } - - // This middleware will be used by source endpoint when we completely deprecate v0, v1 versions. - public static isVersionAllowed(ctx: Context, next: Next) { - const { version } = ctx.params; - if (version === 'v0' || version === 'v1') { - ctx.status = 500; - ctx.body = - '/v0, /v1 versioned endpoints are deprecated. Use /v2 version endpoint. This is probably caused because of source transformation call from an outdated rudder-server version. Please upgrade rudder-server to a minimum of 1.xx.xx version.'; - } else if (version === 'v2') { - next(); - } else { - ctx.status = 404; - ctx.body = 'Path not found. Verify the version of your api call.'; - } - } } diff --git a/src/types/index.ts b/src/types/index.ts index 54ff3a994e..ee225bb0c0 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -352,17 +352,20 @@ type Source = { }; type SourceInput = { - event: NonNullable[]; + event: { + query_parameters?: any; + [key: string]: any; + }; source?: Source; }; type SourceRequestV2 = { - method: string; - url: string; - proto: string; + method?: string; + url?: string; + proto?: string; body: string; - headers: NonNullable; - query_parameters: NonNullable; + headers?: Record; + query_parameters?: Record; }; type SourceInputV2 = { From b10a39f1e329f4b2efe2d433e064febcbb13354d Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 11 Nov 2024 11:12:07 +0000 Subject: [PATCH 06/32] chore(release): 1.84.0 --- CHANGELOG.md | 17 +++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c7591edec..c3c5528ff6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.84.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.83.2...v1.84.0) (2024-11-11) + + +### Features + +* gaec migration ([#3855](https://github.com/rudderlabs/rudder-transformer/issues/3855)) ([7a26459](https://github.com/rudderlabs/rudder-transformer/commit/7a264590b61d3d31d5559c8ac53fd572b40cddec)) +* **GARL:** support vdm next for GARL ([#3835](https://github.com/rudderlabs/rudder-transformer/issues/3835)) ([f4b38eb](https://github.com/rudderlabs/rudder-transformer/commit/f4b38eba3ca8dff602915853fda5cd7ca284bba3)) +* update on twitter_ads ([#3856](https://github.com/rudderlabs/rudder-transformer/issues/3856)) ([adc8976](https://github.com/rudderlabs/rudder-transformer/commit/adc8976990fa98c5b874472aee180cadfabb0088)) + + +### Bug Fixes + +* adding throttled status code for server unavailable error in salesforce ([#3862](https://github.com/rudderlabs/rudder-transformer/issues/3862)) ([fa93f09](https://github.com/rudderlabs/rudder-transformer/commit/fa93f0917d4f75fc197a6ea4c574d37faa0a3f77)) +* linkedin ads conversionValue object as well as price is not mandatory ([#3860](https://github.com/rudderlabs/rudder-transformer/issues/3860)) ([bfd7edc](https://github.com/rudderlabs/rudder-transformer/commit/bfd7edc5608c60a39644a1d4ad6e15e5dbcbea0e)) +* marketo bulk upload handle special chars ([#3859](https://github.com/rudderlabs/rudder-transformer/issues/3859)) ([f959a7d](https://github.com/rudderlabs/rudder-transformer/commit/f959a7dc2487dc7e36377f5f2e265014f692f476)) +* unsafe property getting set via set value library ([#3853](https://github.com/rudderlabs/rudder-transformer/issues/3853)) ([80d7b41](https://github.com/rudderlabs/rudder-transformer/commit/80d7b417be7a0e459de49caca25aba43ffdba337)) + ### [1.83.2](https://github.com/rudderlabs/rudder-transformer/compare/v1.83.1...v1.83.2) (2024-11-05) diff --git a/package-lock.json b/package-lock.json index 0965688626..0422078f21 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.83.2", + "version": "1.84.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.83.2", + "version": "1.84.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 65b7313e88..907c340cbf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.83.2", + "version": "1.84.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From 7ad4423499959684ecabff6b50a77695c1749f35 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Tue, 12 Nov 2024 12:42:44 +0530 Subject: [PATCH 07/32] chore: remove query_parameter injection --- .../util/conversionStrategies/strategyV2ToV0.ts | 1 - .../util/conversionStrategies/strategyV2ToV1.ts | 3 --- src/controllers/util/index.test.ts | 12 ++++++------ 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV0.ts b/src/controllers/util/conversionStrategies/strategyV2ToV0.ts index 031039e538..5d90b8cdda 100644 --- a/src/controllers/util/conversionStrategies/strategyV2ToV0.ts +++ b/src/controllers/util/conversionStrategies/strategyV2ToV0.ts @@ -6,7 +6,6 @@ export class StrategyV2ToV0 extends VersionConversionStrategy { try { const v0Event = JSON.parse(sourceEvent.request.body); - v0Event.query_parameters = sourceEvent.request.query_parameters; return { output: v0Event }; } catch (err) { const conversionError = diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts index 0872d549f0..d651917096 100644 --- a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts +++ b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts @@ -6,9 +6,6 @@ export class StrategyV2ToV1 extends VersionConversionStrategy { try { const v1Event = { event: JSON.parse(sourceEvent.request.body), source: sourceEvent.source }; - if (sourceEvent.request) { - v1Event.event.query_parameters = sourceEvent.request.query_parameters; - } return { output: v1Event }; } catch (err) { const conversionError = diff --git a/src/controllers/util/index.test.ts b/src/controllers/util/index.test.ts index 138572a8ea..f1503f7f81 100644 --- a/src/controllers/util/index.test.ts +++ b/src/controllers/util/index.test.ts @@ -190,9 +190,9 @@ describe('adaptInputToVersion', () => { const expected = { implementationVersion: 'v0', input: [ - { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, - { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, - { output: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } } }, + { output: { key: 'value' } }, + { output: { key: 'value' } }, + { output: { key: 'value' } }, ], }; @@ -277,19 +277,19 @@ describe('adaptInputToVersion', () => { input: [ { output: { - event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + event: { key: 'value' }, source: { id: 'source_id', config: { configField1: 'configVal1' } }, }, }, { output: { - event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + event: { key: 'value' }, source: { id: 'source_id', config: { configField1: 'configVal1' } }, }, }, { output: { - event: { key: 'value', query_parameters: { paramkey: ['paramvalue'] } }, + event: { key: 'value' }, source: { id: 'source_id', config: { configField1: 'configVal1' } }, }, }, From 640a11eb3dca5735fed3ad9ad5bd058974b069d6 Mon Sep 17 00:00:00 2001 From: Sandeep Digumarty Date: Tue, 12 Nov 2024 13:12:18 +0530 Subject: [PATCH 08/32] feat: moved userSchema to connection config in GARL vdmv2 (#3870) * feat: moved userSchema to connection config in GARL vdmv2 * feat: moved userSchema to connection config in GARL vdmv2 * chore: added tests --- .../config.js | 2 + .../recordTransform.js | 28 ++- .../transform.js | 4 +- .../google_adwords_remarketing_lists/util.js | 41 +++- .../util.test.js | 2 +- .../router/data.ts | 223 +++++++++++++++++- .../router/record.ts | 119 +++++++++- 7 files changed, 405 insertions(+), 14 deletions(-) diff --git a/src/v0/destinations/google_adwords_remarketing_lists/config.js b/src/v0/destinations/google_adwords_remarketing_lists/config.js index 0478a1b11b..1e943aee56 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/config.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/config.js @@ -7,6 +7,7 @@ const CONFIG_CATEGORIES = { AUDIENCE_LIST: { type: 'audienceList', name: 'offlineDataJobs' }, ADDRESSINFO: { type: 'addressInfo', name: 'addressInfo' }, }; +const ADDRESS_INFO_ATTRIBUTES = ['firstName', 'lastName', 'country', 'postalCode']; const attributeMapping = { email: 'hashedEmail', phone: 'hashedPhoneNumber', @@ -31,6 +32,7 @@ module.exports = { hashAttributes, offlineDataJobsMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.AUDIENCE_LIST.name], addressInfoMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.ADDRESSINFO.name], + ADDRESS_INFO_ATTRIBUTES, consentConfigMap, destType: 'google_adwords_remarketing_lists', }; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js index f8a2b0e586..1c6284cd09 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js @@ -11,7 +11,11 @@ const { isEventSentByVDMV2Flow, } = require('../../util'); const { populateConsentFromConfig } = require('../../util/googleUtils'); -const { populateIdentifiers, responseBuilder, getOperationAudienceId } = require('./util'); +const { + populateIdentifiersForRecordEvent, + responseBuilder, + getOperationAudienceId, +} = require('./util'); const { getErrorResponse, createFinalResponse } = require('../../util/recordUtils'); const { offlineDataJobsMapping, consentConfigMap } = require('./config'); @@ -23,6 +27,7 @@ const processRecordEventArray = ( developerToken, audienceId, typeOfList, + userSchema, isHashRequired, operationType, ) => { @@ -36,10 +41,10 @@ const processRecordEventArray = ( metadata.push(record.metadata); }); - const userIdentifiersList = populateIdentifiers( + const userIdentifiersList = populateIdentifiersForRecordEvent( fieldsArray, - destination, typeOfList, + userSchema, isHashRequired, ); @@ -91,7 +96,7 @@ function preparepayload(events, config) { const { destination, message, metadata } = events[0]; const accessToken = getAccessToken(metadata, 'access_token'); const developerToken = getValueFromMessage(metadata, 'secret.developer_token'); - const { audienceId, typeOfList, isHashRequired } = config; + const { audienceId, typeOfList, isHashRequired, userSchema } = config; const groupedRecordsByAction = lodash.groupBy(events, (record) => record.message.action?.toLowerCase(), @@ -110,6 +115,7 @@ function preparepayload(events, config) { developerToken, audienceId, typeOfList, + userSchema, isHashRequired, 'remove', ); @@ -124,6 +130,7 @@ function preparepayload(events, config) { developerToken, audienceId, typeOfList, + userSchema, isHashRequired, 'add', ); @@ -138,6 +145,7 @@ function preparepayload(events, config) { developerToken, audienceId, typeOfList, + userSchema, isHashRequired, 'add', ); @@ -161,19 +169,26 @@ function preparepayload(events, config) { function processRecordInputsV0(groupedRecordInputs) { const { destination, message } = groupedRecordInputs[0]; - const { audienceId, typeOfList, isHashRequired } = destination.Config; + const { audienceId, typeOfList, isHashRequired, userSchema } = destination.Config; return preparepayload(groupedRecordInputs, { audienceId: getOperationAudienceId(audienceId, message), typeOfList, + userSchema, isHashRequired, }); } function processRecordInputsV1(groupedRecordInputs) { - const { connection } = groupedRecordInputs[0]; + const { connection, message } = groupedRecordInputs[0]; const { audienceId, typeOfList, isHashRequired } = connection.config.destination; + const identifiers = message?.identifiers; + let userSchema; + if (identifiers) { + userSchema = Object.keys(identifiers); + } + const events = groupedRecordInputs.map((record) => ({ ...record, message: { @@ -185,6 +200,7 @@ function processRecordInputsV1(groupedRecordInputs) { return preparepayload(events, { audienceId, typeOfList, + userSchema, isHashRequired, }); } diff --git a/src/v0/destinations/google_adwords_remarketing_lists/transform.js b/src/v0/destinations/google_adwords_remarketing_lists/transform.js index 299ab94846..4d173589e8 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/transform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/transform.js @@ -37,7 +37,7 @@ function extraKeysPresent(dictionary, keyList) { const createPayload = (message, destination) => { const { listData } = message.properties; const properties = ['add', 'remove']; - const { typeOfList, isHashRequired } = destination.Config; + const { typeOfList, userSchema, isHashRequired } = destination.Config; let outputPayloads = {}; const typeOfOperation = Object.keys(listData); @@ -45,8 +45,8 @@ const createPayload = (message, destination) => { if (properties.includes(key)) { const userIdentifiersList = populateIdentifiers( listData[key], - destination, typeOfList, + userSchema, isHashRequired, ); if (userIdentifiersList.length === 0) { diff --git a/src/v0/destinations/google_adwords_remarketing_lists/util.js b/src/v0/destinations/google_adwords_remarketing_lists/util.js index f4c33a9a6f..8e0aed0365 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/util.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/util.js @@ -18,6 +18,7 @@ const { TYPEOFLIST, BASE_ENDPOINT, hashAttributes, + ADDRESS_INFO_ATTRIBUTES, } = require('./config'); const hashEncrypt = (object) => { @@ -68,14 +69,13 @@ const responseBuilder = ( * Logics: Here we are creating an array with all the attributes provided in the add/remove array * inside listData. * @param {Array} attributeArray rudder event message properties listData add - * @param {object} Config rudder event destination * @param {string} typeOfList + * @param {Array} userSchema * @param {boolean} isHashRequired * @returns */ -const populateIdentifiers = (attributeArray, { Config }, typeOfList, isHashRequired) => { +const populateIdentifiers = (attributeArray, typeOfList, userSchema, isHashRequired) => { const userIdentifier = []; - const { userSchema } = Config; let attribute; if (TYPEOFLIST[typeOfList]) { attribute = TYPEOFLIST[typeOfList]; @@ -115,6 +115,40 @@ const populateIdentifiers = (attributeArray, { Config }, typeOfList, isHashRequi return userIdentifier; }; +const populateIdentifiersForRecordEvent = ( + identifiersArray, + typeOfList, + userSchema, + isHashRequired, +) => { + const userIdentifiers = []; + + if (isDefinedAndNotNullAndNotEmpty(identifiersArray)) { + // traversing through every element in the add array + identifiersArray.forEach((identifiers) => { + if (isHashRequired) { + hashEncrypt(identifiers); + } + if (TYPEOFLIST[typeOfList] && identifiers[TYPEOFLIST[typeOfList]]) { + userIdentifiers.push({ [TYPEOFLIST[typeOfList]]: identifiers[TYPEOFLIST[typeOfList]] }); + } else { + Object.entries(attributeMapping).forEach(([key, mappedKey]) => { + if (identifiers[key] && userSchema.includes(key)) + userIdentifiers.push({ [mappedKey]: identifiers[key] }); + }); + const addressInfo = constructPayload(identifiers, addressInfoMapping); + if ( + isDefinedAndNotNullAndNotEmpty(addressInfo) && + (userSchema.includes('addressInfo') || + userSchema.some((schema) => ADDRESS_INFO_ATTRIBUTES.includes(schema))) + ) + userIdentifiers.push({ addressInfo }); + } + }); + } + return userIdentifiers; +}; + const getOperationAudienceId = (audienceId, message) => { let operationAudienceId = audienceId; const mappedToDestination = get(message, MappedToDestinationKey); @@ -132,4 +166,5 @@ module.exports = { populateIdentifiers, responseBuilder, getOperationAudienceId, + populateIdentifiersForRecordEvent, }; diff --git a/src/v0/destinations/google_adwords_remarketing_lists/util.test.js b/src/v0/destinations/google_adwords_remarketing_lists/util.test.js index 0b74b07b8e..a41c00f12f 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/util.test.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/util.test.js @@ -199,8 +199,8 @@ describe('GARL utils test', () => { const { typeOfList, isHashRequired } = baseDestination.Config; const identifier = populateIdentifiers( attributeArray, - baseDestination, typeOfList, + baseDestination.Config.userSchema, isHashRequired, ); expect(identifier).toEqual(hashedArray); diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts index a5e28996b1..6878e81f0d 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts @@ -1,5 +1,9 @@ import { rETLAudienceRouterRequest } from './audience'; -import { rETLRecordRouterRequest } from './record'; +import { + rETLRecordRouterRequest, + rETLRecordRouterRequestVDMv2General, + rETLRecordRouterRequestVDMv2UserId, +} from './record'; import { API_VERSION } from '../../../../../src/v0/destinations/google_adwords_remarketing_lists/config'; export const data = [ @@ -732,4 +736,221 @@ export const data = [ }, }, }, + { + name: 'google_adwords_remarketing_lists record event tests VDMv2 General typeOfList', + description: 'Test 2', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: rETLRecordRouterRequestVDMv2General, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/7693729833/offlineUserDataJobs`, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + }, + params: { + listId: '7090784486', + customerId: '7693729833', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + }, + body: { + JSON: { + operations: [ + { + create: { + userIdentifiers: [ + { + hashedEmail: + 'd3142c8f9c9129484daf28df80cc5c955791efed5e69afabb603bc8cb9ffd419', + }, + { + hashedPhoneNumber: + '8846dcb6ab2d73a0e67dbd569fa17cec2d9d391e5b05d1dd42919bc21ae82c45', + }, + { + addressInfo: { + hashedFirstName: + '9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08', + hashedLastName: + 'dcf000c2386fb76d22cefc0d118a8511bb75999019cd373df52044bccd1bd251', + countryCode: 'US', + postalCode: '1245', + }, + }, + ], + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + secret: { + access_token: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + jobId: 1, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', + audienceId: '7090784486', + customerId: '7693729833', + loginCustomerId: '', + subAccount: false, + }, + DestinationDefinition: { + Config: {}, + DisplayName: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + ID: '1aIXqM806xAVm92nx07YwKbRrO9', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + }, + Enabled: true, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + IsConnectionEnabled: true, + IsProcessorEnabled: true, + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Transformations: [], + WorkspaceID: '1TSN08muJTZwH8iCDmnnRt1pmLd', + }, + }, + ], + }, + }, + }, + }, + { + name: 'google_adwords_remarketing_lists record event tests VDMv2 UserId typeOfList', + description: 'Test 3', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: rETLRecordRouterRequestVDMv2UserId, + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: [ + { + version: '1', + type: 'REST', + method: 'POST', + endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/7693729833/offlineUserDataJobs`, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + }, + params: { + listId: '7090784486', + customerId: '7693729833', + consent: { + adPersonalization: 'UNSPECIFIED', + adUserData: 'UNSPECIFIED', + }, + }, + body: { + JSON: { + operations: [ + { + create: { + userIdentifiers: [ + { + thirdPartyUserId: 'useri1234', + }, + ], + }, + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + ], + metadata: [ + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + secret: { + access_token: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + jobId: 2, + }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', + audienceId: '7090784486', + customerId: '7693729833', + loginCustomerId: '', + subAccount: false, + }, + DestinationDefinition: { + Config: {}, + DisplayName: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + ID: '1aIXqM806xAVm92nx07YwKbRrO9', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + }, + Enabled: true, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + IsConnectionEnabled: true, + IsProcessorEnabled: true, + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Transformations: [], + WorkspaceID: '1TSN08muJTZwH8iCDmnnRt1pmLd', + }, + }, + ], + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts index de76aae17c..2661500b4d 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts @@ -1,4 +1,5 @@ -import { Destination, RouterTransformationRequest } from '../../../../../src/types'; +import { Connection, Destination, RouterTransformationRequest } from '../../../../../src/types'; +import { VDM_V2_SCHEMA_VERSION } from '../../../../../src/v0/util/constant'; import { generateGoogleOAuthMetadata } from '../../../testUtils'; const destination: Destination = { @@ -27,6 +28,57 @@ const destination: Destination = { IsProcessorEnabled: true, }; +const destination2: Destination = { + Config: { + rudderAccountId: '258Yea7usSKNpbkIaesL9oJ9iYw', + audienceId: '7090784486', + customerId: '7693729833', + loginCustomerId: '', + subAccount: false, + }, + ID: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Enabled: true, + WorkspaceID: '1TSN08muJTZwH8iCDmnnRt1pmLd', + DestinationDefinition: { + ID: '1aIXqM806xAVm92nx07YwKbRrO9', + Name: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + DisplayName: 'GOOGLE_ADWORDS_REMARKETING_LISTS', + Config: {}, + }, + Transformations: [], + IsConnectionEnabled: true, + IsProcessorEnabled: true, +}; + +const connection1: Connection = { + sourceId: '2MUWghI7u85n91dd1qzGyswpZan', + destinationId: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + enabled: true, + config: { + destination: { + schemaVersion: VDM_V2_SCHEMA_VERSION, + isHashRequired: true, + typeOfList: 'General', + audienceId: '7090784486', + }, + }, +}; + +const connection2: Connection = { + sourceId: '2MUWghI7u85n91dd1qzGyswpZan', + destinationId: '1mMy5cqbtfuaKZv1IhVQKnBdVwe', + enabled: true, + config: { + destination: { + schemaVersion: VDM_V2_SCHEMA_VERSION, + isHashRequired: true, + typeOfList: 'userID', + audienceId: '7090784486', + }, + }, +}; + export const rETLRecordRouterRequest: RouterTransformationRequest = { input: [ { @@ -153,6 +205,71 @@ export const rETLRecordRouterRequest: RouterTransformationRequest = { destType: 'google_adwords_remarketing_lists', }; +export const rETLRecordRouterRequestVDMv2General: RouterTransformationRequest = { + input: [ + { + destination: destination2, + connection: connection1, + message: { + action: 'insert', + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + }, + recordId: '2', + rudderId: '2', + identifiers: { + email: 'test@abc.com', + phone: '@09876543210', + firstName: 'test', + lastName: 'rudderlabs', + country: 'US', + postalCode: '1245', + }, + type: 'record', + }, + metadata: generateGoogleOAuthMetadata(1), + }, + ], + destType: 'google_adwords_remarketing_lists', +}; + +export const rETLRecordRouterRequestVDMv2UserId: RouterTransformationRequest = { + input: [ + { + destination: destination2, + connection: connection2, + message: { + action: 'insert', + context: { + ip: '14.5.67.21', + library: { + name: 'http', + }, + }, + recordId: '2', + rudderId: '2', + identifiers: { + email: 'test@abc.com', + phone: '@09876543210', + firstName: 'test', + lastName: 'rudderlabs', + country: 'US', + postalCode: '1245', + thirdPartyUserId: 'useri1234', + }, + type: 'record', + }, + metadata: generateGoogleOAuthMetadata(2), + }, + ], + destType: 'google_adwords_remarketing_lists', +}; + module.exports = { rETLRecordRouterRequest, + rETLRecordRouterRequestVDMv2General, + rETLRecordRouterRequestVDMv2UserId, }; From 70ab9c814c295fc0e7d0b606e5b981510b0bbe3f Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Tue, 12 Nov 2024 14:50:45 +0530 Subject: [PATCH 09/32] chore: fix workflow to consider empty commits --- .github/workflows/verify.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 4fca34673a..4bd66285bd 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -43,8 +43,12 @@ jobs: - name: Run format Checks run: | - npx prettier --write $(cat changed_files.txt) + if [ -s changed_files.txt ]; then + npx prettier --write $(cat changed_files.txt) + fi + - run: git diff --exit-code + - name: Formatting Error message if: ${{ failure() }} run: | From 1c134f84601aaea78581078137cb9955de576f9e Mon Sep 17 00:00:00 2001 From: Aanshi Lahoti <110057617+aanshi07@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:36:09 +0530 Subject: [PATCH 10/32] feat: iterable EUDC (#3828) * feat: iterable EUDC * chore: simplyfied functions * chore: modifications added * chore: fix * chore: modification for endpoint * chore: removed dataCenter to check default --- src/v0/destinations/iterable/config.js | 36 ++-- src/v0/destinations/iterable/transform.js | 21 ++- src/v0/destinations/iterable/util.js | 32 ++-- .../iterable/processor/aliasTestData.ts | 59 ++++++- .../iterable/processor/identifyTestData.ts | 57 +++++++ .../iterable/processor/pageScreenTestData.ts | 65 +++++++- .../iterable/processor/trackTestData.ts | 55 +++++++ .../destinations/iterable/router/data.ts | 155 ++++++++++++++++++ 8 files changed, 439 insertions(+), 41 deletions(-) diff --git a/src/v0/destinations/iterable/config.js b/src/v0/destinations/iterable/config.js index f74fdb4975..125367875f 100644 --- a/src/v0/destinations/iterable/config.js +++ b/src/v0/destinations/iterable/config.js @@ -1,42 +1,45 @@ const { getMappingConfig } = require('../../util'); -const BASE_URL = 'https://api.iterable.com/api/'; +const BASE_URL = { + USDC: 'https://api.iterable.com/api/', + EUDC: 'https://api.eu.iterable.com/api/', +}; const ConfigCategory = { IDENTIFY_BROWSER: { name: 'IterableRegisterBrowserTokenConfig', action: 'identifyBrowser', - endpoint: `${BASE_URL}users/registerBrowserToken`, + endpoint: `users/registerBrowserToken`, }, IDENTIFY_DEVICE: { name: 'IterableRegisterDeviceTokenConfig', action: 'identifyDevice', - endpoint: `${BASE_URL}users/registerDeviceToken`, + endpoint: `users/registerDeviceToken`, }, IDENTIFY: { name: 'IterableIdentifyConfig', action: 'identify', - endpoint: `${BASE_URL}users/update`, + endpoint: `users/update`, }, PAGE: { name: 'IterablePageConfig', action: 'page', - endpoint: `${BASE_URL}events/track`, + endpoint: `events/track`, }, SCREEN: { name: 'IterablePageConfig', action: 'screen', - endpoint: `${BASE_URL}events/track`, + endpoint: `events/track`, }, TRACK: { name: 'IterableTrackConfig', action: 'track', - endpoint: `${BASE_URL}events/track`, + endpoint: `events/track`, }, TRACK_PURCHASE: { name: 'IterableTrackPurchaseConfig', action: 'trackPurchase', - endpoint: `${BASE_URL}commerce/trackPurchase`, + endpoint: `commerce/trackPurchase`, }, PRODUCT: { name: 'IterableProductConfig', @@ -46,7 +49,7 @@ const ConfigCategory = { UPDATE_CART: { name: 'IterableProductConfig', action: 'updateCart', - endpoint: `${BASE_URL}commerce/updateCart`, + endpoint: `commerce/updateCart`, }, DEVICE: { name: 'IterableDeviceConfig', @@ -56,30 +59,33 @@ const ConfigCategory = { ALIAS: { name: 'IterableAliasConfig', action: 'alias', - endpoint: `${BASE_URL}users/updateEmail`, + endpoint: `users/updateEmail`, }, CATALOG: { name: 'IterableCatalogConfig', action: 'catalogs', - endpoint: `${BASE_URL}catalogs`, + endpoint: `catalogs`, }, }; const mappingConfig = getMappingConfig(ConfigCategory, __dirname); +// Function to construct endpoint based on the selected data center +const constructEndpoint = (dataCenter, category) => { + const baseUrl = BASE_URL[dataCenter] || BASE_URL.USDC; // Default to USDC if not found + return `${baseUrl}${category.endpoint}`; +}; + const IDENTIFY_MAX_BATCH_SIZE = 1000; const IDENTIFY_MAX_BODY_SIZE_IN_BYTES = 4000000; -const IDENTIFY_BATCH_ENDPOINT = 'https://api.iterable.com/api/users/bulkUpdate'; const TRACK_MAX_BATCH_SIZE = 8000; -const TRACK_BATCH_ENDPOINT = 'https://api.iterable.com/api/events/trackBulk'; module.exports = { mappingConfig, ConfigCategory, - TRACK_BATCH_ENDPOINT, + constructEndpoint, TRACK_MAX_BATCH_SIZE, IDENTIFY_MAX_BATCH_SIZE, - IDENTIFY_BATCH_ENDPOINT, IDENTIFY_MAX_BODY_SIZE_IN_BYTES, }; diff --git a/src/v0/destinations/iterable/transform.js b/src/v0/destinations/iterable/transform.js index 207a8d1186..dd67deef69 100644 --- a/src/v0/destinations/iterable/transform.js +++ b/src/v0/destinations/iterable/transform.js @@ -14,6 +14,7 @@ const { filterEventsAndPrepareBatchRequests, registerDeviceTokenEventPayloadBuilder, registerBrowserTokenEventPayloadBuilder, + getCategoryWithEndpoint, } = require('./util'); const { constructPayload, @@ -116,12 +117,11 @@ const responseBuilderForRegisterDeviceOrBrowserTokenEvents = (message, destinati /** * Function to find category value - * @param {*} messageType * @param {*} message * @returns */ -const getCategory = (messageType, message) => { - const eventType = messageType.toLowerCase(); +const getCategory = (message, dataCenter) => { + const eventType = message.type.toLowerCase(); switch (eventType) { case EventType.IDENTIFY: @@ -129,17 +129,17 @@ const getCategory = (messageType, message) => { get(message, MappedToDestinationKey) && getDestinationExternalIDInfoForRetl(message, 'ITERABLE').objectType !== 'users' ) { - return ConfigCategory.CATALOG; + return getCategoryWithEndpoint(ConfigCategory.CATALOG, dataCenter); } - return ConfigCategory.IDENTIFY; + return getCategoryWithEndpoint(ConfigCategory.IDENTIFY, dataCenter); case EventType.PAGE: - return ConfigCategory.PAGE; + return getCategoryWithEndpoint(ConfigCategory.PAGE, dataCenter); case EventType.SCREEN: - return ConfigCategory.SCREEN; + return getCategoryWithEndpoint(ConfigCategory.SCREEN, dataCenter); case EventType.TRACK: - return getCategoryUsingEventName(message); + return getCategoryUsingEventName(message, dataCenter); case EventType.ALIAS: - return ConfigCategory.ALIAS; + return getCategoryWithEndpoint(ConfigCategory.ALIAS, dataCenter); default: throw new InstrumentationError(`Message type ${eventType} not supported`); } @@ -150,8 +150,7 @@ const process = (event) => { if (!message.type) { throw new InstrumentationError('Event type is required'); } - const messageType = message.type.toLowerCase(); - const category = getCategory(messageType, message); + const category = getCategory(message, destination.Config.dataCenter); const response = responseBuilder(message, category, destination); if (hasMultipleResponses(message, category, destination.Config)) { diff --git a/src/v0/destinations/iterable/util.js b/src/v0/destinations/iterable/util.js index 7c1509c2b7..b918600253 100644 --- a/src/v0/destinations/iterable/util.js +++ b/src/v0/destinations/iterable/util.js @@ -14,10 +14,9 @@ const { ConfigCategory, mappingConfig, TRACK_MAX_BATCH_SIZE, - TRACK_BATCH_ENDPOINT, IDENTIFY_MAX_BATCH_SIZE, - IDENTIFY_BATCH_ENDPOINT, IDENTIFY_MAX_BODY_SIZE_IN_BYTES, + constructEndpoint, } = require('./config'); const { JSON_MIME_TYPE } = require('../../util/constant'); const { EventType, MappedToDestinationKey } = require('../../../constants'); @@ -88,12 +87,17 @@ const hasMultipleResponses = (message, category, config) => { return isIdentifyEvent && isIdentifyCategory && hasToken && hasRegisterDeviceOrBrowserKey; }; +const getCategoryWithEndpoint = (categoryConfig, dataCenter) => ({ + ...categoryConfig, + endpoint: constructEndpoint(dataCenter, categoryConfig), +}); + /** * Returns category value * @param {*} message * @returns */ -const getCategoryUsingEventName = (message) => { +const getCategoryUsingEventName = (message, dataCenter) => { let { event } = message; if (typeof event === 'string') { event = event.toLowerCase(); @@ -101,12 +105,12 @@ const getCategoryUsingEventName = (message) => { switch (event) { case 'order completed': - return ConfigCategory.TRACK_PURCHASE; + return getCategoryWithEndpoint(ConfigCategory.TRACK_PURCHASE, dataCenter); case 'product added': case 'product removed': - return ConfigCategory.UPDATE_CART; + return getCategoryWithEndpoint(ConfigCategory.UPDATE_CART, dataCenter); default: - return ConfigCategory.TRACK; + return getCategoryWithEndpoint(ConfigCategory.TRACK, dataCenter); } }; @@ -444,8 +448,8 @@ const processUpdateUserBatch = (chunk, registerDeviceOrBrowserTokenEvents) => { batchEventResponse.batchedRequest.body.JSON = { users: batch.users }; const { destination, metadata, nonBatchedRequests } = batch; - const { apiKey } = destination.Config; - + const { apiKey, dataCenter } = destination.Config; + const IDENTIFY_BATCH_ENDPOINT = constructEndpoint(dataCenter, { endpoint: 'users/bulkUpdate' }); const batchedResponse = combineBatchedAndNonBatchedEvents( apiKey, metadata, @@ -552,8 +556,8 @@ const processTrackBatch = (chunk) => { const metadata = []; const { destination } = chunk[0]; - const { apiKey } = destination.Config; - + const { apiKey, dataCenter } = destination.Config; + const TRACK_BATCH_ENDPOINT = constructEndpoint(dataCenter, { endpoint: 'events/trackBulk' }); chunk.forEach((event) => { metadata.push(event.metadata); events.push(get(event, `${MESSAGE_JSON_PATH}`)); @@ -653,12 +657,13 @@ const mapRegisterDeviceOrBrowserTokenEventsWithJobId = (events) => { */ const categorizeEvent = (event) => { const { message, metadata, destination, error } = event; + const { dataCenter } = destination.Config; if (error) { return { type: 'error', data: event }; } - if (message.endpoint === ConfigCategory.IDENTIFY.endpoint) { + if (message.endpoint === constructEndpoint(dataCenter, ConfigCategory.IDENTIFY)) { return { type: 'updateUser', data: { message, metadata, destination } }; } @@ -667,8 +672,8 @@ const categorizeEvent = (event) => { } if ( - message.endpoint === ConfigCategory.IDENTIFY_BROWSER.endpoint || - message.endpoint === ConfigCategory.IDENTIFY_DEVICE.endpoint + message.endpoint === constructEndpoint(dataCenter, ConfigCategory.IDENTIFY_BROWSER) || + message.endpoint === constructEndpoint(dataCenter, ConfigCategory.IDENTIFY_DEVICE) ) { return { type: 'registerDeviceOrBrowser', data: { message, metadata, destination } }; } @@ -753,4 +758,5 @@ module.exports = { filterEventsAndPrepareBatchRequests, registerDeviceTokenEventPayloadBuilder, registerBrowserTokenEventPayloadBuilder, + getCategoryWithEndpoint, }; diff --git a/test/integrations/destinations/iterable/processor/aliasTestData.ts b/test/integrations/destinations/iterable/processor/aliasTestData.ts index cac43767bb..1ee4134859 100644 --- a/test/integrations/destinations/iterable/processor/aliasTestData.ts +++ b/test/integrations/destinations/iterable/processor/aliasTestData.ts @@ -1,4 +1,8 @@ -import { generateMetadata, transformResultBuilder } from './../../../testUtils'; +import { + generateMetadata, + overrideDestination, + transformResultBuilder, +} from './../../../testUtils'; import { Destination } from '../../../../../src/types'; import { ProcessorTestData } from '../../../testTypes'; @@ -15,6 +19,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -94,4 +99,56 @@ export const aliasTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-alias-test-1', + name: 'iterable', + description: 'Alias call with dataCenter as EUDC', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain update email payload', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId: 'anonId', + userId: 'new@email.com', + previousId: 'old@email.com', + name: 'ApplicationLoaded', + context: {}, + properties, + type: 'alias', + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: 'https://api.eu.iterable.com/api/users/updateEmail', + JSON: { + currentEmail: 'old@email.com', + newEmail: 'new@email.com', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/processor/identifyTestData.ts b/test/integrations/destinations/iterable/processor/identifyTestData.ts index d05f87a11f..21d294e232 100644 --- a/test/integrations/destinations/iterable/processor/identifyTestData.ts +++ b/test/integrations/destinations/iterable/processor/identifyTestData.ts @@ -2,6 +2,7 @@ import { generateMetadata, transformResultBuilder, generateIndentifyPayload, + overrideDestination, } from './../../../testUtils'; import { Destination } from '../../../../../src/types'; import { ProcessorTestData } from '../../../testTypes'; @@ -19,6 +20,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -55,6 +57,7 @@ const sentAt = '2020-08-28T16:26:16.473Z'; const originalTimestamp = '2020-08-28T16:26:06.468Z'; const updateUserEndpoint = 'https://api.iterable.com/api/users/update'; +const updateUserEndpointEUDC = 'https://api.eu.iterable.com/api/users/update'; export const identifyTestData: ProcessorTestData[] = [ { @@ -404,4 +407,58 @@ export const identifyTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-identify-test-7', + name: 'iterable', + description: 'Indentify call to update user in iterable with EUDC dataCenter', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain update user payload with all user traits and updateUserEndpointEUDC', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId, + context: { + traits: user1Traits, + }, + traits: user1Traits, + type: 'identify', + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: updateUserEndpointEUDC, + JSON: { + email: user1Traits.email, + userId: anonymousId, + dataFields: user1Traits, + preferUserId: false, + mergeNestedObjects: true, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/processor/pageScreenTestData.ts b/test/integrations/destinations/iterable/processor/pageScreenTestData.ts index 074d6b56df..a27cf9fe3b 100644 --- a/test/integrations/destinations/iterable/processor/pageScreenTestData.ts +++ b/test/integrations/destinations/iterable/processor/pageScreenTestData.ts @@ -1,4 +1,8 @@ -import { generateMetadata, transformResultBuilder } from './../../../testUtils'; +import { + generateMetadata, + overrideDestination, + transformResultBuilder, +} from './../../../testUtils'; import { Destination } from '../../../../../src/types'; import { ProcessorTestData } from '../../../testTypes'; @@ -15,6 +19,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -43,6 +48,7 @@ const sentAt = '2020-08-28T16:26:16.473Z'; const originalTimestamp = '2020-08-28T16:26:06.468Z'; const pageEndpoint = 'https://api.iterable.com/api/events/track'; +const pageEndpointEUDC = 'https://api.eu.iterable.com/api/events/track'; export const pageScreenTestData: ProcessorTestData[] = [ { @@ -406,4 +412,61 @@ export const pageScreenTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-page-test-4', + name: 'iterable', + description: 'Page call with dataCenter as EUDC', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain endpoint as pageEndpointEUDC', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId, + name: 'ApplicationLoaded', + context: { + traits: { + email: 'sayan@gmail.com', + }, + }, + properties, + type: 'page', + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: pageEndpointEUDC, + JSON: { + userId: anonymousId, + dataFields: properties, + email: 'sayan@gmail.com', + createdAt: 1598631966468, + eventName: 'ApplicationLoaded page', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/processor/trackTestData.ts b/test/integrations/destinations/iterable/processor/trackTestData.ts index 296275ad77..2b7d2a9c47 100644 --- a/test/integrations/destinations/iterable/processor/trackTestData.ts +++ b/test/integrations/destinations/iterable/processor/trackTestData.ts @@ -1,6 +1,7 @@ import { generateMetadata, generateTrackPayload, + overrideDestination, transformResultBuilder, } from './../../../testUtils'; import { Destination } from '../../../../../src/types'; @@ -19,6 +20,7 @@ const destination: Destination = { Transformations: [], Config: { apiKey: 'testApiKey', + dataCenter: 'USDC', preferUserId: false, trackAllPages: true, trackNamedPages: false, @@ -126,6 +128,7 @@ const sentAt = '2020-08-28T16:26:16.473Z'; const originalTimestamp = '2020-08-28T16:26:06.468Z'; const endpoint = 'https://api.iterable.com/api/events/track'; +const endpointEUDC = 'https://api.eu.iterable.com/api/events/track'; const updateCartEndpoint = 'https://api.iterable.com/api/commerce/updateCart'; const trackPurchaseEndpoint = 'https://api.iterable.com/api/commerce/trackPurchase'; @@ -714,4 +717,56 @@ export const trackTestData: ProcessorTestData[] = [ }, }, }, + { + id: 'iterable-track-test-9', + name: 'iterable', + description: 'Track call to add event with user with EUDC dataCenter', + scenario: 'Business', + successCriteria: + 'Response should contain status code 200 and it should contain event properties, event name and endpointEUDC', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: overrideDestination(destination, { dataCenter: 'EUDC' }), + message: { + anonymousId, + event: 'Email Opened', + type: 'track', + context: {}, + properties, + sentAt, + originalTimestamp, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + userId: '', + headers, + endpoint: endpointEUDC, + JSON: { + userId: 'anonId', + createdAt: 1598631966468, + eventName: 'Email Opened', + dataFields: properties, + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/router/data.ts b/test/integrations/destinations/iterable/router/data.ts index 09eedc8eb8..1917c078eb 100644 --- a/test/integrations/destinations/iterable/router/data.ts +++ b/test/integrations/destinations/iterable/router/data.ts @@ -247,6 +247,7 @@ export const data = [ destination: { Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -308,6 +309,7 @@ export const data = [ destConfig: { defaultConfig: [ 'apiKey', + 'dataCenter', 'mapToSingleEvent', 'trackAllPages', 'trackCategorisedPages', @@ -339,6 +341,7 @@ export const data = [ }, Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: true, trackAllPages: false, trackCategorisedPages: true, @@ -414,6 +417,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -442,6 +446,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -472,6 +477,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -623,6 +629,7 @@ export const data = [ destination: { Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -686,6 +693,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -732,6 +740,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: true, trackCategorisedPages: false, @@ -765,6 +774,7 @@ export const data = [ destination: { Config: { apiKey: '62d12498c37c4fd8a1a546c2d35c2f60', + dataCenter: 'USDC', mapToSingleEvent: false, trackAllPages: false, trackCategorisedPages: true, @@ -821,6 +831,7 @@ export const data = [ destConfig: { defaultConfig: [ 'apiKey', + 'dataCenter', 'mapToSingleEvent', 'trackAllPages', 'trackCategorisedPages', @@ -852,6 +863,7 @@ export const data = [ }, Config: { apiKey: '12345', + dataCenter: 'USDC', mapToSingleEvent: true, trackAllPages: false, trackCategorisedPages: true, @@ -867,4 +879,147 @@ export const data = [ }, }, }, + { + name: 'iterable', + description: 'Simple identify call with EUDC dataCenter', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: { + receivedAt: '2022-09-27T11:12:59.080Z', + sentAt: '2022-09-27T11:13:03.777Z', + messageId: '9ad41366-8060-4c9f-b181-f6bea67d5469', + originalTimestamp: '2022-09-27T11:13:03.777Z', + traits: { ruchira: 'donaldbaker@ellis.com', new_field2: 'GB' }, + channel: 'sources', + rudderId: '3d51640c-ab09-42c1-b7b2-db6ab433b35e', + context: { + sources: { + version: 'feat.SupportForTrack', + job_run_id: 'ccpdlajh6cfi19mr1vs0', + task_run_id: 'ccpdlajh6cfi19mr1vsg', + batch_id: '4917ad78-280b-40d2-a30d-119434152a0f', + job_id: '2FLKJDcTdjPHQpq7pUjB34dQ5w6/Syncher', + task_id: 'rows_100', + }, + mappedToDestination: 'true', + externalId: [ + { id: 'Tiffany', type: 'ITERABLE-test-ruchira', identifierType: 'itemId' }, + ], + }, + timestamp: '2022-09-27T11:12:59.079Z', + type: 'identify', + userId: 'Tiffany', + recordId: '10', + request_ip: '10.1.86.248', + }, + metadata: { jobId: 2, userId: 'u1' }, + destination: { + Config: { + apiKey: '583af2f8-15ba-49c0-8511-76383e7de07e', + dataCenter: 'EUDC', + hubID: '22066036', + }, + Enabled: true, + }, + }, + { + message: { + receivedAt: '2022-09-27T11:12:59.080Z', + sentAt: '2022-09-27T11:13:03.777Z', + messageId: '9ad41366-8060-4c9f-b181-f6bea67d5469', + originalTimestamp: '2022-09-27T11:13:03.777Z', + traits: { ruchira: 'abc@ellis.com', new_field2: 'GB1' }, + channel: 'sources', + rudderId: '3d51640c-ab09-42c1-b7b2-db6ab433b35e', + context: { + sources: { + version: 'feat.SupportForTrack', + job_run_id: 'ccpdlajh6cfi19mr1vs0', + task_run_id: 'ccpdlajh6cfi19mr1vsg', + batch_id: '4917ad78-280b-40d2-a30d-119434152a0f', + job_id: '2FLKJDcTdjPHQpq7pUjB34dQ5w6/Syncher', + task_id: 'rows_100', + }, + mappedToDestination: 'true', + externalId: [ + { id: 'ABC', type: 'ITERABLE-test-ruchira', identifierType: 'itemId' }, + ], + }, + timestamp: '2022-09-27T11:12:59.079Z', + type: 'identify', + userId: 'Tiffany', + recordId: '10', + request_ip: '10.1.86.248', + }, + metadata: { jobId: 2, userId: 'u1' }, + destination: { + Config: { + apiKey: '583af2f8-15ba-49c0-8511-76383e7de07e', + dataCenter: 'EUDC', + hubID: '22066036', + }, + Enabled: true, + }, + }, + ], + destType: 'iterable', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batchedRequest: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://api.eu.iterable.com/api/catalogs/test-ruchira/items', + headers: { + 'Content-Type': 'application/json', + api_key: '583af2f8-15ba-49c0-8511-76383e7de07e', + }, + params: {}, + body: { + JSON: { + documents: { + Tiffany: { ruchira: 'donaldbaker@ellis.com', new_field2: 'GB' }, + ABC: { ruchira: 'abc@ellis.com', new_field2: 'GB1' }, + }, + replaceUploadedFieldsOnly: true, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + }, + metadata: [ + { jobId: 2, userId: 'u1' }, + { jobId: 2, userId: 'u1' }, + ], + batched: true, + statusCode: 200, + destination: { + Config: { + apiKey: '583af2f8-15ba-49c0-8511-76383e7de07e', + dataCenter: 'EUDC', + hubID: '22066036', + }, + Enabled: true, + }, + }, + ], + }, + }, + }, + }, ]; From 51bbc02d5b00ce1b8fe8c91b4a7041e926bae9bd Mon Sep 17 00:00:00 2001 From: Sandeep Digumarty Date: Wed, 13 Nov 2024 16:30:35 +0530 Subject: [PATCH 11/32] feat: now getting consent related fields from connection config from retl for GARL (#3877) --- .../recordTransform.js | 38 +++++++++++++++++-- .../router/data.ts | 4 +- .../router/record.ts | 2 + 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js index 1c6284cd09..5866b66538 100644 --- a/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js +++ b/src/v0/destinations/google_adwords_remarketing_lists/recordTransform.js @@ -29,6 +29,8 @@ const processRecordEventArray = ( typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, operationType, ) => { let outputPayloads = {}; @@ -81,7 +83,10 @@ const processRecordEventArray = ( const toSendEvents = []; Object.values(outputPayloads).forEach((data) => { - const consentObj = populateConsentFromConfig(destination.Config, consentConfigMap); + const consentObj = populateConsentFromConfig( + { userDataConsent, personalizationConsent }, + consentConfigMap, + ); toSendEvents.push( responseBuilder(accessToken, developerToken, data, destination, audienceId, consentObj), ); @@ -96,7 +101,14 @@ function preparepayload(events, config) { const { destination, message, metadata } = events[0]; const accessToken = getAccessToken(metadata, 'access_token'); const developerToken = getValueFromMessage(metadata, 'secret.developer_token'); - const { audienceId, typeOfList, isHashRequired, userSchema } = config; + const { + audienceId, + typeOfList, + isHashRequired, + userSchema, + userDataConsent, + personalizationConsent, + } = config; const groupedRecordsByAction = lodash.groupBy(events, (record) => record.message.action?.toLowerCase(), @@ -117,6 +129,8 @@ function preparepayload(events, config) { typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, 'remove', ); } @@ -132,6 +146,8 @@ function preparepayload(events, config) { typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, 'add', ); } @@ -147,6 +163,8 @@ function preparepayload(events, config) { typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, 'add', ); } @@ -169,19 +187,29 @@ function preparepayload(events, config) { function processRecordInputsV0(groupedRecordInputs) { const { destination, message } = groupedRecordInputs[0]; - const { audienceId, typeOfList, isHashRequired, userSchema } = destination.Config; + const { + audienceId, + typeOfList, + isHashRequired, + userSchema, + userDataConsent, + personalizationConsent, + } = destination.Config; return preparepayload(groupedRecordInputs, { audienceId: getOperationAudienceId(audienceId, message), typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, }); } function processRecordInputsV1(groupedRecordInputs) { const { connection, message } = groupedRecordInputs[0]; - const { audienceId, typeOfList, isHashRequired } = connection.config.destination; + const { audienceId, typeOfList, isHashRequired, userDataConsent, personalizationConsent } = + connection.config.destination; const identifiers = message?.identifiers; let userSchema; @@ -202,6 +230,8 @@ function processRecordInputsV1(groupedRecordInputs) { typeOfList, userSchema, isHashRequired, + userDataConsent, + personalizationConsent, }); } diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts index 6878e81f0d..12d5c65f8f 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/data.ts @@ -884,8 +884,8 @@ export const data = [ listId: '7090784486', customerId: '7693729833', consent: { - adPersonalization: 'UNSPECIFIED', - adUserData: 'UNSPECIFIED', + adPersonalization: 'GRANTED', + adUserData: 'GRANTED', }, }, body: { diff --git a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts index 2661500b4d..b3f1095b1d 100644 --- a/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts +++ b/test/integrations/destinations/google_adwords_remarketing_lists/router/record.ts @@ -75,6 +75,8 @@ const connection2: Connection = { isHashRequired: true, typeOfList: 'userID', audienceId: '7090784486', + personalizationConsent: 'GRANTED', + userDataConsent: 'GRANTED', }, }, }; From e6b5fb749dcf66036257a439ce994b9aa9eacebf Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Wed, 13 Nov 2024 17:58:38 +0530 Subject: [PATCH 12/32] chore: output of conversion is optional --- .../conversionStrategies/strategyV1ToV2.ts | 2 +- .../conversionStrategies/strategyV2ToV0.ts | 2 +- .../conversionStrategies/strategyV2ToV1.ts | 2 +- src/controllers/util/index.test.ts | 2 -- src/services/source/nativeIntegration.ts | 25 ++++++++++++++----- src/types/index.ts | 2 +- 6 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/controllers/util/conversionStrategies/strategyV1ToV2.ts b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts index b4f04ef858..7cf4e77808 100644 --- a/src/controllers/util/conversionStrategies/strategyV1ToV2.ts +++ b/src/controllers/util/conversionStrategies/strategyV1ToV2.ts @@ -35,7 +35,7 @@ export class StrategyV1ToV2 extends VersionConversionStrategy, conversionError }; + return { conversionError }; } }); } diff --git a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts index d651917096..52cade0d9d 100644 --- a/src/controllers/util/conversionStrategies/strategyV2ToV1.ts +++ b/src/controllers/util/conversionStrategies/strategyV2ToV1.ts @@ -10,7 +10,7 @@ export class StrategyV2ToV1 extends VersionConversionStrategy { implementationVersion: 'v0', input: [ { - output: {}, conversionError: new SyntaxError('Unexpected end of JSON input'), }, ], @@ -322,7 +321,6 @@ describe('adaptInputToVersion', () => { implementationVersion: 'v1', input: [ { - output: {}, conversionError: new SyntaxError('Unexpected end of JSON input'), }, ], diff --git a/src/services/source/nativeIntegration.ts b/src/services/source/nativeIntegration.ts index 58a6a19649..078716df96 100644 --- a/src/services/source/nativeIntegration.ts +++ b/src/services/source/nativeIntegration.ts @@ -53,12 +53,25 @@ export class NativeIntegrationSourceService implements SourceService { metaTO, ); } - const newSourceEvent = sourceEvent.output; - const { headers } = newSourceEvent; - delete newSourceEvent.headers; - const respEvents: RudderMessage | RudderMessage[] | SourceTransformationResponse = - await sourceHandler.process(newSourceEvent); - return SourcePostTransformationService.handleSuccessEventsSource(respEvents, { headers }); + + if (sourceEvent.output) { + const newSourceEvent = sourceEvent.output; + + const { headers } = newSourceEvent; + if (headers) { + delete newSourceEvent.headers; + } + + const respEvents: RudderMessage | RudderMessage[] | SourceTransformationResponse = + await sourceHandler.process(newSourceEvent); + return SourcePostTransformationService.handleSuccessEventsSource(respEvents, { + headers, + }); + } + return SourcePostTransformationService.handleFailureEventsSource( + new Error('Error post version converstion, converstion output is undefined'), + metaTO, + ); } catch (error: FixMe) { stats.increment('source_transform_errors', { source: sourceType, diff --git a/src/types/index.ts b/src/types/index.ts index ee225bb0c0..7c07f659df 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -374,7 +374,7 @@ type SourceInputV2 = { }; type SourceInputConversionResult = { - output: T; + output?: T; conversionError?: Error; }; From e90d2ad878f1350bc5b905a27772f9dc380ce5a5 Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Wed, 13 Nov 2024 18:32:09 +0530 Subject: [PATCH 13/32] chore: added test cases for v1-v2 conversion errors --- src/controllers/util/index.test.ts | 41 ++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/controllers/util/index.test.ts b/src/controllers/util/index.test.ts index c5cefdeeeb..4559bccc52 100644 --- a/src/controllers/util/index.test.ts +++ b/src/controllers/util/index.test.ts @@ -402,6 +402,47 @@ describe('adaptInputToVersion', () => { expect(result).toEqual(expected); }); + + it('should fail trying to convert input from v1 to v2 format when the request version is v1 and the implementation version is v2', () => { + const sourceType = 'someSourceType'; + const requestVersion = 'v1'; + + // Mock return value for getSourceVersionsMap + jest + .spyOn(ControllerUtility as any, 'getSourceVersionsMap') + .mockReturnValue(new Map([['someSourceType', 'v2']])); + + const input = [ + { + event: { + key: 'value', + query_parameters: { paramkey: ['paramvalue'] }, + largeNumber: BigInt(12345678901234567890n), + }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + { + event: { key: 'value', largeNumber: BigInt(12345678901234567890n) }, + source: { id: 'source_id', config: { configField1: 'configVal1' } }, + }, + ]; + + const expected = { + implementationVersion: 'v2', + input: [ + { + conversionError: new TypeError('Do not know how to serialize a BigInt'), + }, + { + conversionError: new TypeError('Do not know how to serialize a BigInt'), + }, + ], + }; + + const result = ControllerUtility.adaptInputToVersion(sourceType, requestVersion, input); + + expect(result).toEqual(expected); + }); }); type timestampTestCases = { From 12ac3de6e7cc91a6cd52c33bc342f74bbaa8a631 Mon Sep 17 00:00:00 2001 From: Manish Kumar <144022547+manish339k@users.noreply.github.com> Date: Fri, 15 Nov 2024 14:46:40 +0530 Subject: [PATCH 14/32] feat: added support to eu/us2 datacenter for gainsight px destination (#3871) * feat: added support to eu/us2 datacenter for gainsight px destination * fix: added unit test --- src/v0/destinations/gainsight_px/config.js | 31 ++++++++++++++++--- .../destinations/gainsight_px/config.test.js | 27 ++++++++++++++++ src/v0/destinations/gainsight_px/transform.js | 13 ++++---- src/v0/destinations/gainsight_px/util.js | 10 +++--- 4 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 src/v0/destinations/gainsight_px/config.test.js diff --git a/src/v0/destinations/gainsight_px/config.js b/src/v0/destinations/gainsight_px/config.js index cc058f88d2..a5ced4f1a7 100644 --- a/src/v0/destinations/gainsight_px/config.js +++ b/src/v0/destinations/gainsight_px/config.js @@ -1,12 +1,27 @@ const { getMappingConfig } = require('../../util'); const BASE_ENDPOINT = 'https://api.aptrinsic.com/v1'; -const ENDPOINTS = { - USERS_ENDPOINT: `${BASE_ENDPOINT}/users`, - CUSTOM_EVENTS_ENDPOINT: `${BASE_ENDPOINT}/events/custom`, - ACCOUNTS_ENDPOINT: `${BASE_ENDPOINT}/accounts`, +const BASE_EU_ENDPOINT = 'https://api-eu.aptrinsic.com/v1'; +const BASE_US2_ENDPOINT = 'https://api-us2.aptrinsic.com/v1'; + +const getBaseEndpoint = (Config) => { + const { dataCenter } = Config; + switch (dataCenter) { + case 'EU': + return BASE_EU_ENDPOINT; + case 'US2': + return BASE_US2_ENDPOINT; + default: + return BASE_ENDPOINT; + } }; +const getUsersEndpoint = (Config) => `${getBaseEndpoint(Config)}/users`; + +const getCustomEventsEndpoint = (Config) => `${getBaseEndpoint(Config)}/events/custom`; + +const getAccountsEndpoint = (Config) => `${getBaseEndpoint(Config)}/accounts`; + const CONFIG_CATEGORIES = { IDENTIFY: { type: 'identify', name: 'GainsightPX_Identify' }, TRACK: { type: 'track', name: 'GainsightPX_Track' }, @@ -79,10 +94,16 @@ const ACCOUNT_EXCLUSION_FIELDS = [ ]; module.exports = { - ENDPOINTS, USER_EXCLUSION_FIELDS, ACCOUNT_EXCLUSION_FIELDS, identifyMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.IDENTIFY.name], trackMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.TRACK.name], groupMapping: MAPPING_CONFIG[CONFIG_CATEGORIES.GROUP.name], + getUsersEndpoint, + getCustomEventsEndpoint, + getAccountsEndpoint, + BASE_ENDPOINT, + BASE_EU_ENDPOINT, + BASE_US2_ENDPOINT, + getBaseEndpoint, }; diff --git a/src/v0/destinations/gainsight_px/config.test.js b/src/v0/destinations/gainsight_px/config.test.js new file mode 100644 index 0000000000..825396d350 --- /dev/null +++ b/src/v0/destinations/gainsight_px/config.test.js @@ -0,0 +1,27 @@ +const { BASE_ENDPOINT, BASE_EU_ENDPOINT, BASE_US2_ENDPOINT, getBaseEndpoint } = require('./config'); + +describe('getBaseEndpoint method test', () => { + it('Should return BASE_ENDPOINT when destination.Config.dataCenter is not "EU" or "US2"', () => { + const Config = { + dataCenter: 'US', + }; + const result = getBaseEndpoint(Config); + expect(result).toBe(BASE_ENDPOINT); + }); + + it('Should return BASE_EU_ENDPOINT when destination.Config.dataCenter is "EU"', () => { + const Config = { + dataCenter: 'EU', + }; + const result = getBaseEndpoint(Config); + expect(result).toBe(BASE_EU_ENDPOINT); + }); + + it('Should return BASE_US2_ENDPOINT when destination.Config.dataCenter is "US2"', () => { + const Config = { + dataCenter: 'US2', + }; + const result = getBaseEndpoint(Config); + expect(result).toBe(BASE_US2_ENDPOINT); + }); +}); diff --git a/src/v0/destinations/gainsight_px/transform.js b/src/v0/destinations/gainsight_px/transform.js index 0911b76b6c..496099a6b4 100644 --- a/src/v0/destinations/gainsight_px/transform.js +++ b/src/v0/destinations/gainsight_px/transform.js @@ -27,12 +27,13 @@ const { formatEventProps, } = require('./util'); const { - ENDPOINTS, USER_EXCLUSION_FIELDS, ACCOUNT_EXCLUSION_FIELDS, trackMapping, groupMapping, identifyMapping, + getUsersEndpoint, + getCustomEventsEndpoint, } = require('./config'); const { JSON_MIME_TYPE } = require('../../util/constant'); @@ -92,7 +93,7 @@ const identifyResponseBuilder = async (message, { Config }, metadata) => { if (isUserPresent) { // update user response.method = defaultPutRequestConfig.requestMethod; - response.endpoint = `${ENDPOINTS.USERS_ENDPOINT}/${userId}`; + response.endpoint = `${getUsersEndpoint(Config)}/${userId}`; response.body.JSON = removeUndefinedAndNullValues(payload); return response; } @@ -100,7 +101,7 @@ const identifyResponseBuilder = async (message, { Config }, metadata) => { // create new user payload.identifyId = userId; response.method = defaultPostRequestConfig.requestMethod; - response.endpoint = ENDPOINTS.USERS_ENDPOINT; + response.endpoint = getUsersEndpoint(Config); response.body.JSON = removeUndefinedAndNullValues(payload); return response; }; @@ -162,7 +163,7 @@ const newGroupResponseBuilder = async (message, { Config }, metadata) => { 'X-APTRINSIC-API-KEY': Config.apiKey, 'Content-Type': JSON_MIME_TYPE, }; - response.endpoint = `${ENDPOINTS.USERS_ENDPOINT}/${userId}`; + response.endpoint = `${getUsersEndpoint(Config)}/${userId}`; response.body.JSON = { accountId: groupId, }; @@ -230,7 +231,7 @@ const groupResponseBuilder = async (message, { Config }, metadata) => { 'X-APTRINSIC-API-KEY': Config.apiKey, 'Content-Type': JSON_MIME_TYPE, }; - response.endpoint = `${ENDPOINTS.USERS_ENDPOINT}/${userId}`; + response.endpoint = `${getUsersEndpoint(Config)}/${userId}`; response.body.JSON = { accountId: groupId, }; @@ -271,7 +272,7 @@ const trackResponseBuilder = (message, { Config }) => { 'X-APTRINSIC-API-KEY': Config.apiKey, 'Content-Type': JSON_MIME_TYPE, }; - response.endpoint = ENDPOINTS.CUSTOM_EVENTS_ENDPOINT; + response.endpoint = getCustomEventsEndpoint(Config); return response; }; diff --git a/src/v0/destinations/gainsight_px/util.js b/src/v0/destinations/gainsight_px/util.js index 7300189297..71d85438de 100644 --- a/src/v0/destinations/gainsight_px/util.js +++ b/src/v0/destinations/gainsight_px/util.js @@ -1,9 +1,9 @@ const { NetworkError } = require('@rudderstack/integrations-lib'); -const { ENDPOINTS } = require('./config'); const tags = require('../../util/tags'); const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); const { JSON_MIME_TYPE } = require('../../util/constant'); const { handleHttpRequest } = require('../../../adapters/network'); +const { getUsersEndpoint, getAccountsEndpoint } = require('./config'); const handleErrorResponse = (error, customErrMessage, expectedErrStatus, defaultStatus = 400) => { let destResp; @@ -38,10 +38,10 @@ const handleErrorResponse = (error, customErrMessage, expectedErrStatus, default * @returns */ const objectExists = async (id, Config, objectType, metadata) => { - let url = `${ENDPOINTS.USERS_ENDPOINT}/${id}`; + let url = `${getUsersEndpoint(Config)}/${id}`; if (objectType === 'account') { - url = `${ENDPOINTS.ACCOUNTS_ENDPOINT}/${id}`; + url = `${getAccountsEndpoint(Config)}/${id}`; } const { httpResponse: res } = await handleHttpRequest( 'get', @@ -70,7 +70,7 @@ const objectExists = async (id, Config, objectType, metadata) => { const createAccount = async (payload, Config, metadata) => { const { httpResponse: res } = await handleHttpRequest( 'post', - ENDPOINTS.ACCOUNTS_ENDPOINT, + getAccountsEndpoint(Config), payload, { headers: { @@ -96,7 +96,7 @@ const createAccount = async (payload, Config, metadata) => { const updateAccount = async (accountId, payload, Config, metadata) => { const { httpResponse: res } = await handleHttpRequest( 'put', - `${ENDPOINTS.ACCOUNTS_ENDPOINT}/${accountId}`, + `${getAccountsEndpoint(Config)}/${accountId}`, payload, { headers: { From 6554893a38102c72f6619a4dd5f361dcfe2f1d61 Mon Sep 17 00:00:00 2001 From: Utsab Chowdhury Date: Fri, 15 Nov 2024 15:21:34 +0530 Subject: [PATCH 15/32] chore: clean up Marketo implementation for file upload et al (#3872) chore. clean up marketo implementation for fileupload et al. --- src/util/fetchDestinationHandlers.ts | 10 +- .../marketo_bulk_upload/config.js | 55 -- .../marketo_bulk_upload/fetchJobStatus.js | 153 ----- .../marketo_bulk_upload/fileUpload.js | 275 --------- .../marketo_bulk_upload.util.test.js | 542 ------------------ .../destinations/marketo_bulk_upload/poll.js | 126 ---- .../marketo_bulk_upload_example.csv | 0 .../destinations/marketo_bulk_upload/util.js | 436 -------------- .../marketo_bulk_upload_fileUpload_input.json | 272 --------- ...marketo_bulk_upload_fileUpload_output.json | 63 -- .../data/marketo_bulk_upload_input.json | 270 --------- .../marketo_bulk_upload_jobStatus_input.json | 102 ---- .../marketo_bulk_upload_jobStatus_output.json | 28 - .../data/marketo_bulk_upload_output.json | 89 --- .../data/marketo_bulk_upload_poll_input.json | 59 -- .../data/marketo_bulk_upload_poll_output.json | 17 - test/__tests__/marketo_bulk_upload.test.js | 127 ---- 17 files changed, 3 insertions(+), 2621 deletions(-) delete mode 100644 src/v0/destinations/marketo_bulk_upload/config.js delete mode 100644 src/v0/destinations/marketo_bulk_upload/fetchJobStatus.js delete mode 100644 src/v0/destinations/marketo_bulk_upload/fileUpload.js delete mode 100644 src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js delete mode 100644 src/v0/destinations/marketo_bulk_upload/poll.js delete mode 100644 src/v0/destinations/marketo_bulk_upload/uploadFile/marketo_bulk_upload_example.csv delete mode 100644 src/v0/destinations/marketo_bulk_upload/util.js delete mode 100644 test/__tests__/data/marketo_bulk_upload_fileUpload_input.json delete mode 100644 test/__tests__/data/marketo_bulk_upload_fileUpload_output.json delete mode 100644 test/__tests__/data/marketo_bulk_upload_input.json delete mode 100644 test/__tests__/data/marketo_bulk_upload_jobStatus_input.json delete mode 100644 test/__tests__/data/marketo_bulk_upload_jobStatus_output.json delete mode 100644 test/__tests__/data/marketo_bulk_upload_output.json delete mode 100644 test/__tests__/data/marketo_bulk_upload_poll_input.json delete mode 100644 test/__tests__/data/marketo_bulk_upload_poll_output.json delete mode 100644 test/__tests__/marketo_bulk_upload.test.js diff --git a/src/util/fetchDestinationHandlers.ts b/src/util/fetchDestinationHandlers.ts index 2661ef2e68..fa8cbb47c3 100644 --- a/src/util/fetchDestinationHandlers.ts +++ b/src/util/fetchDestinationHandlers.ts @@ -1,22 +1,18 @@ -import * as V0MarketoBulkUploadFileUpload from '../v0/destinations/marketo_bulk_upload/fileUpload'; -import * as V0MarketoBulkUploadPollStatus from '../v0/destinations/marketo_bulk_upload/poll'; -import * as V0MarketoBulkUploadJobStatus from '../v0/destinations/marketo_bulk_upload/fetchJobStatus'; - const fileUploadHandlers = { v0: { - marketo_bulk_upload: V0MarketoBulkUploadFileUpload, + marketo_bulk_upload: undefined, }, }; const pollStatusHandlers = { v0: { - marketo_bulk_upload: V0MarketoBulkUploadPollStatus, + marketo_bulk_upload: undefined, }, }; const jobStatusHandlers = { v0: { - marketo_bulk_upload: V0MarketoBulkUploadJobStatus, + marketo_bulk_upload: undefined, }, }; diff --git a/src/v0/destinations/marketo_bulk_upload/config.js b/src/v0/destinations/marketo_bulk_upload/config.js deleted file mode 100644 index e3268711fe..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/config.js +++ /dev/null @@ -1,55 +0,0 @@ -const ABORTABLE_CODES = ['601', '603', '605', '609', '610']; -const RETRYABLE_CODES = ['713', '602', '604', '611']; -const THROTTLED_CODES = ['502', '606', '607', '608', '615']; - -const MARKETO_FILE_SIZE = 10485760; -const MARKETO_FILE_PATH = `${__dirname}/uploadFile/marketo_bulkupload.csv`; - -const FETCH_ACCESS_TOKEN = 'marketo_bulk_upload_access_token_fetching'; - -const POLL_ACTIVITY = 'marketo_bulk_upload_polling'; -const POLL_STATUS_ERR_MSG = 'Could not poll status'; - -const UPLOAD_FILE = 'marketo_bulk_upload_upload_file'; -const FILE_UPLOAD_ERR_MSG = 'Could not upload file'; - -const JOB_STATUS_ACTIVITY = 'marketo_bulk_upload_get_job_status'; -const FETCH_FAILURE_JOB_STATUS_ERR_MSG = 'Could not fetch failure job status'; -const FETCH_WARNING_JOB_STATUS_ERR_MSG = 'Could not fetch warning job status'; -const ACCESS_TOKEN_FETCH_ERR_MSG = 'Error during fetching access token'; - -const SCHEMA_DATA_TYPE_MAP = { - string: 'string', - number: 'number', - boolean: 'boolean', - undefined: 'undefined', - float: 'number', - text: 'string', - currency: 'string', - integer: 'number', - reference: 'string', - datetime: 'string', - date: 'string', - email: 'string', - phone: 'string', - url: 'string', - object: 'object', -}; - -module.exports = { - ABORTABLE_CODES, - RETRYABLE_CODES, - THROTTLED_CODES, - MARKETO_FILE_SIZE, - POLL_ACTIVITY, - UPLOAD_FILE, - JOB_STATUS_ACTIVITY, - MARKETO_FILE_PATH, - FETCH_ACCESS_TOKEN, - POLL_STATUS_ERR_MSG, - FILE_UPLOAD_ERR_MSG, - FETCH_FAILURE_JOB_STATUS_ERR_MSG, - FETCH_WARNING_JOB_STATUS_ERR_MSG, - ACCESS_TOKEN_FETCH_ERR_MSG, - SCHEMA_DATA_TYPE_MAP, -}; diff --git a/src/v0/destinations/marketo_bulk_upload/fetchJobStatus.js b/src/v0/destinations/marketo_bulk_upload/fetchJobStatus.js deleted file mode 100644 index db3b13eeb8..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/fetchJobStatus.js +++ /dev/null @@ -1,153 +0,0 @@ -/* eslint-disable no-restricted-syntax */ -/* eslint-disable no-prototype-builtins */ -const { PlatformError } = require('@rudderstack/integrations-lib'); -const { getAccessToken } = require('./util'); -const { handleHttpRequest } = require('../../../adapters/network'); -const stats = require('../../../util/stats'); -const { JSON_MIME_TYPE } = require('../../util/constant'); -const { - handleFetchJobStatusResponse, - getFieldSchemaMap, - checkEventStatusViaSchemaMatching, -} = require('./util'); -const { removeUndefinedValues } = require('../../util'); - -const getJobsStatus = async (event, type, accessToken) => { - const { config, importId } = event; - const { munchkinId } = config; - let url; - // Get status of each lead for failed leads - // DOC: https://developers.marketo.com/rest-api/bulk-import/bulk-lead-import/#failures - const requestOptions = { - headers: { - 'Content-Type': JSON_MIME_TYPE, - Authorization: `Bearer ${accessToken}`, - }, - }; - if (type === 'fail') { - url = `https://${munchkinId}.mktorest.com/bulk/v1/leads/batch/${importId}/failures.json`; - } else { - url = `https://${munchkinId}.mktorest.com/bulk/v1/leads/batch/${importId}/warnings.json`; - } - const startTime = Date.now(); - const { processedResponse: resp } = await handleHttpRequest('get', url, requestOptions, { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/leads/batch/', - requestMethod: 'GET', - module: 'router', - }); - const endTime = Date.now(); - const requestTime = endTime - startTime; - - stats.histogram('marketo_bulk_upload_fetch_job_time', requestTime); - - return handleFetchJobStatusResponse(resp, type); -}; - -/** - * Handles the response from the server based on the provided type. - * Retrieves the job status using the getJobsStatus function and processes the response data. - * Matches the response data with the data received from the server. - * Returns a response object containing the failed keys, failed reasons, warning keys, warning reasons, and succeeded keys. - * @param {Object} event - An object containing the input data and metadata. - * @param {string} type - A string indicating the type of job status to retrieve ("fail" or "warn"). - * @returns {Object} - A response object with the failed keys, failed reasons, warning keys, warning reasons, and succeeded keys. - */ -const responseHandler = async (event, type) => { - let FailedKeys = []; - const unsuccessfulJobIdsArr = []; - let successfulJobIdsArr = []; - let reasons = {}; - - const { config } = event; - const accessToken = await getAccessToken(config); - - /** - * { - "FailedKeys" : [jobID1,jobID3], - "FailedReasons" : { - "jobID1" : "failure-reason-1", - "jobID3" : "failure-reason-2", - }, - "WarningKeys" : [jobID2,jobID4], - "WarningReasons" : { - "jobID2" : "warning-reason-1", - "jobID4" : "warning-reason-2", - }, - "SucceededKeys" : [jobID5] -} - */ - - const jobStatus = - type === 'fail' - ? await getJobsStatus(event, 'fail', accessToken) - : await getJobsStatus(event, 'warn', accessToken); - const jobStatusArr = jobStatus.toString().split('\n'); // responseArr = ['field1,field2,Import Failure Reason', 'val1,val2,reason',...] - const { input, metadata } = event; - let headerArr; - if (metadata?.csvHeader) { - headerArr = metadata.csvHeader.split(','); - } else { - throw new PlatformError('No csvHeader in metadata'); - } - const startTime = Date.now(); - const data = {}; - const fieldSchemaMapping = await getFieldSchemaMap(accessToken, config.munchkinId); - const unsuccessfulJobInfo = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - const mismatchJobIdArray = Object.keys(unsuccessfulJobInfo); - const dataTypeMismatchKeys = mismatchJobIdArray.map((strJobId) => parseInt(strJobId, 10)); - reasons = { ...unsuccessfulJobInfo }; - - const filteredEvents = input.filter( - (item) => !dataTypeMismatchKeys.includes(item.metadata.job_id), - ); - // create a map of job_id and data sent from server - // {: ','} - filteredEvents.forEach((i) => { - const response = headerArr.map((fieldName) => Object.values(i)[0][fieldName]).join(','); - data[i.metadata.job_id] = response; - }); - - // match marketo response data with received data from server - for (const element of jobStatusArr) { - // split response by comma but ignore commas inside double quotes - const elemArr = element.split(/,(?=(?:(?:[^"]*"){2})*[^"]*$)/); - // ref : - // https://developers.marketo.com/rest-api/bulk-import/bulk-custom-object-import/#:~:text=Now%20we%E2%80%99ll%20make%20Get%20Import%20Custom%20Object%20Failures%20endpoint%20call%20to%20get%20additional%20failure%20detail%3A - const reasonMessage = elemArr.pop(); // get the column named "Import Failure Reason" - for (const [key, val] of Object.entries(data)) { - // joining the parameter values sent from marketo match it with received data from server - if (val === `${elemArr.map((item) => item.replace(/"/g, '')).join(',')}`) { - // add job keys if warning/failure - if (!unsuccessfulJobIdsArr.includes(key)) { - unsuccessfulJobIdsArr.push(key); - } - reasons[key] = reasonMessage; - } - } - } - - FailedKeys = unsuccessfulJobIdsArr.map((strJobId) => parseInt(strJobId, 10)); - successfulJobIdsArr = Object.keys(data).filter((x) => !unsuccessfulJobIdsArr.includes(x)); - - const SucceededKeys = successfulJobIdsArr.map((strJobId) => parseInt(strJobId, 10)); - const endTime = Date.now(); - const requestTime = endTime - startTime; - stats.histogram('marketo_bulk_upload_fetch_job_create_response_time', requestTime); - const response = { - statusCode: 200, - metadata: { - FailedKeys: [...dataTypeMismatchKeys, ...FailedKeys], - FailedReasons: reasons, - SucceededKeys, - }, - }; - return removeUndefinedValues(response); -}; - -const processJobStatus = async (event, type) => { - const resp = await responseHandler(event, type); - return resp; -}; -module.exports = { processJobStatus }; diff --git a/src/v0/destinations/marketo_bulk_upload/fileUpload.js b/src/v0/destinations/marketo_bulk_upload/fileUpload.js deleted file mode 100644 index b49a265fd5..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/fileUpload.js +++ /dev/null @@ -1,275 +0,0 @@ -/* eslint-disable no-plusplus */ -const FormData = require('form-data'); -const fs = require('fs'); -const { - NetworkError, - ConfigurationError, - RetryableError, - TransformationError, -} = require('@rudderstack/integrations-lib'); -const { - getAccessToken, - getMarketoFilePath, - handleFileUploadResponse, - getFieldSchemaMap, - hydrateStatusForServer, -} = require('./util'); -const { isHttpStatusSuccess } = require('../../util'); -const { MARKETO_FILE_SIZE, UPLOAD_FILE } = require('./config'); -const { - getHashFromArray, - removeUndefinedAndNullValues, - isDefinedAndNotNullAndNotEmpty, -} = require('../../util'); -const { handleHttpRequest } = require('../../../adapters/network'); -const { client } = require('../../../util/errorNotifier'); -const stats = require('../../../util/stats'); - -const fetchFieldSchemaNames = async (config, accessToken) => { - const fieldSchemaMapping = await getFieldSchemaMap(accessToken, config.munchkinId); - if (Object.keys(fieldSchemaMapping).length > 0) { - const fieldSchemaNames = Object.keys(fieldSchemaMapping); - return { fieldSchemaNames }; - } - throw new RetryableError('Failed to fetch Marketo Field Schema', 500, fieldSchemaMapping); -}; - -const getHeaderFields = (config, fieldSchemaNames) => { - const { columnFieldsMapping } = config; - - columnFieldsMapping.forEach((colField) => { - if (!fieldSchemaNames.includes(colField.to)) { - throw new ConfigurationError( - `The field ${colField.to} is not present in Marketo Field Schema. Aborting`, - ); - } - }); - const columnField = getHashFromArray(columnFieldsMapping, 'to', 'from', false); - return Object.keys(columnField); -}; -/** - * Processes input data to create a CSV file and returns the file data along with successful and unsuccessful job IDs. - * The file name is made unique with combination of UUID and current timestamp to avoid any overrides. It also has a - * maximum size limit of 10MB . The events that could be accomodated inside the file is marked as successful and the - * rest are marked as unsuccessful. Also the file is deleted when reading is complete. - * @param {Array} inputEvents - An array of input events. - * @param {Object} config - destination config - * @param {Array} headerArr - An array of header fields. - * @returns {Object} - An object containing the file stream, successful job IDs, and unsuccessful job IDs. - */ -const getFileData = async (inputEvents, config, headerArr) => { - const input = inputEvents; - const messageArr = []; - let startTime; - let endTime; - let requestTime; - startTime = Date.now(); - - input.forEach((i) => { - const inputData = i; - const jobId = inputData.metadata.job_id; - const data = {}; - data[jobId] = inputData.message; - messageArr.push(data); - }); - - if (isDefinedAndNotNullAndNotEmpty(config.deDuplicationField)) { - // dedup starts - // Time Complexity = O(n2) - const dedupMap = new Map(); - // iterating input and storing the occurences of messages - // with same dedup property received from config - // Example: dedup-property = email - // k (key) v (index of occurence in input) - // user@email [4,7,9] - // user2@email [2,3] - // user3@email [1] - input.forEach((element, index) => { - const indexAr = dedupMap.get(element.message[config.deDuplicationField]) || []; - indexAr.push(index); - dedupMap.set(element.message[config.deDuplicationField], indexAr); - return dedupMap; - }); - // 1. iterating dedupMap - // 2. storing the duplicate occurences in dupValues arr - // 3. iterating dupValues arr, and mapping each property on firstBorn - // 4. as dupValues arr is sorted hence the firstBorn will inherit properties of last occurence (most updated one) - // 5. store firstBorn to first occurence in input as it should get the highest priority - dedupMap.forEach((indexes) => { - let firstBorn = {}; - indexes.forEach((idx) => { - headerArr.forEach((headerStr) => { - // if duplicate item has defined property to offer we take it else old one remains - firstBorn[headerStr] = input[idx].message[headerStr] || firstBorn[headerStr]; - }); - }); - firstBorn = removeUndefinedAndNullValues(firstBorn); - input[indexes[0]].message = firstBorn; - }); - // dedup ends - } - - const csv = []; - csv.push(headerArr.toString()); - endTime = Date.now(); - requestTime = endTime - startTime; - stats.histogram('marketo_bulk_upload_create_header_time', requestTime); - const unsuccessfulJobs = []; - const successfulJobs = []; - const MARKETO_FILE_PATH = getMarketoFilePath(); - startTime = Date.now(); - messageArr.forEach((row) => { - const csvSize = JSON.stringify(csv); // stringify and remove all "stringification" extra data - const response = headerArr - .map((fieldName) => JSON.stringify(Object.values(row)[0][fieldName], '')) - .join(','); - if (csvSize.length <= MARKETO_FILE_SIZE) { - csv.push(response); - successfulJobs.push(Object.keys(row)[0]); - } else { - unsuccessfulJobs.push(Object.keys(row)[0]); - } - }); - endTime = Date.now(); - requestTime = endTime - startTime; - stats.histogram('marketo_bulk_upload_create_csvloop_time', requestTime); - const fileSize = Buffer.from(csv.join('\n')).length; - if (csv.length > 1) { - startTime = Date.now(); - fs.writeFileSync(MARKETO_FILE_PATH, csv.join('\n')); - const readStream = fs.readFileSync(MARKETO_FILE_PATH); - fs.unlinkSync(MARKETO_FILE_PATH); - endTime = Date.now(); - requestTime = endTime - startTime; - stats.histogram('marketo_bulk_upload_create_file_time', requestTime); - stats.histogram('marketo_bulk_upload_upload_file_size', fileSize); - - return { readStream, successfulJobs, unsuccessfulJobs }; - } - return { successfulJobs, unsuccessfulJobs }; -}; - -const getImportID = async (input, config, accessToken, csvHeader) => { - let readStream; - let successfulJobs; - let unsuccessfulJobs; - try { - ({ readStream, successfulJobs, unsuccessfulJobs } = await getFileData( - input, - config, - csvHeader, - )); - } catch (err) { - client.notify(err, `Marketo File Upload: Error while creating file: ${err.message}`, { - config, - csvHeader, - }); - throw new TransformationError( - `Marketo File Upload: Error while creating file: ${err.message}`, - 500, - ); - } - - const formReq = new FormData(); - const { munchkinId, deDuplicationField } = config; - // create file for multipart form - if (readStream) { - formReq.append('format', 'csv'); - formReq.append('file', readStream, 'marketo_bulk_upload.csv'); - formReq.append('access_token', accessToken); - // Upload data received from server as files to marketo - // DOC: https://developers.marketo.com/rest-api/bulk-import/bulk-lead-import/#import_file - const requestOptions = { - headers: { - ...formReq.getHeaders(), - }, - }; - if (isDefinedAndNotNullAndNotEmpty(deDuplicationField)) { - requestOptions.params = { - lookupField: deDuplicationField, - }; - } - const startTime = Date.now(); - const { processedResponse: resp } = await handleHttpRequest( - 'post', - `https://${munchkinId}.mktorest.com/bulk/v1/leads.json`, - formReq, - requestOptions, - { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/leads.json', - requestMethod: 'POST', - module: 'router', - }, - ); - const endTime = Date.now(); - const requestTime = endTime - startTime; - stats.counter('marketo_bulk_upload_upload_file_succJobs', successfulJobs.length); - stats.counter('marketo_bulk_upload_upload_file_unsuccJobs', unsuccessfulJobs.length); - if (!isHttpStatusSuccess(resp.status)) { - throw new NetworkError( - `Unable to upload file due to error : ${JSON.stringify(resp.response)}`, - hydrateStatusForServer(resp.status, 'During uploading file'), - ); - } - return handleFileUploadResponse(resp, successfulJobs, unsuccessfulJobs, requestTime); - } - return { importId: null, successfulJobs, unsuccessfulJobs }; -}; - -/** - * - * @param {*} input - * @param {*} config - * @returns returns the final response of fileUpload.js - */ -const responseHandler = async (input, config) => { - const accessToken = await getAccessToken(config); - /** - { - "importId" : , - "pollURL" : , - } - */ - const { fieldSchemaNames } = await fetchFieldSchemaNames(config, accessToken); - const headerForCsv = getHeaderFields(config, fieldSchemaNames); - if (Object.keys(headerForCsv).length === 0) { - throw new ConfigurationError( - 'Faulty configuration. Please map your traits to Marketo column fields', - ); - } - const { importId, successfulJobs, unsuccessfulJobs } = await getImportID( - input, - config, - accessToken, - headerForCsv, - ); - // if upload is successful - if (importId) { - const csvHeader = headerForCsv.toString(); - const metadata = { successfulJobs, unsuccessfulJobs, csvHeader }; - const response = { - statusCode: 200, - importId, - metadata, - }; - return response; - } - // if importId is returned null - stats.increment(UPLOAD_FILE, { - status: 500, - state: 'Retryable', - }); - return { - statusCode: 500, - FailedReason: '[Marketo File upload]: No import id received', - }; -}; -const processFileData = async (event) => { - const { input, config } = event; - const resp = await responseHandler(input, config); - return resp; -}; - -module.exports = { processFileData }; diff --git a/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js b/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js deleted file mode 100644 index 13e1b3a09a..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/marketo_bulk_upload.util.test.js +++ /dev/null @@ -1,542 +0,0 @@ -const { - handleCommonErrorResponse, - handlePollResponse, - handleFileUploadResponse, - getAccessToken, - checkEventStatusViaSchemaMatching, -} = require('./util'); - -const { - AbortedError, - RetryableError, - NetworkError, - TransformationError, -} = require('@rudderstack/integrations-lib'); -const util = require('./util.js'); -const networkAdapter = require('../../../adapters/network'); -const { handleHttpRequest } = networkAdapter; - -// Mock the handleHttpRequest function -jest.mock('../../../adapters/network'); - -const successfulResponse = { - status: 200, - response: { - access_token: '', - token_type: 'bearer', - expires_in: 3600, - scope: 'dummy@scope.com', - success: true, - }, -}; - -const unsuccessfulResponse = { - status: 400, - response: '[ENOTFOUND] :: DNS lookup failed', -}; - -const emptyResponse = { - response: '', -}; - -const invalidClientErrorResponse = { - status: 401, - response: { - error: 'invalid_client', - error_description: 'Bad client credentials', - }, -}; - -describe('handleCommonErrorResponse', () => { - test('should throw AbortedError for abortable error codes', () => { - const resp = { - response: { - errors: [{ code: 1003, message: 'Aborted' }], - }, - }; - expect(() => handleCommonErrorResponse(resp, 'opErrorMessage', 'opActivity')).toThrow( - AbortedError, - ); - }); - - test('should throw ThrottledError for throttled error codes', () => { - const resp = { - response: { - errors: [{ code: 615, message: 'Throttled' }], - }, - }; - expect(() => handleCommonErrorResponse(resp, 'opErrorMessage', 'opActivity')).toThrow( - RetryableError, - ); - }); - - test('should throw RetryableError for other error codes', () => { - const resp = { - response: { - errors: [{ code: 2000, message: 'Retryable' }], - }, - }; - expect(() => handleCommonErrorResponse(resp, 'opErrorMessage', 'opActivity')).toThrow( - RetryableError, - ); - }); - - test('should throw RetryableError by default', () => { - const resp = { - response: { - errors: [], - }, - }; - expect(() => handleCommonErrorResponse(resp, 'opErrorMessage', 'opActivity')).toThrow( - RetryableError, - ); - }); -}); - -describe('handlePollResponse', () => { - // Tests that the function returns the response object if the polling operation was successful - it('should return the response object when the polling operation was successful', () => { - const pollStatus = { - response: { - success: true, - result: [ - { - batchId: '123', - status: 'Complete', - numOfLeadsProcessed: 2, - numOfRowsFailed: 1, - numOfRowsWithWarning: 0, - message: 'Import completed with errors, 2 records imported (2 members), 1 failed', - }, - ], - }, - }; - - const result = handlePollResponse(pollStatus); - - expect(result).toEqual(pollStatus.response); - }); - - // Tests that the function throws an AbortedError if the response contains an abortable error code - it('should throw an AbortedError when the response contains an abortable error code', () => { - const pollStatus = { - response: { - errors: [ - { - code: 1003, - message: 'Empty file', - }, - ], - }, - }; - - expect(() => handlePollResponse(pollStatus)).toThrow(AbortedError); - }); - - // Tests that the function throws a ThrottledError if the response contains a throttled error code - it('should throw a ThrottledError when the response contains a throttled error code', () => { - const pollStatus = { - response: { - errors: [ - { - code: 615, - message: 'Exceeded concurrent usage limit', - }, - ], - }, - }; - - expect(() => handlePollResponse(pollStatus)).toThrow(RetryableError); - }); - - // Tests that the function throws a RetryableError if the response contains an error code that is not abortable or throttled - it('should throw a RetryableError when the response contains an error code that is not abortable or throttled', () => { - const pollStatus = { - response: { - errors: [ - { - code: 601, - message: 'Unauthorized', - }, - ], - }, - }; - - expect(() => handlePollResponse(pollStatus)).toThrow(RetryableError); - }); - - // Tests that the function returns null if the polling operation was not successful - it('should return null when the polling operation was not successful', () => { - const pollStatus = { - response: { - success: false, - }, - }; - - const result = handlePollResponse(pollStatus); - - expect(result).toBeNull(); - }); -}); - -describe('handleFileUploadResponse', () => { - // Tests that the function returns an object with importId, successfulJobs, and unsuccessfulJobs when the response indicates a successful upload. - it('should return an object with importId, successfulJobs, and unsuccessfulJobs when the response indicates a successful upload', () => { - const resp = { - response: { - success: true, - result: [ - { - importId: '3404', - status: 'Queued', - }, - ], - }, - }; - const successfulJobs = []; - const unsuccessfulJobs = []; - const requestTime = 100; - - const result = handleFileUploadResponse(resp, successfulJobs, unsuccessfulJobs, requestTime); - - expect(result).toEqual({ - importId: '3404', - successfulJobs: [], - unsuccessfulJobs: [], - }); - }); - - // Tests that the function throws a RetryableError when the response indicates an empty file. - it('should throw a RetryableError when the response indicates an empty file', () => { - const resp = { - response: { - errors: [ - { - code: '1003', - message: 'Empty File', - }, - ], - }, - }; - const successfulJobs = []; - const unsuccessfulJobs = []; - const requestTime = 100; - - expect(() => { - handleFileUploadResponse(resp, successfulJobs, unsuccessfulJobs, requestTime); - }).toThrow(RetryableError); - }); - - // Tests that the function throws a RetryableError when the response indicates more than 10 concurrent uses. - it('should throw a RetryableError when the response indicates more than 10 concurrent uses', () => { - const resp = { - response: { - errors: [ - { - code: '615', - message: 'Concurrent Use Limit Exceeded', - }, - ], - }, - }; - const successfulJobs = []; - const unsuccessfulJobs = []; - const requestTime = 100; - - expect(() => { - handleFileUploadResponse(resp, successfulJobs, unsuccessfulJobs, requestTime); - }).toThrow(RetryableError); - }); - - // Tests that the function throws a RetryableError when the response contains an error code between 1000 and 1077. - it('should throw a Aborted when the response contains an error code between 1000 and 1077', () => { - const resp = { - response: { - errors: [ - { - code: 1001, - message: 'Some Error', - }, - ], - }, - }; - const successfulJobs = []; - const unsuccessfulJobs = []; - const requestTime = 100; - - expect(() => { - handleFileUploadResponse(resp, successfulJobs, unsuccessfulJobs, requestTime); - }).toThrow(AbortedError); - }); -}); - -describe('getAccessToken', () => { - beforeEach(() => { - handleHttpRequest.mockClear(); - }); - - it('should retrieve and return access token on successful response', async () => { - const url = - 'https://dummyMunchkinId.mktorest.com/identity/oauth/token?client_id=dummyClientId&client_secret=dummyClientSecret&grant_type=client_credentials'; - - handleHttpRequest.mockResolvedValueOnce({ - processedResponse: successfulResponse, - }); - - const config = { - clientId: 'dummyClientId', - clientSecret: 'dummyClientSecret', - munchkinId: 'dummyMunchkinId', - }; - - const result = await getAccessToken(config); - expect(result).toBe(''); - expect(handleHttpRequest).toHaveBeenCalledTimes(1); - // Ensure your mock response structure is consistent with the actual behavior - expect(handleHttpRequest).toHaveBeenCalledWith('get', url, { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/identity/oauth/token', - feature: 'transformation', - module: 'router', - requestMethod: 'GET', - }); - }); - - it('should throw a NetworkError on unsuccessful HTTP status', async () => { - handleHttpRequest.mockResolvedValueOnce({ - processedResponse: unsuccessfulResponse, - }); - - const config = { - clientId: 'dummyClientId', - clientSecret: 'dummyClientSecret', - munchkinId: 'dummyMunchkinId', - }; - - await expect(getAccessToken(config)).rejects.toThrow(NetworkError); - }); - - it('should throw a RetryableError when expires_in is 0', async () => { - handleHttpRequest.mockResolvedValueOnce({ - processedResponse: { - ...successfulResponse, - response: { ...successfulResponse.response, expires_in: 0 }, - }, - }); - - const config = { - clientId: 'dummyClientId', - clientSecret: 'dummyClientSecret', - munchkinId: 'dummyMunchkinId', - }; - - await expect(getAccessToken(config)).rejects.toThrow(RetryableError); - }); - - it('should throw an AbortedError on unsuccessful response', async () => { - handleHttpRequest.mockResolvedValueOnce({ processedResponse: invalidClientErrorResponse }); - - const config = { - clientId: 'invalidClientID', - clientSecret: 'dummyClientSecret', - munchkinId: 'dummyMunchkinId', - }; - - await expect(getAccessToken(config)).rejects.toThrow(NetworkError); - }); - - it('should throw transformation error response', async () => { - handleHttpRequest.mockResolvedValueOnce({ processedResponse: emptyResponse }); - - const config = { - clientId: 'dummyClientId', - clientSecret: 'dummyClientSecret', - munchkinId: 'dummyMunchkinId', - }; - - await expect(getAccessToken(config)).rejects.toThrow(TransformationError); - }); -}); - -describe('checkEventStatusViaSchemaMatching', () => { - // The function correctly identifies fields with expected data types. - it('if event data types match with expected data types we send no field as mismatch', () => { - const event = { - input: [ - { - message: { - email: 'value1', - id: 123, - isLead: true, - }, - metadata: { - job_id: 'job1', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'string', - id: 'integer', - isLead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({}); - }); - - // The function correctly identifies fields with unexpected data types. - it('if event data types do not match with expected data types we send that field as mismatch', () => { - const event = { - input: [ - { - message: { - email: 123, - city: '123', - islead: true, - }, - metadata: { - job_id: 'job1', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'string', - city: 'number', - islead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({ - job1: 'invalid email', - }); - }); - - // The function correctly handles events with multiple fields. - it('For array of events the mismatch object fills up with each event errors', () => { - const event = { - input: [ - { - message: { - id: 'value1', - testCustomFieldScore: 123, - isLead: true, - }, - metadata: { - job_id: 'job1', - }, - }, - { - message: { - email: 'value2', - id: 456, - testCustomFieldScore: false, - }, - metadata: { - job_id: 'job2', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'email', - id: 'integer', - testCustomFieldScore: 'integer', - isLead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({ - job1: 'invalid id', - job2: 'invalid testCustomFieldScore', - }); - }); - - // The function correctly handles events with missing fields. - it('it is not mandatory to send all the fields present in schema', () => { - const event = { - input: [ - { - message: { - email: 'value1', - isLead: true, - }, - metadata: { - job_id: 'job1', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'string', - id: 'number', - isLead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({}); - }); - - // The function correctly handles events with additional fields. But this will not happen in our use case - it('for any field beyond schema fields will be mapped as invalid', () => { - const event = { - input: [ - { - message: { - email: 'value1', - id: 124, - isLead: true, - abc: 'value2', - }, - metadata: { - job_id: 'job1', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'string', - id: 'number', - isLead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({ - job1: 'invalid abc', - }); - }); - - // The function correctly handles events with null values. - it('should ignore event properties with null values', () => { - const event = { - input: [ - { - message: { - email: 'value1', - id: null, - isLead: true, - }, - metadata: { - job_id: 'job1', - }, - }, - ], - }; - const fieldSchemaMapping = { - email: 'string', - id: 'number', - isLead: 'boolean', - }; - - const result = checkEventStatusViaSchemaMatching(event, fieldSchemaMapping); - - expect(result).toEqual({}); - }); -}); diff --git a/src/v0/destinations/marketo_bulk_upload/poll.js b/src/v0/destinations/marketo_bulk_upload/poll.js deleted file mode 100644 index f53347d6e5..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/poll.js +++ /dev/null @@ -1,126 +0,0 @@ -const { NetworkError } = require('@rudderstack/integrations-lib'); -const { removeUndefinedValues, isHttpStatusSuccess } = require('../../util'); -const { getAccessToken, handlePollResponse, hydrateStatusForServer } = require('./util'); -const { handleHttpRequest } = require('../../../adapters/network'); -const stats = require('../../../util/stats'); -const { JSON_MIME_TYPE } = require('../../util/constant'); -const { POLL_ACTIVITY } = require('./config'); - -const getPollStatus = async (event) => { - const accessToken = await getAccessToken(event.config); - const { munchkinId } = event.config; - - // To see the status of the import job polling is done - // DOC: https://developers.marketo.com/rest-api/bulk-import/bulk-lead-import/#polling_job_status - const requestOptions = { - headers: { - 'Content-Type': JSON_MIME_TYPE, - Authorization: `Bearer ${accessToken}`, - }, - }; - const pollUrl = `https://${munchkinId}.mktorest.com/bulk/v1/leads/batch/${event.importId}.json`; - const { processedResponse: pollStatus } = await handleHttpRequest( - 'get', - pollUrl, - requestOptions, - { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/leads/batch/importId.json', - requestMethod: 'GET', - module: 'router', - }, - ); - if (!isHttpStatusSuccess(pollStatus.status)) { - stats.counter(POLL_ACTIVITY, 1, { - status: pollStatus.status, - state: 'Retryable', - }); - throw new NetworkError( - `Could not poll status: due to error ${JSON.stringify(pollStatus.response)}`, - hydrateStatusForServer(pollStatus.status, 'During fetching poll status'), - ); - } - return handlePollResponse(pollStatus); -}; - -const responseHandler = async (event) => { - const pollResp = await getPollStatus(event); - // Server expects : - /** - * - * { - "Complete": true, - "statusCode": 200, - "hasFailed": true, - "InProgress": false, - "FailedJobURLs": "", // transformer URL - "HasWarning": false, - "WarningJobURLs": "", // transformer URL - } // Succesful Upload - { - "success": false, - "statusCode": 400, - "errorResponse": - } // Failed Upload - { - "success": false, - "Inprogress": true, - statusCode: 500, - } // Importing or Queue - - */ - if (pollResp) { - // As marketo lead import API or bulk API does not support record level error response we are considering - // file level errors only. - // ref: https://nation.marketo.com/t5/ideas/support-error-code-in-record-level-in-lead-bulk-api/idi-p/262191 - const { status, numOfRowsFailed, numOfRowsWithWarning, message } = pollResp.result[0]; - if (status === 'Complete') { - const response = { - Complete: true, - statusCode: 200, - InProgress: false, - hasFailed: numOfRowsFailed > 0, - FailedJobURLs: numOfRowsFailed > 0 ? '/getFailedJobs' : undefined, - HasWarning: numOfRowsWithWarning > 0, - WarningJobURLs: numOfRowsWithWarning > 0 ? '/getWarningJobs' : undefined, - }; - return removeUndefinedValues(response); - } - if (status === 'Importing' || status === 'Queued') { - return { - Complete: false, - statusCode: 500, - hasFailed: false, - InProgress: true, - HasWarning: false, - }; - } - if (status === 'Failed') { - return { - Complete: false, - statusCode: 500, - hasFailed: false, - InProgress: false, - HasWarning: false, - Error: message || 'Marketo Poll Status Failed', - }; - } - } - // when pollResp is null - return { - Complete: false, - statusCode: 500, - hasFailed: false, - InProgress: false, - HasWarning: false, - Error: 'No poll response received from Marketo', - }; -}; - -const processPolling = async (event) => { - const resp = await responseHandler(event); - return resp; -}; - -module.exports = { processPolling }; diff --git a/src/v0/destinations/marketo_bulk_upload/uploadFile/marketo_bulk_upload_example.csv b/src/v0/destinations/marketo_bulk_upload/uploadFile/marketo_bulk_upload_example.csv deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/v0/destinations/marketo_bulk_upload/util.js b/src/v0/destinations/marketo_bulk_upload/util.js deleted file mode 100644 index 033239b5e4..0000000000 --- a/src/v0/destinations/marketo_bulk_upload/util.js +++ /dev/null @@ -1,436 +0,0 @@ -const { - AbortedError, - RetryableError, - NetworkError, - TransformationError, - isDefinedAndNotNull, -} = require('@rudderstack/integrations-lib'); -const { handleHttpRequest } = require('../../../adapters/network'); -const tags = require('../../util/tags'); -const { isHttpStatusSuccess, generateUUID } = require('../../util'); -const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); -const stats = require('../../../util/stats'); -const { - ABORTABLE_CODES, - THROTTLED_CODES, - POLL_ACTIVITY, - UPLOAD_FILE, - FETCH_ACCESS_TOKEN, - POLL_STATUS_ERR_MSG, - FILE_UPLOAD_ERR_MSG, - ACCESS_TOKEN_FETCH_ERR_MSG, - SCHEMA_DATA_TYPE_MAP, -} = require('./config'); -const logger = require('../../../logger'); - -const getMarketoFilePath = () => - `${__dirname}/uploadFile/${Date.now()}_marketo_bulk_upload_${generateUUID()}.csv`; - -// Server only aborts when status code is 400 -const hydrateStatusForServer = (statusCode, context) => { - const status = Number(statusCode); - if (Number.isNaN(status)) { - throw new TransformationError(`${context}: Couldn't parse status code ${statusCode}`); - } - if (status >= 400 && status <= 499) { - return 400; - } - return status; -}; - -/** - * Handles common error responses returned from API calls. - * Checks the error code and throws the appropriate error object based on the code. - * - * @param {object} resp - The response object containing the error information. - * @param {string} opErrorMessage - The error message to be used if the error code is not recognized. - * @param {string} opActivity - The activity name for tracking purposes. - * @throws {AbortedError} - If the error code is abortable. - * @throws {ThrottledError} - If the error code is within the range of throttled codes. - * @throws {RetryableError} - If the error code is neither abortable nor throttled. - * - * @example - * const resp = { - * response: { - * errors: [ - * { - * code: "1003", - * message: "Empty File" - * } - * ] - * } - * }; - * - * try { - * handleCommonErrorResponse(resp, "Error message", "Activity"); - * } catch (error) { - * console.log(error); - * } - */ -const handleCommonErrorResponse = (apiCallResult, opErrorMessage, opActivity) => { - // checking for invalid/expired token errors and evicting cache in that case - // rudderJobMetadata contains some destination info which is being used to evict the cache - if ( - apiCallResult.response?.errors && - apiCallResult.response?.errors?.length > 0 && - apiCallResult.response?.errors.some( - (errorObj) => errorObj.code === '601' || errorObj.code === '602', - ) - ) { - throw new RetryableError( - `[${opErrorMessage}]Error message: ${apiCallResult.response?.errors[0]?.message}`, - ); - } - if ( - apiCallResult.response?.errors?.length > 0 && - apiCallResult.response?.errors[0] && - ((apiCallResult.response?.errors[0]?.code >= 1000 && - apiCallResult.response?.errors[0]?.code <= 1077) || - ABORTABLE_CODES.includes(apiCallResult.response?.errors[0]?.code)) - ) { - // for empty file the code is 1003 and that should be retried - stats.increment(opActivity, { - status: 400, - state: 'Abortable', - }); - throw new AbortedError(apiCallResult.response?.errors[0]?.message || opErrorMessage, 400); - } else if (THROTTLED_CODES.includes(apiCallResult.response?.errors[0]?.code)) { - // for more than 10 concurrent uses the code is 615 and that should be retried - stats.increment(opActivity, { - status: 429, - state: 'Retryable', - }); - throw new RetryableError( - `[${opErrorMessage}]Error message: ${apiCallResult.response?.errors[0]?.message}`, - 500, - ); - } - // by default every thing will be retried - stats.increment(opActivity, { - status: 500, - state: 'Retryable', - }); - throw new RetryableError( - `[${opErrorMessage}]Error message: ${apiCallResult.response?.errors[0]?.message}`, - 500, - ); -}; - -const getAccessTokenURL = (config) => { - const { clientId, clientSecret, munchkinId } = config; - const url = `https://${munchkinId}.mktorest.com/identity/oauth/token?client_id=${clientId}&client_secret=${clientSecret}&grant_type=client_credentials`; - return url; -}; - -// Fetch access token from client id and client secret -// DOC: https://developers.marketo.com/rest-api/authentication/ -const getAccessToken = async (config) => { - const url = getAccessTokenURL(config); - const { processedResponse: accessTokenResponse } = await handleHttpRequest('get', url, { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/identity/oauth/token', - requestMethod: 'GET', - module: 'router', - }); - - // sample response : {response: '[ENOTFOUND] :: DNS lookup failed', status: 400} - if (!isHttpStatusSuccess(accessTokenResponse.status)) { - throw new NetworkError( - `Could not retrieve authorisation token due to error ${JSON.stringify(accessTokenResponse)}`, - hydrateStatusForServer(accessTokenResponse.status, FETCH_ACCESS_TOKEN), - { - [tags.TAG_NAMES.ERROR_TYPE]: getDynamicErrorType(accessTokenResponse.status), - }, - accessTokenResponse, - ); - } - if (accessTokenResponse.response?.success === false) { - handleCommonErrorResponse(accessTokenResponse, ACCESS_TOKEN_FETCH_ERR_MSG, FETCH_ACCESS_TOKEN); - } - - // when access token is present - if (accessTokenResponse.response.access_token) { - /* This scenario will handle the case when we get the following response - status: 200 - respnse: {"access_token":"","token_type":"bearer","expires_in":0,"scope":"dummy@scope.com"} - wherein "expires_in":0 denotes that we should refresh the accessToken but its not expired yet. - */ - if (accessTokenResponse.response?.expires_in === 0) { - throw new RetryableError( - `Request Failed for marketo_bulk_upload, Access Token Expired (Retryable).`, - 500, - ); - } - return accessTokenResponse.response.access_token; - } - throw new RetryableError( - `Could not retrieve authorisation token due to error ${JSON.stringify(accessTokenResponse)}`, - 500, - ); -}; - -/** - * Handles the response of a polling operation. - * Checks for any errors in the response and calls the `handleCommonErrorResponse` function to handle them. - * If the response is successful, increments the stats and returns the response. - * Otherwise, returns null. - * - * @param {object} pollStatus - The response object from the polling operation. - * @returns {object|null} - The response object if the polling operation was successful, otherwise null. - */ -const handlePollResponse = (pollStatus) => { - // DOC: https://developers.marketo.com/rest-api/error-codes/ - if (pollStatus.response.errors) { - /* Sample error response for poll is: - - { - "requestId": "e42b#14272d07d78", - "success": false, - "errors": [ - { - "code": "601", - "message": "Unauthorized" - } - ] - } - */ - handleCommonErrorResponse(pollStatus, POLL_STATUS_ERR_MSG, POLL_ACTIVITY); - } - - /* - Sample Successful Poll response structure: - { - "requestId":"8136#146daebc2ed", - "success":true, - "result":[ - { - "batchId":, - "status":"Complete", - "numOfLeadsProcessed":2, - "numOfRowsFailed":1, - "numOfRowsWithWarning":0, - "message":"Import completed with errors, 2 records imported (2 members), 1 failed" - } - ] - } - */ - if (pollStatus.response?.success) { - stats.counter(POLL_ACTIVITY, 1, { - status: 200, - state: 'Success', - }); - - if (pollStatus.response?.result?.length > 0) { - return pollStatus.response; - } - } - - return null; -}; - -const handleFetchJobStatusResponse = (resp, type) => { - const marketoResponse = resp.response; - const marketoReposnseStatus = resp.status; - - if (!isHttpStatusSuccess(marketoReposnseStatus)) { - logger.info('[Network Error]:Failed during fetching job status', { marketoResponse, type }); - throw new NetworkError( - `Unable to fetch job status: due to error ${JSON.stringify(marketoResponse)}`, - hydrateStatusForServer(marketoReposnseStatus, 'During fetching job status'), - ); - } - - if (marketoResponse?.success === false) { - logger.info('[Application Error]Failed during fetching job status', { marketoResponse, type }); - throw new RetryableError( - `Failure during fetching job status due to error : ${marketoResponse}`, - 500, - resp, - ); - } - - /* - successful response : - { - response: 'city, email,Import Failure ReasonChennai,s…a,Value for lookup field 'email' not found', - status: 200 - } - - */ - - return marketoResponse; -}; - -/** - * Handles the response received after a file upload request. - * Checks for errors in the response and throws appropriate error objects based on the error codes. - * If the response indicates a successful upload, extracts the importId and returns it along with other job details. - * - * @param {object} resp - The response object received after a file upload request. - * @param {array} successfulJobs - An array to store details of successful jobs. - * @param {array} unsuccessfulJobs - An array to store details of unsuccessful jobs. - * @param {number} requestTime - The time taken for the request in milliseconds. - * @returns {object} - An object containing the importId, successfulJobs, and unsuccessfulJobs. - */ -const handleFileUploadResponse = (resp, successfulJobs, unsuccessfulJobs, requestTime) => { - /* - For unsuccessful response - { - "requestId": "e42b#14272d07d78", - "success": false, - "errors": [ - { - "code": "1003", - "message": "Empty File" - } - ] - } - */ - if (resp.response?.errors) { - if (resp.response?.errors[0]?.code === '1003') { - stats.increment(UPLOAD_FILE, { - status: 500, - state: 'Retryable', - }); - throw new RetryableError( - `[${FILE_UPLOAD_ERR_MSG}]:Error Message ${resp.response.errors[0]?.message}`, - 500, - ); - } else { - handleCommonErrorResponse(resp, FILE_UPLOAD_ERR_MSG, UPLOAD_FILE); - } - } - - /** - * SuccessFul Upload Response : - { - "requestId": "d01f#15d672f8560", - "result": [ - { - "batchId": 3404, - "importId": "3404", - "status": "Queued" - } - ], - "success": true - } - */ - if ( - resp.response?.success && - resp.response?.result?.length > 0 && - resp.response?.result[0]?.importId - ) { - const { importId } = resp.response.result[0]; - stats.histogram('marketo_bulk_upload_upload_file_time', requestTime); - - stats.increment(UPLOAD_FILE, { - status: 200, - state: 'Success', - }); - return { importId, successfulJobs, unsuccessfulJobs }; - } - // if neither successful, nor the error message is appropriate sending importId as default null - return { importId: null, successfulJobs, unsuccessfulJobs }; -}; - -/** - * Retrieves the field schema mapping for a given access token and munchkin ID from the Marketo API. - * - * @param {string} accessToken - The access token used to authenticate the API request. - * @param {string} munchkinId - The munchkin ID of the Marketo instance. - * @returns {object} - The field schema mapping retrieved from the Marketo API. - */ -const getFieldSchemaMap = async (accessToken, munchkinId) => { - let fieldArr = []; - const fieldMap = {}; // map to store field name and data type - // ref: https://developers.marketo.com/rest-api/endpoint-reference/endpoint-index/#:~:text=Describe%20Lead2,leads/describe2.json - const { processedResponse: fieldSchemaMapping } = await handleHttpRequest( - 'get', - `https://${munchkinId}.mktorest.com/rest/v1/leads/describe2.json`, - { - params: { - access_token: accessToken, - }, - }, - { - destType: 'marketo_bulk_upload', - feature: 'transformation', - endpointPath: '/leads/describe2.json', - requestMethod: 'GET', - module: 'router', - }, - ); - if (fieldSchemaMapping.response.errors) { - handleCommonErrorResponse( - fieldSchemaMapping, - 'Error while fetching Marketo Field Schema', - 'FieldSchemaMapping', - ); - } - if ( - fieldSchemaMapping.response?.success && - fieldSchemaMapping.response?.result.length > 0 && - fieldSchemaMapping.response?.result[0] - ) { - fieldArr = - fieldSchemaMapping.response.result && Array.isArray(fieldSchemaMapping.response.result) - ? fieldSchemaMapping.response.result[0]?.fields - : []; - - fieldArr.forEach((field) => { - fieldMap[field?.name] = field?.dataType; - }); - } else { - throw new RetryableError( - `Failed to fetch Marketo Field Schema due to error ${JSON.stringify(fieldSchemaMapping)}`, - 500, - fieldSchemaMapping, - ); - } - return fieldMap; -}; - -/** - * Compares the data types of the fields in an event message with the expected data types defined in the field schema mapping. - * Identifies any mismatched fields and returns them as a map of job IDs and the corresponding invalid fields. - * - * @param {object} event - An object containing an `input` array of events. Each event has a `message` object with field-value pairs and a `metadata` object with a `job_id` property. - * @param {object} fieldSchemaMapping - An object containing the field schema mapping, which includes the expected data types for each field. - * @returns {object} - An object containing the job IDs as keys and the corresponding invalid fields as values. - */ -const checkEventStatusViaSchemaMatching = (event, fieldMap) => { - const mismatchedFields = {}; - const events = event.input; - events.forEach((ev) => { - const { message, metadata } = ev; - // eslint-disable-next-line @typescript-eslint/naming-convention - const { job_id } = metadata; - - Object.entries(message).forEach(([paramName, paramValue]) => { - const expectedDataType = SCHEMA_DATA_TYPE_MAP[fieldMap[paramName]]; - const actualDataType = typeof paramValue; - - if ( - isDefinedAndNotNull(paramValue) && - !mismatchedFields[job_id] && - actualDataType !== expectedDataType - ) { - mismatchedFields[job_id] = `invalid ${paramName}`; - } - }); - }); - return mismatchedFields; -}; - -module.exports = { - checkEventStatusViaSchemaMatching, - handlePollResponse, - handleFetchJobStatusResponse, - handleFileUploadResponse, - handleCommonErrorResponse, - hydrateStatusForServer, - getAccessToken, - getMarketoFilePath, - getFieldSchemaMap, -}; diff --git a/test/__tests__/data/marketo_bulk_upload_fileUpload_input.json b/test/__tests__/data/marketo_bulk_upload_fileUpload_input.json deleted file mode 100644 index 737cd36ec3..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_fileUpload_input.json +++ /dev/null @@ -1,272 +0,0 @@ -[ - { - "request": { - "body": { - "config": { - "munchkinId": "munchkinId", - "clientId": "b", - "clientSecret": "clientSecret", - "columnFieldsMapping": [ - { - "to": "email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "munchkinId", - "clientId": "b", - "clientSecret": "clientSecret", - "columnFieldsMapping": [ - { - "to": "email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "munchkinId", - "clientId": "wrongClientId", - "clientSecret": "clientSecret", - "columnFieldsMapping": [ - { - "to": "email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "a", - "clientId": "b", - "clientSecret": "forThrottle", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "munchkinId", - "clientId": "b", - "clientSecret": "clientSecret", - "columnFieldsMapping": [ - { - "to": "email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "testMunchkin1", - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "to": "email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "testMunchkin2", - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "testMunchkin3", - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - }, - { - "request": { - "body": { - "config": { - "munchkinId": "testMunchkin4", - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "input": [ - { - "message": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "metadata": { - "job_id": 17 - } - } - ], - "destType": "MARKETO_BULK_UPLOAD" - } - } - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_fileUpload_output.json b/test/__tests__/data/marketo_bulk_upload_fileUpload_output.json deleted file mode 100644 index 0ea94284ae..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_fileUpload_output.json +++ /dev/null @@ -1,63 +0,0 @@ -[ - { - "statusCode": 200, - "importId": "2977", - "metadata": { - "successfulJobs": ["17"], - "unsuccessfulJobs": [], - "csvHeader": "email" - } - }, - { - "statusCode": 200, - "importId": "2977", - "metadata": { - "successfulJobs": ["17"], - "unsuccessfulJobs": [], - "csvHeader": "email" - } - }, - { - "statusCode": 200, - "importId": "2977", - "metadata": { - "successfulJobs": ["17"], - "unsuccessfulJobs": [], - "csvHeader": "email" - } - }, - { - "statusCode": 400, - "error": "The field Email is not present in Marketo Field Schema. Aborting", - "metadata": null - }, - { - "statusCode": 200, - "importId": "2977", - "metadata": { - "successfulJobs": ["17"], - "unsuccessfulJobs": [], - "csvHeader": "email" - } - }, - { - "statusCode": 400, - "error": "[Could not upload file]Error message: undefined", - "metadata": null - }, - { - "statusCode": 400, - "error": "[Could not upload file]Error message: There are 10 imports currently being processed. Please try again later", - "metadata": null - }, - { - "statusCode": 400, - "error": "[Could not upload file]Error message: Empty file", - "metadata": null - }, - { - "statusCode": 400, - "error": "[Could not upload file]Error message: Any other error", - "metadata": null - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_input.json b/test/__tests__/data/marketo_bulk_upload_input.json deleted file mode 100644 index ce48c8e7fe..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_input.json +++ /dev/null @@ -1,270 +0,0 @@ -[ - { - "message": { - "type": "identify", - "traits": { - "name": "Carlo Lombard", - "plan": "Quarterly Team+ Plan for Enuffsaid Media", - "email": "carlo@enuffsaid.media" - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "name" - }, - { - "to": "email__c", - "from": "email" - }, - { - "to": "plan__c", - "from": "plan" - } - ] - } - } - }, - { - "message": { - "traits": { - "name": "Carlo Lombard", - "plan": "Quarterly Team+ Plan for Enuffsaid Media", - "email": "carlo@enuffsaid.media" - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "name" - }, - { - "to": "email__c", - "from": "email" - }, - { - "to": "plan__c", - "from": "plan" - } - ] - } - } - }, - { - "message": { - "type": "track", - "traits": { - "name": "Carlo Lombard", - "plan": "Quarterly Team+ Plan for Enuffsaid Media", - "email": "carlo@enuffsaid.media" - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "name" - }, - { - "to": "email__c", - "from": "email" - }, - { - "to": "plan__c", - "from": "plan" - } - ] - } - } - }, - { - "message": { - "type": "identify", - "traits": { - "name": "Carlo Lombard", - "plan": "Quarterly Team+ Plan for Enuffsaid Media", - "email": "carlo@enuffsaid.media" - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "1" - }, - { - "to": "email__c", - "from": "email1" - }, - { - "to": "plan__c", - "from": "plan1" - } - ] - } - } - }, - { - "message": { - "type": "identify", - "traits": { - "name": "Carlo Lombard", - "plan": "Quarterly Team+ Plan for Enuffsaid Media", - "email": "carlo@enuffsaid.media" - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "name" - }, - { - "to": "email__c", - "from": "email1" - }, - { - "to": "plan__c", - "from": "plan1" - } - ] - } - } - }, - { - "message": { - "type": "identify", - "traits": { - "name": "Carlo Lombard", - "plan": 1 - }, - "userId": 476335, - "context": { - "ip": "14.0.2.238", - "page": { - "url": "enuffsaid.proposify.com", - "path": "/settings", - "method": "POST", - "referrer": "https://enuffsaid.proposify.com/login" - }, - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36" - }, - "rudderId": "786dfec9-jfh", - "messageId": "5d9bc6e2-ekjh" - }, - "destination": { - "ID": "1mMy5cqbtfuaKZv1IhVQKnBdVwe", - "Config": { - "munchkinId": "XXXX", - "clientId": "YYYY", - "clientSecret": "ZZZZ", - "columnFieldsMapping": [ - { - "to": "name__c", - "from": "name" - }, - { - "to": "email__c", - "from": "email" - }, - { - "to": "plan__c", - "from": "plan" - } - ] - } - } - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_jobStatus_input.json b/test/__tests__/data/marketo_bulk_upload_jobStatus_input.json deleted file mode 100644 index b36363e0db..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_jobStatus_input.json +++ /dev/null @@ -1,102 +0,0 @@ -[ - { - "request": { - "body": { - "destType": "MARKETO_BULK_UPLOAD", - "importId": 12345, - "input": [ - { - "message": { - "firstName": "aa", - "email": "bb" - }, - "metadata": { - "job_id": 2 - } - }, - { - "message": { - "firstName": "aa", - "email": "bb", - "phone": "99" - }, - "metadata": { - "job_id": 4 - } - }, - { - "message": { - "firstName": "aa", - "email": "bb" - }, - "metadata": { - "job_id": 3 - } - } - ], - "config": { - "clientId": "b", - "clientSecret": "c", - "munchkinId": "a", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "metadata": {} - } - } - }, - { - "request": { - "body": { - "destType": "MARKETO_BULK_UPLOAD", - "importId": 12345, - "input": [ - { - "message": { - "firstName": "aa", - "email": "bb" - }, - "metadata": { - "job_id": 2 - } - }, - { - "message": { - "firstName": "aa", - "email": "bb", - "phone": "99" - }, - "metadata": { - "job_id": 4 - } - }, - { - "message": { - "firstName": "aa", - "email": "bb" - }, - "metadata": { - "job_id": 3 - } - } - ], - "config": { - "clientId": "b", - "clientSecret": "c", - "munchkinId": "testMunchkin3", - "columnFieldsMapping": [ - { - "to": "Email", - "from": "email" - } - ] - }, - "metadata": {} - } - } - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_jobStatus_output.json b/test/__tests__/data/marketo_bulk_upload_jobStatus_output.json deleted file mode 100644 index 60628f6b3f..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_jobStatus_output.json +++ /dev/null @@ -1,28 +0,0 @@ -[ - { - "type": "warn", - "data": [ - { - "statusCode": 400, - "error": "No csvHeader in metadata" - }, - { - "statusCode": 400, - "error": "Unable to fetch job status: due to error \"\"" - } - ] - }, - { - "type": "fail", - "data": [ - { - "statusCode": 400, - "error": "No csvHeader in metadata" - }, - { - "statusCode": 400, - "error": "Unable to fetch job status: due to error \"\"" - } - ] - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_output.json b/test/__tests__/data/marketo_bulk_upload_output.json deleted file mode 100644 index 9911a7e831..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_output.json +++ /dev/null @@ -1,89 +0,0 @@ -[ - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "/fileUpload", - "headers": {}, - "params": {}, - "body": { - "JSON": { - "name__c": "Carlo Lombard", - "email__c": "carlo@enuffsaid.media", - "plan__c": "Quarterly Team+ Plan for Enuffsaid Media" - }, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "statusCode": 400, - "error": "Event type is required", - "statTags": { - "destination": "marketo_bulk_upload", - "stage": "transform", - "scope": "exception" - } - }, - { - "statusCode": 400, - "error": "Event type track is not supported", - "statTags": { - "destination": "marketo_bulk_upload", - "stage": "transform", - "scope": "exception" - } - }, - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "/fileUpload", - "headers": {}, - "params": {}, - "body": { - "JSON": {}, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "/fileUpload", - "headers": {}, - "params": {}, - "body": { - "JSON": { - "name__c": "Carlo Lombard" - }, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - }, - { - "version": "1", - "type": "REST", - "method": "POST", - "endpoint": "/fileUpload", - "headers": {}, - "params": {}, - "body": { - "JSON": { - "name__c": "Carlo Lombard", - "plan__c": 1 - }, - "JSON_ARRAY": {}, - "XML": {}, - "FORM": {} - }, - "files": {} - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_poll_input.json b/test/__tests__/data/marketo_bulk_upload_poll_input.json deleted file mode 100644 index f5457bd79c..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_poll_input.json +++ /dev/null @@ -1,59 +0,0 @@ -[ - { - "request": { - "body": { - "destType": "MARKETO_BULK_UPLOAD", - "importId": 1234, - "config": { - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "from": "email", - "to": "Email" - } - ], - "munchkinId": "a" - } - } - } - }, - { - "request": { - "body": { - "destType": "MARKETO_BULK_UPLOAD", - "importId": 1234, - "config": { - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "from": "email", - "to": "Email" - } - ], - "munchkinId": "testMunchkin4" - } - } - } - }, - { - "request": { - "body": { - "destType": "MARKETO_BULK_UPLOAD", - "importId": 1234, - "config": { - "clientId": "b", - "clientSecret": "c", - "columnFieldsMapping": [ - { - "from": "email", - "to": "Email" - } - ], - "munchkinId": "testMunchkin500" - } - } - } - } -] diff --git a/test/__tests__/data/marketo_bulk_upload_poll_output.json b/test/__tests__/data/marketo_bulk_upload_poll_output.json deleted file mode 100644 index 92e312072e..0000000000 --- a/test/__tests__/data/marketo_bulk_upload_poll_output.json +++ /dev/null @@ -1,17 +0,0 @@ -[ - { - "Complete": true, - "statusCode": 200, - "hasFailed": false, - "InProgress": false, - "HasWarning": false - }, - { - "statusCode": 400, - "error": "Any 400 error" - }, - { - "statusCode": 400, - "error": "[Could not poll status]Error message: Any 500 error" - } -] diff --git a/test/__tests__/marketo_bulk_upload.test.js b/test/__tests__/marketo_bulk_upload.test.js deleted file mode 100644 index 6cf4d559b9..0000000000 --- a/test/__tests__/marketo_bulk_upload.test.js +++ /dev/null @@ -1,127 +0,0 @@ -const fs = require("fs"); -const path = require("path"); -const vRouter = require("../../src/legacy/router"); - -const version = "v0"; -const integration = "marketo_bulk_upload"; -const transformer = require(`../../src/${version}/destinations/${integration}/transform`); - -jest.mock("axios"); -let reqTransformBody; -let respTransformBody; -let respFileUploadBody; -let reqFileUploadBody; -let reqPollBody; -let respPollBody; -let reqJobStatusBody; -let respJobStatusBody; - -try { - reqTransformBody = JSON.parse( - fs.readFileSync(path.resolve(__dirname, `./data/${integration}_input.json`)) - ); - respTransformBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_output.json`) - ) - ); - reqFileUploadBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_fileUpload_input.json`) - ) - ); - respFileUploadBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_fileUpload_output.json`) - ) - ); - reqPollBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_poll_input.json`) - ) - ); - respPollBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_poll_output.json`) - ) - ); - reqJobStatusBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_jobStatus_input.json`) - ) - ); - respJobStatusBody = JSON.parse( - fs.readFileSync( - path.resolve(__dirname, `./data/${integration}_jobStatus_output.json`) - ) - ); -} catch (error) { - throw new Error("Could not read files." + error); -} - -describe(`${integration} Tests`, () => { - describe("Transformer.js", () => { - reqTransformBody.forEach(async (input, index) => { - it(`Payload - ${index}`, async () => { - try { - const output = await transformer.process(input); - expect(output).toEqual(respTransformBody[index]); - } catch (error) { - expect(error.message).toEqual(respTransformBody[index].error); - } - }); - }); - }); - - describe("fileUpload.js", () => { - reqFileUploadBody.forEach(async (input, index) => { - it(`Payload - ${index}`, async () => { - try { - const output = await vRouter.fileUpload(input); - expect(output).toEqual(respFileUploadBody[index]); - } catch (error) { - expect(error.message).toEqual(respFileUploadBody[index].error); - } - }); - }); - }); - - describe("poll.js", () => { - reqPollBody.forEach(async (input, index) => { - it(`Payload - ${index}`, async () => { - try { - const output = await vRouter.pollStatus(input); - expect(output).toEqual(respPollBody[index]); - } catch (error) { - expect(error.message).toEqual(respPollBody[index].error); - } - }); - }); - }); - - describe("fetchJobStatus.js for warn", () => { - reqJobStatusBody.forEach(async (input, index) => { - it(`Payload - ${index}`, async () => { - try { - const output = await vRouter.getJobStatus(input, "warn"); - expect(output).toEqual(respJobStatusBody[0].data[index]); - } catch (error) { - expect(error.message).toEqual(respJobStatusBody[0].data[index].error); - } - }); - }); - }); - - describe("fetchJobStatus.js for fail", () => { - reqJobStatusBody.forEach(async (input, index) => { - it(`Payload - ${index}`, async () => { - try { - const output = await vRouter.getJobStatus(input, "fail"); - expect(output).toEqual(respJobStatusBody[1].data[index]); - } catch (error) { - expect(error.message).toEqual(respJobStatusBody[1].data[index].error); - } - }); - }); - }); -}); From 3d7db7366e30df31c37cc473e344da82b49ed885 Mon Sep 17 00:00:00 2001 From: Manish Kumar <144022547+manish339k@users.noreply.github.com> Date: Fri, 15 Nov 2024 16:59:49 +0530 Subject: [PATCH 16/32] feat: onboarding intercom v2 retl support (#3843) * feat: onboarding intercom v2 retl support * fix: fixing export error * fix: searching contact for insert record * fix: added more tests * fix: addressing comment * fix: minor change --- src/v0/destinations/intercom_v2/config.js | 7 + src/v0/destinations/intercom_v2/transform.js | 51 ++++- src/v0/destinations/intercom_v2/utils.js | 25 ++- .../destinations/intercom_v2/network.ts | 102 +++++++++ .../destinations/intercom_v2/router/data.ts | 199 ++++++++++++++++++ .../destinations/intercom_v2/router/rETL.ts | 182 ++++++++++++++++ 6 files changed, 558 insertions(+), 8 deletions(-) create mode 100644 test/integrations/destinations/intercom_v2/router/rETL.ts diff --git a/src/v0/destinations/intercom_v2/config.js b/src/v0/destinations/intercom_v2/config.js index c7cb43b093..5ff5566d2d 100644 --- a/src/v0/destinations/intercom_v2/config.js +++ b/src/v0/destinations/intercom_v2/config.js @@ -6,6 +6,12 @@ const ApiVersions = { v2: '2.10', }; +const RecordAction = { + INSERT: 'insert', + UPDATE: 'update', + DELETE: 'delete', +}; + const ConfigCategory = { IDENTIFY: { name: 'IntercomIdentifyConfig', @@ -25,4 +31,5 @@ module.exports = { ConfigCategory, MappingConfig, ApiVersions, + RecordAction, }; diff --git a/src/v0/destinations/intercom_v2/transform.js b/src/v0/destinations/intercom_v2/transform.js index 8d97e20bde..3f9457410f 100644 --- a/src/v0/destinations/intercom_v2/transform.js +++ b/src/v0/destinations/intercom_v2/transform.js @@ -1,4 +1,4 @@ -const { InstrumentationError } = require('@rudderstack/integrations-lib'); +const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); const { handleRtTfSingleEventError, getSuccessRespEvents, @@ -17,13 +17,14 @@ const { addOrUpdateTagsToCompany, getStatusCode, getBaseEndpoint, + getRecordAction, } = require('./utils'); const { getName, filterCustomAttributes, addMetadataToPayload, } = require('../../../cdk/v2/destinations/intercom/utils'); -const { MappingConfig, ConfigCategory } = require('./config'); +const { MappingConfig, ConfigCategory, RecordAction } = require('./config'); const transformIdentifyPayload = (event) => { const { message, destination } = event; @@ -38,7 +39,7 @@ const transformIdentifyPayload = (event) => { } payload.name = getName(message); payload.custom_attributes = message.traits || message.context.traits || {}; - payload.custom_attributes = filterCustomAttributes(payload, 'user', destination); + payload.custom_attributes = filterCustomAttributes(payload, 'user', destination, message); return payload; }; @@ -66,7 +67,7 @@ const transformGroupPayload = (event) => { const category = ConfigCategory.GROUP; const payload = constructPayload(message, MappingConfig[category.name]); payload.custom_attributes = message.traits || message.context.traits || {}; - payload.custom_attributes = filterCustomAttributes(payload, 'company', destination); + payload.custom_attributes = filterCustomAttributes(payload, 'company', destination, message); return payload; }; @@ -131,6 +132,45 @@ const constructGroupResponse = async (event) => { return getResponse(method, endpoint, headers, finalPayload); }; +const constructRecordResponse = async (event) => { + const { message, destination, metadata } = event; + const { identifiers, fields } = message; + + let method = 'POST'; + let endpoint = `${getBaseEndpoint(destination)}/contacts`; + let payload = {}; + + const action = getRecordAction(message); + const contactId = await searchContact(event); + + if ((action === RecordAction.UPDATE || action === RecordAction.DELETE) && !contactId) { + throw new ConfigurationError('Contact is not present. Aborting.'); + } + + switch (action) { + case RecordAction.INSERT: + payload = { ...identifiers, ...fields }; + if (contactId) { + endpoint += `/${contactId}`; + payload = { ...fields }; + method = 'PUT'; + } + break; + case RecordAction.UPDATE: + endpoint += `/${contactId}`; + payload = { ...fields }; + method = 'PUT'; + break; + case RecordAction.DELETE: + endpoint += `/${contactId}`; + method = 'DELETE'; + break; + default: + throw new InstrumentationError(`action ${action} is not supported.`); + } + return getResponse(method, endpoint, getHeaders(metadata), payload); +}; + const processEvent = async (event) => { const { message } = event; const messageType = getEventType(message); @@ -145,6 +185,9 @@ const processEvent = async (event) => { case EventType.GROUP: response = await constructGroupResponse(event); break; + case EventType.RECORD: + response = constructRecordResponse(event); + break; default: throw new InstrumentationError(`message type ${messageType} is not supported.`); } diff --git a/src/v0/destinations/intercom_v2/utils.js b/src/v0/destinations/intercom_v2/utils.js index 69ea1385d9..df44b92e24 100644 --- a/src/v0/destinations/intercom_v2/utils.js +++ b/src/v0/destinations/intercom_v2/utils.js @@ -28,6 +28,8 @@ const { getAccessToken } = require('../../util'); const { ApiVersions, destType } = require('./config'); const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); +const getRecordAction = (message) => message?.action?.toLowerCase(); + /** * method to handle error during api call * ref docs: https://developers.intercom.com/docs/references/rest-api/errors/http-responses/ @@ -99,11 +101,25 @@ const getResponse = (method, endpoint, headers, payload) => { const searchContact = async (event) => { const { message, destination, metadata } = event; - const lookupField = getLookUpField(message); - let lookupFieldValue = getFieldValueFromMessage(message, lookupField); - if (!lookupFieldValue) { - lookupFieldValue = message?.context?.traits?.[lookupField]; + + const extractLookupFieldAndValue = () => { + const messageType = getEventType(message); + if (messageType === EventType.RECORD) { + const { identifiers } = message; + return Object.entries(identifiers || {})[0] || [null, null]; + } + const lookupField = getLookUpField(message); + const lookupFieldValue = + getFieldValueFromMessage(message, lookupField) || message?.context?.traits?.[lookupField]; + return [lookupField, lookupFieldValue]; + }; + + const [lookupField, lookupFieldValue] = extractLookupFieldAndValue(); + + if (!lookupField || !lookupFieldValue) { + throw new InstrumentationError('Missing lookup field or lookup field value for searchContact'); } + const data = JSON.stringify({ query: { operator: 'AND', @@ -329,4 +345,5 @@ module.exports = { attachContactToCompany, addOrUpdateTagsToCompany, getBaseEndpoint, + getRecordAction, }; diff --git a/test/integrations/destinations/intercom_v2/network.ts b/test/integrations/destinations/intercom_v2/network.ts index 26ff3c38ee..e4cae04d07 100644 --- a/test/integrations/destinations/intercom_v2/network.ts +++ b/test/integrations/destinations/intercom_v2/network.ts @@ -746,6 +746,108 @@ const deliveryCallsData = [ }, }, }, + { + httpReq: { + method: 'post', + url: 'https://api.intercom.io/contacts/search', + data: { + query: { + operator: 'AND', + value: [{ field: 'email', operator: '=', value: 'test-rETL-available@gmail.com' }], + }, + }, + headers, + }, + httpRes: { + status: 200, + statusText: 'ok', + data: { + type: 'list', + total_count: 0, + pages: { + type: 'pages', + page: 1, + per_page: 50, + total_pages: 0, + }, + data: [ + { + type: 'contact', + id: 'retl-available-contact-id', + workspace_id: 'rudderWorkspace', + external_id: 'detach-company-user-id', + role: 'user', + email: 'test-rETL-available@gmail.com', + }, + ], + }, + }, + }, + { + httpReq: { + method: 'post', + url: 'https://api.intercom.io/contacts/search', + data: { + query: { + operator: 'AND', + value: [{ field: 'email', operator: '=', value: 'test-rETL-unavailable@gmail.com' }], + }, + }, + headers, + }, + httpRes: { + status: 200, + statusText: 'ok', + data: { + type: 'list', + total_count: 0, + pages: { + type: 'pages', + page: 1, + per_page: 50, + total_pages: 0, + }, + data: [], + }, + }, + }, + { + httpReq: { + method: 'post', + url: 'https://api.au.intercom.io/contacts/search', + data: { + query: { + operator: 'AND', + value: [{ field: 'external_id', operator: '=', value: 'known-user-id-1' }], + }, + }, + headers, + }, + httpRes: { + status: 200, + statusText: 'ok', + data: { + type: 'list', + total_count: 0, + pages: { + type: 'pages', + page: 1, + per_page: 50, + total_pages: 0, + }, + data: [ + { + type: 'contact', + id: 'contact-id-by-intercom-known-user-id-1', + workspace_id: 'rudderWorkspace', + external_id: 'user-id-1', + role: 'user', + email: 'test@rudderlabs.com', + }, + ], + }, + }, + }, ]; export const networkCallsData = [...deliveryCallsData]; diff --git a/test/integrations/destinations/intercom_v2/router/data.ts b/test/integrations/destinations/intercom_v2/router/data.ts index 7656914059..75f5ba6ae7 100644 --- a/test/integrations/destinations/intercom_v2/router/data.ts +++ b/test/integrations/destinations/intercom_v2/router/data.ts @@ -17,6 +17,7 @@ import { userTraits, } from '../common'; import { RouterTestData } from '../../../testTypes'; +import { rETLRecordV2RouterRequest } from './rETL'; const routerRequest1: RouterTransformationRequest = { input: [ @@ -222,6 +223,26 @@ const routerRequest3: RouterTransformationRequest = { }, metadata: generateMetadata(3), }, + { + destination: destinationApiServerAU, + message: { + userId: 'known-user-id-1', + channel, + context: { + traits: { ...userTraits, external_id: 'known-user-id-1' }, + }, + type: 'identify', + integrations: { + All: true, + Intercom: { + lookup: 'external_id', + }, + }, + originalTimestamp, + timestamp, + }, + metadata: generateMetadata(4), + }, ], destType: 'intercom_v2', }; @@ -735,6 +756,38 @@ export const data: RouterTestData[] = [ metadata: [generateMetadata(3)], statusCode: 400, }, + { + batched: false, + batchedRequest: { + body: { + JSON: { + email: 'test@rudderlabs.com', + external_id: 'known-user-id-1', + name: 'John Snow', + owner_id: 13, + phone: '+91 9999999999', + custom_attributes: { + address: 'california usa', + age: 23, + }, + }, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + endpoint: + 'https://api.au.intercom.io/contacts/contact-id-by-intercom-known-user-id-1', + files: {}, + headers, + method: 'PUT', + params: {}, + type: 'REST', + version: '1', + }, + destination: destinationApiServerAU, + metadata: [generateMetadata(4)], + statusCode: 200, + }, ], }, }, @@ -880,4 +933,150 @@ export const data: RouterTestData[] = [ }, }, }, + { + id: 'INTERCOM-V2-router-test-6', + scenario: 'Framework', + successCriteria: 'Some events should be transformed successfully and some should fail for rETL', + name: 'intercom_v2', + description: 'INTERCOM V2 rETL tests', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: rETLRecordV2RouterRequest, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batched: false, + batchedRequest: { + body: { + JSON: { + email: 'test-rETL-unavailable@gmail.com', + external_id: 'rEtl_external_id', + }, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + endpoint: 'https://api.intercom.io/contacts', + files: {}, + headers, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: destination, + metadata: [generateMetadata(1)], + statusCode: 200, + }, + { + batched: false, + batchedRequest: { + body: { + JSON: { + external_id: 'rEtl_external_id', + }, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + endpoint: 'https://api.intercom.io/contacts/retl-available-contact-id', + files: {}, + headers, + method: 'PUT', + params: {}, + type: 'REST', + version: '1', + }, + destination: destination, + metadata: [generateMetadata(2)], + statusCode: 200, + }, + { + batched: false, + batchedRequest: { + body: { + JSON: {}, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + endpoint: 'https://api.intercom.io/contacts/retl-available-contact-id', + files: {}, + headers, + method: 'DELETE', + params: {}, + type: 'REST', + version: '1', + }, + destination: destination, + metadata: [generateMetadata(3)], + statusCode: 200, + }, + { + batched: false, + error: 'Contact is not present. Aborting.', + statTags: { + ...RouterInstrumentationErrorStatTags, + errorType: 'configuration', + }, + destination, + metadata: [generateMetadata(4)], + statusCode: 400, + }, + { + batched: false, + batchedRequest: { + body: { + JSON: { + external_id: 'rEtl_external_id', + }, + XML: {}, + FORM: {}, + JSON_ARRAY: {}, + }, + endpoint: 'https://api.intercom.io/contacts/retl-available-contact-id', + files: {}, + headers, + method: 'PUT', + params: {}, + type: 'REST', + version: '1', + }, + destination: destination, + metadata: [generateMetadata(5)], + statusCode: 200, + }, + { + batched: false, + error: 'action dummyaction is not supported.', + statTags: { + ...RouterInstrumentationErrorStatTags, + }, + destination, + metadata: [generateMetadata(6)], + statusCode: 400, + }, + { + batched: false, + error: 'Missing lookup field or lookup field value for searchContact', + statTags: { + ...RouterInstrumentationErrorStatTags, + }, + destination, + metadata: [generateMetadata(7)], + statusCode: 400, + }, + ], + }, + }, + }, + }, ]; diff --git a/test/integrations/destinations/intercom_v2/router/rETL.ts b/test/integrations/destinations/intercom_v2/router/rETL.ts new file mode 100644 index 0000000000..0a36b8cfa6 --- /dev/null +++ b/test/integrations/destinations/intercom_v2/router/rETL.ts @@ -0,0 +1,182 @@ +import { RouterTransformationRequest } from '../../../../../src/types'; +import { destination } from '../common'; +import { generateMetadata } from '../../../testUtils'; + +export const rETLRecordV2RouterRequest: RouterTransformationRequest = { + input: [ + { + destination, + message: { + type: 'record', + action: 'insert', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '1', + rudderId: '1', + identifiers: { + email: 'test-rETL-unavailable@gmail.com', + }, + }, + metadata: generateMetadata(1), + }, + { + destination, + message: { + type: 'record', + action: 'update', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '2', + rudderId: '2', + identifiers: { + email: 'test-rETL-available@gmail.com', + }, + }, + metadata: generateMetadata(2), + }, + { + destination, + message: { + type: 'record', + action: 'delete', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '3', + rudderId: '3', + identifiers: { + email: 'test-rETL-available@gmail.com', + }, + }, + metadata: generateMetadata(3), + }, + { + destination, + message: { + type: 'record', + action: 'update', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '1', + rudderId: '1', + identifiers: { + email: 'test-rETL-unavailable@gmail.com', + }, + }, + metadata: generateMetadata(4), + }, + { + destination, + message: { + type: 'record', + action: 'insert', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '1', + rudderId: '1', + identifiers: { + email: 'test-rETL-available@gmail.com', + }, + }, + metadata: generateMetadata(5), + }, + { + destination, + message: { + type: 'record', + action: 'dummyAction', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '1', + rudderId: '1', + identifiers: { + email: 'test-rETL-available@gmail.com', + }, + }, + metadata: generateMetadata(6), + }, + { + destination, + message: { + type: 'record', + action: 'insert', + fields: { + external_id: 'rEtl_external_id', + }, + channel: 'sources', + context: { + sources: { + job_id: 'job-id', + version: 'local', + job_run_id: 'job_run_id', + task_run_id: 'job_run_id', + }, + }, + recordId: '1', + rudderId: '1', + identifiers: {}, + }, + metadata: generateMetadata(7), + }, + ], + destType: 'intercom_v2', +}; From becb4fa54e9093ed69779f54c36864cb9d28d321 Mon Sep 17 00:00:00 2001 From: Aanshi Lahoti <110057617+aanshi07@users.noreply.github.com> Date: Fri, 15 Nov 2024 22:44:16 +0530 Subject: [PATCH 17/32] feat: iterable EUDC deleteUsers (#3881) --- src/v0/destinations/iterable/deleteUsers.js | 8 +++-- .../destinations/iterable/deleteUsers/data.ts | 36 +++++++++++++++++++ .../destinations/iterable/network.ts | 17 +++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/v0/destinations/iterable/deleteUsers.js b/src/v0/destinations/iterable/deleteUsers.js index 015a9de9a0..79c4c0affd 100644 --- a/src/v0/destinations/iterable/deleteUsers.js +++ b/src/v0/destinations/iterable/deleteUsers.js @@ -6,13 +6,14 @@ const { getDynamicErrorType } = require('../../../adapters/utils/networkUtils'); const { executeCommonValidations } = require('../../util/regulation-api'); const tags = require('../../util/tags'); const { JSON_MIME_TYPE } = require('../../util/constant'); +const { constructEndpoint } = require('./config'); -// Ref-> https://developers.intercom.com/intercom-api-reference/v1.3/reference/permanently-delete-a-user +// Ref-> https://support.iterable.com/hc/en-us/articles/360032290032-Deleting-Users const userDeletionHandler = async (userAttributes, config) => { if (!config) { throw new ConfigurationError('Config for deletion not present'); } - const { apiKey } = config; + const { apiKey, dataCenter } = config; if (!apiKey) { throw new ConfigurationError('api key for deletion not present'); } @@ -26,7 +27,8 @@ const userDeletionHandler = async (userAttributes, config) => { const failedUserDeletions = []; await Promise.all( validUserIds.map(async (uId) => { - const url = `https://api.iterable.com/api/users/byUserId/${uId}`; + const endpointCategory = { endpoint: `users/byUserId/${uId}` }; + const url = constructEndpoint(dataCenter, endpointCategory); const requestOptions = { headers: { 'Content-Type': JSON_MIME_TYPE, diff --git a/test/integrations/destinations/iterable/deleteUsers/data.ts b/test/integrations/destinations/iterable/deleteUsers/data.ts index 79d801f4ee..9e7eab1ee1 100644 --- a/test/integrations/destinations/iterable/deleteUsers/data.ts +++ b/test/integrations/destinations/iterable/deleteUsers/data.ts @@ -183,4 +183,40 @@ export const data = [ }, }, }, + { + name: destType, + description: 'Test 5: should pass when dataCenter is selected as EUDC', + feature: 'userDeletion', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destType: destType.toUpperCase(), + userAttributes: [ + { + userId: 'rudder7', + }, + ], + config: { + apiKey: 'dummyApiKey', + dataCenter: 'EUDC', + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + statusCode: 200, + status: 'successful', + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/destinations/iterable/network.ts b/test/integrations/destinations/iterable/network.ts index 39544b2647..1cf26dfd4f 100644 --- a/test/integrations/destinations/iterable/network.ts +++ b/test/integrations/destinations/iterable/network.ts @@ -105,5 +105,22 @@ const deleteNwData = [ status: 200, }, }, + { + httpReq: { + method: 'delete', + url: 'https://api.eu.iterable.com/api/users/byUserId/rudder7', + headers: { + api_key: 'dummyApiKey', + }, + }, + httpRes: { + data: { + msg: 'All users associated with rudder7 were successfully deleted', + code: 'Success', + params: null, + }, + status: 200, + }, + }, ]; export const networkCallsData = [...deleteNwData]; From c43f0bd057b111f42da3429771de269014b54ccc Mon Sep 17 00:00:00 2001 From: Yashasvi Bajpai <33063622+yashasvibajpai@users.noreply.github.com> Date: Sat, 16 Nov 2024 03:43:59 +0530 Subject: [PATCH 18/32] chore(shopify): seperate pixel server side events logic from legacy tracker implementation (#3849) * chore: initial commit, adding transformation and utils * chore: handle new qparam * chore: fix tests, qparams and mocks * chore: add testsx1 * chore: remove redundant id assigns and code * chore: fixing mocking, temp commit * chore: refactor and restructure test and transformations * chore: cleanup redundant testdata * chore: updates to facilitate utils mocking * chore: address events, add unit tests for new util functions * chore: update fxn name * chore: remove redis related check function in pixel transformation * chore: remove redundant switch case --------- Co-authored-by: Sai Sankeerth --- src/v0/sources/shopify/util.js | 23 +- src/v1/sources/shopify/transform.js | 20 +- .../serverSideTransform.js | 174 ++ .../serverSideUtils.test.js | 112 ++ .../webhookTransformations/serverSideUtlis.js | 45 + .../pixelTransform.js | 14 +- .../pixelTransform.redisCartToken.test.js | 16 +- .../pixelUtils.js | 6 +- .../pixelUtils.test.js | 5 +- test/integrations/component.test.ts | 1 + test/integrations/sources/shopify/data.ts | 14 +- test/integrations/sources/shopify/mocks.ts | 5 + .../shopify/v1ServerSideEventsTests.ts | 596 ------ .../CheckoutEventsTests.ts | 1687 +++++++++++++++++ .../webhookTestScenarios/GenericTrackTests.ts | 557 ++++++ .../webhookTestScenarios/IdentifyTests.ts | 256 +++ 16 files changed, 2890 insertions(+), 641 deletions(-) create mode 100644 src/v1/sources/shopify/webhookTransformations/serverSideTransform.js create mode 100644 src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js create mode 100644 src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js rename src/v1/sources/shopify/{ => webpixelTransformations}/pixelTransform.js (94%) rename src/v1/sources/shopify/{ => webpixelTransformations}/pixelTransform.redisCartToken.test.js (87%) rename src/v1/sources/shopify/{ => webpixelTransformations}/pixelUtils.js (97%) rename src/v1/sources/shopify/{ => webpixelTransformations}/pixelUtils.test.js (99%) create mode 100644 test/integrations/sources/shopify/mocks.ts delete mode 100644 test/integrations/sources/shopify/v1ServerSideEventsTests.ts create mode 100644 test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts create mode 100644 test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts create mode 100644 test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts diff --git a/src/v0/sources/shopify/util.js b/src/v0/sources/shopify/util.js index 981832363e..b7e79e35a1 100644 --- a/src/v0/sources/shopify/util.js +++ b/src/v0/sources/shopify/util.js @@ -2,15 +2,9 @@ /* eslint-disable @typescript-eslint/naming-convention */ const { v5 } = require('uuid'); const sha256 = require('sha256'); -const { TransformationError } = require('@rudderstack/integrations-lib'); +const { TransformationError, isDefinedAndNotNull } = require('@rudderstack/integrations-lib'); const stats = require('../../../util/stats'); -const { - constructPayload, - extractCustomFields, - flattenJson, - generateUUID, - isDefinedAndNotNull, -} = require('../../util'); +const utils = require('../../util'); const { RedisDB } = require('../../../util/redis/redisConnector'); const { lineItemsMappingJSON, @@ -92,8 +86,8 @@ const getProductsListFromLineItems = (lineItems) => { } const products = []; lineItems.forEach((lineItem) => { - const product = constructPayload(lineItem, lineItemsMappingJSON); - extractCustomFields(lineItem, product, 'root', LINE_ITEM_EXCLUSION_FIELDS); + const product = utils.constructPayload(lineItem, lineItemsMappingJSON); + utils.extractCustomFields(lineItem, product, 'root', LINE_ITEM_EXCLUSION_FIELDS); product.variant = getVariantString(lineItem); products.push(product); }); @@ -103,14 +97,14 @@ const getProductsListFromLineItems = (lineItems) => { const createPropertiesForEcomEvent = (message) => { const { line_items: lineItems } = message; const productsList = getProductsListFromLineItems(lineItems); - const mappedPayload = constructPayload(message, productMappingJSON); - extractCustomFields(message, mappedPayload, 'root', PRODUCT_MAPPING_EXCLUSION_FIELDS); + const mappedPayload = utils.constructPayload(message, productMappingJSON); + utils.extractCustomFields(message, mappedPayload, 'root', PRODUCT_MAPPING_EXCLUSION_FIELDS); mappedPayload.products = productsList; return mappedPayload; }; const extractEmailFromPayload = (event) => { - const flattenedPayload = flattenJson(event); + const flattenedPayload = utils.flattenJson(event); let email; const regex_email = /\bemail\b/i; Object.entries(flattenedPayload).some(([key, value]) => { @@ -182,7 +176,7 @@ const getAnonymousIdAndSessionId = async (message, metricMetadata, redisData = n return { anonymousId, sessionId }; } return { - anonymousId: isDefinedAndNotNull(anonymousId) ? anonymousId : generateUUID(), + anonymousId: isDefinedAndNotNull(anonymousId) ? anonymousId : utils.generateUUID(), sessionId, }; } @@ -281,4 +275,5 @@ module.exports = { checkAndUpdateCartItems, getHashLineItems, getDataFromRedis, + getVariantString, }; diff --git a/src/v1/sources/shopify/transform.js b/src/v1/sources/shopify/transform.js index dee5a14a9d..5ebf4a34fc 100644 --- a/src/v1/sources/shopify/transform.js +++ b/src/v1/sources/shopify/transform.js @@ -1,19 +1,27 @@ /* eslint-disable @typescript-eslint/naming-convention */ -const { processEventFromPixel } = require('./pixelTransform'); +const { processPixelWebEvents } = require('./webpixelTransformations/pixelTransform'); const { process: processWebhookEvents } = require('../../../v0/sources/shopify/transform'); +const { + process: processPixelWebhookEvents, +} = require('./webhookTransformations/serverSideTransform'); const process = async (inputEvent) => { const { event } = inputEvent; - // check on the source Config to identify the event is from the tracker-based (legacy) - // or the pixel-based (latest) implementation. + const { query_parameters } = event; + // check identify the event is from the web pixel based on the pixelEventLabel property. const { pixelEventLabel: pixelClientEventLabel } = event; if (pixelClientEventLabel) { // this is a event fired from the web pixel loaded on the browser // by the user interactions with the store. - const responseV2 = await processEventFromPixel(event); - return responseV2; + const pixelWebEventResponse = await processPixelWebEvents(event); + return pixelWebEventResponse; } - // this is for common logic for server-side events processing for both pixel and tracker apps. + if (query_parameters && query_parameters?.version?.[0] === 'pixel') { + // this is a server-side event from the webhook subscription made by the pixel app. + const pixelWebhookEventResponse = await processPixelWebhookEvents(event); + return pixelWebhookEventResponse; + } + // this is a server-side event from the webhook subscription made by the legacy tracker-based app. const response = await processWebhookEvents(event); return response; }; diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js new file mode 100644 index 0000000000..c31bc74bf1 --- /dev/null +++ b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js @@ -0,0 +1,174 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +const lodash = require('lodash'); +const get = require('get-value'); +// const { RedisError } = require('@rudderstack/integrations-lib'); +const stats = require('../../../../util/stats'); +const { + getShopifyTopic, + // createPropertiesForEcomEvent, + extractEmailFromPayload, + getAnonymousIdAndSessionId, + // getHashLineItems, +} = require('../../../../v0/sources/shopify/util'); +// const logger = require('../../../logger'); +const { removeUndefinedAndNullValues, isDefinedAndNotNull } = require('../../../../v0/util'); +// const { RedisDB } = require('../../../util/redis/redisConnector'); +const Message = require('../../../../v0/sources/message'); +const { EventType } = require('../../../../constants'); +const { + INTEGERATION, + MAPPING_CATEGORIES, + IDENTIFY_TOPICS, + ECOM_TOPICS, + RUDDER_ECOM_MAP, + SUPPORTED_TRACK_EVENTS, + SHOPIFY_TRACK_MAP, + lineItemsMappingJSON, +} = require('../../../../v0/sources/shopify/config'); +const { + createPropertiesForEcomEventFromWebhook, + getProductsFromLineItems, +} = require('./serverSideUtlis'); + +const NO_OPERATION_SUCCESS = { + outputToSource: { + body: Buffer.from('OK').toString('base64'), + contentType: 'text/plain', + }, + statusCode: 200, +}; + +const identifyPayloadBuilder = (event) => { + const message = new Message(INTEGERATION); + message.setEventType(EventType.IDENTIFY); + message.setPropertiesV2(event, MAPPING_CATEGORIES[EventType.IDENTIFY]); + if (event.updated_at) { + // converting shopify updated_at timestamp to rudder timestamp format + message.setTimestamp(new Date(event.updated_at).toISOString()); + } + return message; +}; + +const ecomPayloadBuilder = (event, shopifyTopic) => { + const message = new Message(INTEGERATION); + message.setEventType(EventType.TRACK); + message.setEventName(RUDDER_ECOM_MAP[shopifyTopic]); + + const properties = createPropertiesForEcomEventFromWebhook(event); + message.properties = removeUndefinedAndNullValues(properties); + // Map Customer details if present + const customerDetails = get(event, 'customer'); + if (customerDetails) { + message.setPropertiesV2(customerDetails, MAPPING_CATEGORIES[EventType.IDENTIFY]); + } + if (event.updated_at) { + message.setTimestamp(new Date(event.updated_at).toISOString()); + } + if (event.customer) { + message.setPropertiesV2(event.customer, MAPPING_CATEGORIES[EventType.IDENTIFY]); + } + if (event.shipping_address) { + message.setProperty('traits.shippingAddress', event.shipping_address); + } + if (event.billing_address) { + message.setProperty('traits.billingAddress', event.billing_address); + } + if (!message.userId && event.user_id) { + message.setProperty('userId', event.user_id); + } + return message; +}; + +const trackPayloadBuilder = (event, shopifyTopic) => { + const message = new Message(INTEGERATION); + message.setEventType(EventType.TRACK); + message.setEventName(SHOPIFY_TRACK_MAP[shopifyTopic]); + // eslint-disable-next-line camelcase + const { line_items: lineItems } = event; + const productsList = getProductsFromLineItems(lineItems, lineItemsMappingJSON); + message.setProperty('properties.products', productsList); + return message; +}; + +const processEvent = async (inputEvent, metricMetadata) => { + let message; + const event = lodash.cloneDeep(inputEvent); + const shopifyTopic = getShopifyTopic(event); + delete event.query_parameters; + switch (shopifyTopic) { + case IDENTIFY_TOPICS.CUSTOMERS_CREATE: + case IDENTIFY_TOPICS.CUSTOMERS_UPDATE: + message = identifyPayloadBuilder(event); + break; + case ECOM_TOPICS.ORDERS_CREATE: + case ECOM_TOPICS.ORDERS_UPDATE: + case ECOM_TOPICS.CHECKOUTS_CREATE: + case ECOM_TOPICS.CHECKOUTS_UPDATE: + message = ecomPayloadBuilder(event, shopifyTopic); + break; + default: + if (!SUPPORTED_TRACK_EVENTS.includes(shopifyTopic)) { + stats.increment('invalid_shopify_event', { + writeKey: metricMetadata.writeKey, + source: metricMetadata.source, + shopifyTopic: metricMetadata.shopifyTopic, + }); + return NO_OPERATION_SUCCESS; + } + message = trackPayloadBuilder(event, shopifyTopic); + break; + } + + if (message.userId) { + message.userId = String(message.userId); + } + if (!get(message, 'traits.email')) { + const email = extractEmailFromPayload(event); + if (email) { + message.setProperty('traits.email', email); + } + } + if (message.type !== EventType.IDENTIFY) { + const { anonymousId } = await getAnonymousIdAndSessionId( + message, + { shopifyTopic, ...metricMetadata }, + null, + ); + if (isDefinedAndNotNull(anonymousId)) { + message.setProperty('anonymousId', anonymousId); + } + } + message.setProperty(`integrations.${INTEGERATION}`, true); + message.setProperty('context.library', { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }); + message.setProperty('context.topic', shopifyTopic); + // attaching cart, checkout and order tokens in context object + message.setProperty(`context.cart_token`, event.cart_token); + message.setProperty(`context.checkout_token`, event.checkout_token); + // raw shopify payload passed inside context object under shopifyDetails + message.setProperty('context.shopifyDetails', event); + if (shopifyTopic === 'orders_updated') { + message.setProperty(`context.order_token`, event.token); + } + message = removeUndefinedAndNullValues(message); + return message; +}; +const process = async (event) => { + const metricMetadata = { + writeKey: event.query_parameters?.writeKey?.[0], + source: 'SHOPIFY', + }; + const response = await processEvent(event, metricMetadata); + return response; +}; + +module.exports = { + process, + processEvent, + identifyPayloadBuilder, + ecomPayloadBuilder, + trackPayloadBuilder, +}; diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js new file mode 100644 index 0000000000..a611d1d8dc --- /dev/null +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js @@ -0,0 +1,112 @@ +const { + getProductsFromLineItems, + createPropertiesForEcomEventFromWebhook, +} = require('./serverSideUtlis'); + +const { constructPayload } = require('../../../../v0/util'); + +const { + lineItemsMappingJSON, + productMappingJSON, +} = require('../../../../v0/sources/shopify/config'); +const Message = require('../../../../v0/sources/message'); +jest.mock('../../../../v0/sources/message'); + +const LINEITEMS = [ + { + id: '41327142600817', + grams: 0, + presentment_title: 'The Collection Snowboard: Hydrogen', + product_id: 7234590408818, + quantity: 1, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + variant_id: 41327142600817, + variant_title: '', + variant_price: '600.00', + vendor: 'Hydrogen Vendor', + line_price: '600.00', + price: '600.00', + applied_discounts: [], + properties: {}, + }, + { + id: 14234727743601, + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Nitrogen', + price: '600.00', + product_exists: true, + product_id: 7234590408817, + properties: [], + quantity: 1, + sku: '', + title: 'The Collection Snowboard: Nitrogen', + total_discount: '0.00', + variant_id: 41327142600817, + vendor: 'Hydrogen Vendor', + }, +]; + +describe('serverSideUtils.js', () => { + beforeEach(() => { + Message.mockClear(); + }); + + describe('Test getProductsFromLineItems function', () => { + it('should return empty array if lineItems is empty', () => { + const lineItems = []; + const result = getProductsFromLineItems(lineItems, lineItemsMappingJSON); + expect(result).toEqual([]); + }); + + it('should return array of products', () => { + const mapping = {}; + const result = getProductsFromLineItems(LINEITEMS, lineItemsMappingJSON); + expect(result).toEqual([ + { brand: 'Hydrogen Vendor', price: '600.00', product_id: 7234590408818, quantity: 1 }, + { + brand: 'Hydrogen Vendor', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + title: 'The Collection Snowboard: Nitrogen', + }, + ]); + }); + }); + + describe('Test createPropertiesForEcomEventFromWebhook function', () => { + it('should return empty array if lineItems is empty', () => { + const message = { + line_items: [], + type: 'track', + event: 'checkout created', + }; + const result = createPropertiesForEcomEventFromWebhook(message); + expect(result).toEqual([]); + }); + + it('should return array of products', () => { + const message = { + line_items: LINEITEMS, + type: 'track', + event: 'checkout updated', + }; + const result = createPropertiesForEcomEventFromWebhook(message); + expect(result).toEqual({ + products: [ + { brand: 'Hydrogen Vendor', price: '600.00', product_id: 7234590408818, quantity: 1 }, + { + brand: 'Hydrogen Vendor', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + title: 'The Collection Snowboard: Nitrogen', + }, + ], + }); + }); + }); +}); diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js new file mode 100644 index 0000000000..eed03de71f --- /dev/null +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js @@ -0,0 +1,45 @@ +const { constructPayload } = require('../../../../v0/util'); + +const { + lineItemsMappingJSON, + productMappingJSON, +} = require('../../../../v0/sources/shopify/config'); + +/** + * Returns an array of products from the lineItems array received from the webhook event + * @param {Array} lineItems + * @param {Object} mapping + * @returns {Array} products + */ +const getProductsFromLineItems = (lineItems, mapping) => { + if (!lineItems || lineItems.length === 0) { + return []; + } + const products = []; + lineItems.forEach((lineItem) => { + // const product = constructPayload(lineItem, lineItemsMappingJSON); + const product = constructPayload(lineItem, mapping); + products.push(product); + }); + return products; +}; + +/** + * Creates properties for the ecommerce webhook events received from the pixel based app + * @param {Object} message + * @returns {Object} properties + */ +const createPropertiesForEcomEventFromWebhook = (message) => { + const { line_items: lineItems } = message; + if (!lineItems || lineItems.length === 0) { + return []; + } + const mappedPayload = constructPayload(message, productMappingJSON); + mappedPayload.products = getProductsFromLineItems(lineItems, lineItemsMappingJSON); + return mappedPayload; +}; + +module.exports = { + createPropertiesForEcomEventFromWebhook, + getProductsFromLineItems, +}; diff --git a/src/v1/sources/shopify/pixelTransform.js b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js similarity index 94% rename from src/v1/sources/shopify/pixelTransform.js rename to src/v1/sources/shopify/webpixelTransformations/pixelTransform.js index e308f626b4..b1d1c8b2fa 100644 --- a/src/v1/sources/shopify/pixelTransform.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js @@ -2,10 +2,10 @@ // eslint-disable-next-line @typescript-eslint/naming-convention const _ = require('lodash'); const { isDefinedNotNullNotEmpty } = require('@rudderstack/integrations-lib'); -const stats = require('../../../util/stats'); -const logger = require('../../../logger'); -const { removeUndefinedAndNullValues } = require('../../../v0/util'); -const { RedisDB } = require('../../../util/redis/redisConnector'); +const stats = require('../../../../util/stats'); +const logger = require('../../../../logger'); +const { removeUndefinedAndNullValues } = require('../../../../v0/util'); +const { RedisDB } = require('../../../../util/redis/redisConnector'); const { pageViewedEventBuilder, cartViewedEventBuilder, @@ -20,7 +20,7 @@ const { INTEGERATION, PIXEL_EVENT_TOPICS, pixelEventToCartTokenLocationMapping, -} = require('./config'); +} = require('../config'); const NO_OPERATION_SUCCESS = { outputToSource: { @@ -152,13 +152,13 @@ function processPixelEvent(inputEvent) { return message; } -const processEventFromPixel = async (event) => { +const processPixelWebEvents = async (event) => { const pixelEvent = processPixelEvent(event); return removeUndefinedAndNullValues(pixelEvent); }; module.exports = { - processEventFromPixel, + processPixelWebEvents, handleCartTokenRedisOperations, extractCartToken, }; diff --git a/src/v1/sources/shopify/pixelTransform.redisCartToken.test.js b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.redisCartToken.test.js similarity index 87% rename from src/v1/sources/shopify/pixelTransform.redisCartToken.test.js rename to src/v1/sources/shopify/webpixelTransformations/pixelTransform.redisCartToken.test.js index 8f54efc373..1e5cb94b19 100644 --- a/src/v1/sources/shopify/pixelTransform.redisCartToken.test.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.redisCartToken.test.js @@ -1,25 +1,25 @@ const { extractCartToken, handleCartTokenRedisOperations } = require('./pixelTransform'); -const { RedisDB } = require('../../../util/redis/redisConnector'); -const stats = require('../../../util/stats'); -const logger = require('../../../logger'); -const { pixelEventToCartTokenLocationMapping } = require('./config'); +const { RedisDB } = require('../../../../util/redis/redisConnector'); +const stats = require('../../../../util/stats'); +const logger = require('../../../../logger'); +const { pixelEventToCartTokenLocationMapping } = require('../config'); -jest.mock('../../../util/redis/redisConnector', () => ({ +jest.mock('../../../../util/redis/redisConnector', () => ({ RedisDB: { setVal: jest.fn(), }, })); -jest.mock('../../../util/stats', () => ({ +jest.mock('../../../../util/stats', () => ({ increment: jest.fn(), })); -jest.mock('../../../logger', () => ({ +jest.mock('../../../../logger', () => ({ info: jest.fn(), error: jest.fn(), })); -jest.mock('./config', () => ({ +jest.mock('../config', () => ({ pixelEventToCartTokenLocationMapping: { cart_viewed: 'properties.cart_id' }, })); diff --git a/src/v1/sources/shopify/pixelUtils.js b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js similarity index 97% rename from src/v1/sources/shopify/pixelUtils.js rename to src/v1/sources/shopify/webpixelTransformations/pixelUtils.js index 9abef5c2f8..0c1007f311 100644 --- a/src/v1/sources/shopify/pixelUtils.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js @@ -1,6 +1,6 @@ /* eslint-disable no-param-reassign */ -const Message = require('../../../v0/sources/message'); -const { EventType } = require('../../../constants'); +const Message = require('../../../../v0/sources/message'); +const { EventType } = require('../../../../constants'); const { INTEGERATION, PIXEL_EVENT_MAPPING, @@ -10,7 +10,7 @@ const { productViewedEventMappingJSON, productToCartEventMappingJSON, checkoutStartedCompletedEventMappingJSON, -} = require('./config'); +} = require('../config'); function getNestedValue(object, path) { const keys = path.split('.'); diff --git a/src/v1/sources/shopify/pixelUtils.test.js b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js similarity index 99% rename from src/v1/sources/shopify/pixelUtils.test.js rename to src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js index 4bff8eada4..e8f53a5f15 100644 --- a/src/v1/sources/shopify/pixelUtils.test.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js @@ -8,10 +8,9 @@ const { checkoutStepEventBuilder, searchEventBuilder, } = require('./pixelUtils'); -const { EventType } = require('../../../constants'); -const Message = require('../../../v0/sources/message'); +const Message = require('../../../../v0/sources/message'); jest.mock('ioredis', () => require('../../../../test/__mocks__/redis')); -jest.mock('../../../v0/sources/message'); +jest.mock('../../../../v0/sources/message'); describe('utilV2.js', () => { beforeEach(() => { diff --git a/test/integrations/component.test.ts b/test/integrations/component.test.ts index daed7c9e1f..baad6813df 100644 --- a/test/integrations/component.test.ts +++ b/test/integrations/component.test.ts @@ -41,6 +41,7 @@ command .option('-i, --index ', 'Enter Test index') .option('-g, --generate ', 'Enter "true" If you want to generate network file') .option('-id, --id ', 'Enter unique "Id" of the test case you want to run') + .option('-s, --source ', 'Enter Source Name') .parse(); const opts = command.opts(); diff --git a/test/integrations/sources/shopify/data.ts b/test/integrations/sources/shopify/data.ts index a2b27cbbcc..d4498e089c 100644 --- a/test/integrations/sources/shopify/data.ts +++ b/test/integrations/sources/shopify/data.ts @@ -1,8 +1,10 @@ -import { skip } from 'node:test'; import { pixelCheckoutEventsTestScenarios } from './pixelTestScenarios/CheckoutEventsTests'; import { pixelCheckoutStepsScenarios } from './pixelTestScenarios/CheckoutStepsTests'; import { pixelEventsTestScenarios } from './pixelTestScenarios/ProductEventsTests'; -import { v1ServerSideEventsScenarios } from './v1ServerSideEventsTests'; +import { checkoutEventsTestScenarios } from './webhookTestScenarios/CheckoutEventsTests'; +import { genericTrackTestScenarios } from './webhookTestScenarios/GenericTrackTests'; +import { identityTestScenarios } from './webhookTestScenarios/IdentifyTests'; +import { mockFns } from './mocks'; const serverSideEventsScenarios = [ { @@ -1422,6 +1424,7 @@ const serverSideEventsScenarios = [ verifiedEmail: true, }, type: 'track', + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', userId: '115310627314723950', }, ], @@ -1430,13 +1433,16 @@ const serverSideEventsScenarios = [ ], }, }, + mockFns, }, ]; export const data = [ + ...serverSideEventsScenarios, + ...checkoutEventsTestScenarios, + ...genericTrackTestScenarios, + ...identityTestScenarios, ...pixelCheckoutEventsTestScenarios, ...pixelCheckoutStepsScenarios, ...pixelEventsTestScenarios, - ...serverSideEventsScenarios, - ...v1ServerSideEventsScenarios, ]; diff --git a/test/integrations/sources/shopify/mocks.ts b/test/integrations/sources/shopify/mocks.ts new file mode 100644 index 0000000000..e1895e7812 --- /dev/null +++ b/test/integrations/sources/shopify/mocks.ts @@ -0,0 +1,5 @@ +import utils from '../../../../src/v0/util'; + +export const mockFns = (_) => { + jest.spyOn(utils, 'generateUUID').mockReturnValue('5d3e2cb6-4011-5c9c-b7ee-11bc1e905097'); +}; diff --git a/test/integrations/sources/shopify/v1ServerSideEventsTests.ts b/test/integrations/sources/shopify/v1ServerSideEventsTests.ts deleted file mode 100644 index 2c323cb370..0000000000 --- a/test/integrations/sources/shopify/v1ServerSideEventsTests.ts +++ /dev/null @@ -1,596 +0,0 @@ -// This file contains the test scenarios for the server-side events from the Shopify GraphQL API for -// the v1 transformation flow -import utils from '../../../../src/v0/util'; -const defaultMockFns = () => { - jest.spyOn(utils, 'generateUUID').mockReturnValue('5d3e2cb6-4011-5c9c-b7ee-11bc1e905097'); -}; -import { dummySourceConfig } from './constants'; - -export const v1ServerSideEventsScenarios = [ - { - name: 'shopify', - description: 'Track Call -> Checkout Updated event', - module: 'source', - version: 'v1', - input: { - request: { - body: [ - { - event: { - id: 35374569160817, - token: 'e89d4437003b6b8480f8bc7f8036a659', - cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - email: 'testuser101@gmail.com', - gateway: null, - buyer_accepts_marketing: false, - buyer_accepts_sms_marketing: false, - sms_marketing_phone: null, - created_at: '2024-09-16T03:50:15+00:00', - updated_at: '2024-09-17T03:29:02-04:00', - landing_site: '/', - note: '', - note_attributes: [], - referring_site: '', - shipping_lines: [ - { - code: 'Standard', - price: '6.90', - original_shop_price: '6.90', - original_shop_markup: '0.00', - source: 'shopify', - title: 'Standard', - presentment_title: 'Standard', - phone: null, - tax_lines: [], - custom_tax_lines: null, - markup: '0.00', - carrier_identifier: null, - carrier_service_id: null, - api_client_id: '580111', - delivery_option_group: { - token: '26492692a443ee35c30eb82073bacaa8', - type: 'one_time_purchase', - }, - delivery_expectation_range: null, - delivery_expectation_type: null, - id: null, - requested_fulfillment_service_id: null, - delivery_category: null, - validation_context: null, - applied_discounts: [], - }, - ], - shipping_address: { - first_name: 'testuser', - address1: 'oakwood bridge', - phone: null, - city: 'KLF', - zip: '85003', - province: 'Arizona', - country: 'United States', - last_name: 'dummy', - address2: 'Hedgetown', - company: null, - latitude: null, - longitude: null, - name: 'testuser dummy', - country_code: 'US', - province_code: 'AZ', - }, - taxes_included: false, - total_weight: 0, - currency: 'USD', - completed_at: null, - phone: null, - customer_locale: 'en-US', - line_items: [ - { - key: '41327143059569', - fulfillment_service: 'manual', - gift_card: false, - grams: 0, - presentment_title: 'The Multi-location Snowboard', - presentment_variant_title: '', - product_id: 7234590638193, - quantity: 1, - requires_shipping: true, - sku: '', - tax_lines: [], - taxable: true, - title: 'The Multi-location Snowboard', - variant_id: 41327143059569, - variant_title: '', - variant_price: '729.95', - vendor: 'pixel-testing-rs', - unit_price_measurement: { - measured_type: null, - quantity_value: null, - quantity_unit: null, - reference_value: null, - reference_unit: null, - }, - compare_at_price: null, - line_price: '729.95', - price: '729.95', - applied_discounts: [], - destination_location_id: null, - user_id: null, - rank: null, - origin_location_id: null, - properties: {}, - }, - ], - name: '#35374569160817', - abandoned_checkout_url: - 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2/recover?key=8195f56ee0de230b3a0469cc692f3436', - discount_codes: [], - tax_lines: [], - presentment_currency: 'USD', - source_name: 'web', - total_line_items_price: '729.95', - total_tax: '0.00', - total_discounts: '0.00', - subtotal_price: '729.95', - total_price: '736.85', - total_duties: '0.00', - device_id: null, - user_id: null, - location_id: null, - source_identifier: null, - source_url: null, - source: null, - closed_at: null, - customer: { - id: 7188389789809, - email: 'testuser101@gmail.com', - accepts_marketing: false, - created_at: null, - updated_at: null, - first_name: 'testuser', - last_name: 'dummy', - orders_count: 0, - state: 'disabled', - total_spent: '0.00', - last_order_id: null, - note: null, - verified_email: true, - multipass_identifier: null, - tax_exempt: false, - phone: null, - tags: '', - currency: 'USD', - accepts_marketing_updated_at: null, - admin_graphql_api_id: 'gid://shopify/Customer/7188389789809', - default_address: { - id: null, - customer_id: 7188389789809, - first_name: 'testuser', - last_name: 'dummy', - company: null, - address1: 'oakwood bridge', - address2: 'Hedgetown', - city: 'KLF', - province: 'Arizona', - country: 'United States', - zip: '85003', - phone: null, - name: 'testuser dummy', - province_code: 'AZ', - country_code: 'US', - country_name: 'United States', - default: true, - }, - last_order_name: null, - marketing_opt_in_level: null, - }, - query_parameters: { - topic: ['checkouts_update'], - writeKey: ['2l9QoM7KRMJLMcYhXNUVDT0Mqbd'], - }, - }, - source: dummySourceConfig, - }, - ], - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - }, - pathSuffix: '', - }, - output: { - response: { - status: 200, - body: [ - { - output: { - batch: [ - { - context: { - library: { - name: 'RudderStack Shopify Cloud', - version: '1.0.0', - }, - integration: { - name: 'SHOPIFY', - }, - topic: 'checkouts_update', - cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - }, - integrations: { - SHOPIFY: true, - }, - type: 'track', - event: 'Checkout Updated', - properties: { - order_id: 35374569160817, - value: '736.85', - tax: '0.00', - currency: 'USD', - token: 'e89d4437003b6b8480f8bc7f8036a659', - cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - email: 'testuser101@gmail.com', - buyer_accepts_marketing: false, - buyer_accepts_sms_marketing: false, - created_at: '2024-09-16T03:50:15+00:00', - updated_at: '2024-09-17T03:29:02-04:00', - landing_site: '/', - note: '', - note_attributes: [], - referring_site: '', - shipping_lines: [ - { - code: 'Standard', - price: '6.90', - original_shop_price: '6.90', - original_shop_markup: '0.00', - source: 'shopify', - title: 'Standard', - presentment_title: 'Standard', - phone: null, - tax_lines: [], - custom_tax_lines: null, - markup: '0.00', - carrier_identifier: null, - carrier_service_id: null, - api_client_id: '580111', - delivery_option_group: { - token: '26492692a443ee35c30eb82073bacaa8', - type: 'one_time_purchase', - }, - delivery_expectation_range: null, - delivery_expectation_type: null, - id: null, - requested_fulfillment_service_id: null, - delivery_category: null, - validation_context: null, - applied_discounts: [], - }, - ], - taxes_included: false, - total_weight: 0, - customer_locale: 'en-US', - name: '#35374569160817', - abandoned_checkout_url: - 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2/recover?key=8195f56ee0de230b3a0469cc692f3436', - discount_codes: [], - tax_lines: [], - presentment_currency: 'USD', - source_name: 'web', - total_line_items_price: '729.95', - total_discounts: '0.00', - subtotal_price: '729.95', - total_duties: '0.00', - products: [ - { - product_id: 7234590638193, - price: '729.95', - brand: 'pixel-testing-rs', - quantity: 1, - key: '41327143059569', - fulfillment_service: 'manual', - gift_card: false, - grams: 0, - presentment_title: 'The Multi-location Snowboard', - presentment_variant_title: '', - requires_shipping: true, - tax_lines: [], - taxable: true, - title: 'The Multi-location Snowboard', - unit_price_measurement: { - measured_type: null, - quantity_value: null, - quantity_unit: null, - reference_value: null, - reference_unit: null, - }, - compare_at_price: null, - line_price: '729.95', - applied_discounts: [], - destination_location_id: null, - user_id: null, - rank: null, - origin_location_id: null, - properties: {}, - variant: '41327143059569 729.95 ', - }, - ], - }, - userId: '7188389789809', - traits: { - email: 'testuser101@gmail.com', - firstName: 'testuser', - lastName: 'dummy', - address: { - id: null, - customer_id: 7188389789809, - first_name: 'testuser', - last_name: 'dummy', - company: null, - address1: 'oakwood bridge', - address2: 'Hedgetown', - city: 'KLF', - province: 'Arizona', - country: 'United States', - zip: '85003', - phone: null, - name: 'testuser dummy', - province_code: 'AZ', - country_code: 'US', - country_name: 'United States', - default: true, - }, - acceptsMarketing: false, - orderCount: 0, - state: 'disabled', - totalSpent: '0.00', - verifiedEmail: true, - taxExempt: false, - tags: '', - currency: 'USD', - adminGraphqlApiId: 'gid://shopify/Customer/7188389789809', - shippingAddress: { - first_name: 'testuser', - address1: 'oakwood bridge', - phone: null, - city: 'KLF', - zip: '85003', - province: 'Arizona', - country: 'United States', - last_name: 'dummy', - address2: 'Hedgetown', - company: null, - latitude: null, - longitude: null, - name: 'testuser dummy', - country_code: 'US', - province_code: 'AZ', - }, - }, - timestamp: '2024-09-17T07:29:02.000Z', - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', - }, - ], - }, - }, - ], - }, - }, - mockFns: () => { - defaultMockFns(); - }, - }, - { - name: 'shopify', - description: 'Track Call -> Cart Update event', - module: 'source', - version: 'v1', - input: { - request: { - body: [ - { - event: { - id: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - line_items: [ - { - id: 41327143059569, - properties: null, - quantity: 3, - variant_id: 41327143059569, - key: '41327143059569:90562f18109e0e6484b0c297e7981b30', - discounted_price: '729.95', - discounts: [], - gift_card: false, - grams: 0, - line_price: '2189.85', - original_line_price: '2189.85', - original_price: '729.95', - price: '729.95', - product_id: 7234590638193, - sku: '', - taxable: true, - title: 'The Multi-location Snowboard', - total_discount: '0.00', - vendor: 'pixel-testing-rs', - discounted_price_set: { - shop_money: { - amount: '729.95', - currency_code: 'USD', - }, - presentment_money: { - amount: '729.95', - currency_code: 'USD', - }, - }, - line_price_set: { - shop_money: { - amount: '2189.85', - currency_code: 'USD', - }, - presentment_money: { - amount: '2189.85', - currency_code: 'USD', - }, - }, - original_line_price_set: { - shop_money: { - amount: '2189.85', - currency_code: 'USD', - }, - presentment_money: { - amount: '2189.85', - currency_code: 'USD', - }, - }, - price_set: { - shop_money: { - amount: '729.95', - currency_code: 'USD', - }, - presentment_money: { - amount: '729.95', - currency_code: 'USD', - }, - }, - total_discount_set: { - shop_money: { - amount: '0.0', - currency_code: 'USD', - }, - presentment_money: { - amount: '0.0', - currency_code: 'USD', - }, - }, - }, - ], - note: '', - updated_at: '2024-09-17T08:15:13.280Z', - created_at: '2024-09-16T03:50:15.478Z', - query_parameters: { - topic: ['carts_update'], - writeKey: ['2l9QoM7KRMJLMcYhXNUVDT0Mqbd'], - }, - }, - source: dummySourceConfig, - }, - ], - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - }, - pathSuffix: '', - }, - output: { - response: { - status: 200, - body: [ - { - output: { - batch: [ - { - context: { - library: { - name: 'RudderStack Shopify Cloud', - version: '1.0.0', - }, - integration: { - name: 'SHOPIFY', - }, - topic: 'carts_update', - }, - integrations: { - SHOPIFY: true, - }, - type: 'track', - event: 'Cart Update', - properties: { - id: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', - note: '', - updated_at: '2024-09-17T08:15:13.280Z', - created_at: '2024-09-16T03:50:15.478Z', - products: [ - { - product_id: 7234590638193, - price: '729.95', - brand: 'pixel-testing-rs', - quantity: 3, - id: 41327143059569, - properties: null, - key: '41327143059569:90562f18109e0e6484b0c297e7981b30', - discounted_price: '729.95', - discounts: [], - gift_card: false, - grams: 0, - line_price: '2189.85', - original_line_price: '2189.85', - original_price: '729.95', - taxable: true, - title: 'The Multi-location Snowboard', - total_discount: '0.00', - discounted_price_set: { - shop_money: { - amount: '729.95', - currency_code: 'USD', - }, - presentment_money: { - amount: '729.95', - currency_code: 'USD', - }, - }, - line_price_set: { - shop_money: { - amount: '2189.85', - currency_code: 'USD', - }, - presentment_money: { - amount: '2189.85', - currency_code: 'USD', - }, - }, - original_line_price_set: { - shop_money: { - amount: '2189.85', - currency_code: 'USD', - }, - presentment_money: { - amount: '2189.85', - currency_code: 'USD', - }, - }, - price_set: { - shop_money: { - amount: '729.95', - currency_code: 'USD', - }, - presentment_money: { - amount: '729.95', - currency_code: 'USD', - }, - }, - total_discount_set: { - shop_money: { - amount: '0.0', - currency_code: 'USD', - }, - presentment_money: { - amount: '0.0', - currency_code: 'USD', - }, - }, - variant: '41327143059569 ', - }, - ], - }, - anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', - }, - ], - }, - }, - ], - }, - }, - mockFns: () => { - defaultMockFns(); - }, - }, -]; diff --git a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts new file mode 100644 index 0000000000..ade496efb7 --- /dev/null +++ b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts @@ -0,0 +1,1687 @@ +// This file contains the test scenarios for the server-side events from the Shopify GraphQL API for +// the v1 transformation flow +import { mockFns } from '../mocks'; +import { dummySourceConfig } from '../constants'; + +export const checkoutEventsTestScenarios = [ + { + id: 'c001', + name: 'shopify', + description: 'Track Call -> Checkout Started event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 35550298931313, + token: '84ad78572dae52a8cbea7d55371afe89', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + email: null, + gateway: null, + buyer_accepts_marketing: false, + buyer_accepts_sms_marketing: false, + sms_marketing_phone: null, + created_at: '2024-11-06T02:22:00+00:00', + updated_at: '2024-11-05T21:22:02-05:00', + landing_site: '/', + note: '', + note_attributes: [], + referring_site: '', + shipping_lines: [], + shipping_address: [], + taxes_included: false, + total_weight: 0, + currency: 'USD', + completed_at: null, + phone: null, + customer_locale: 'en-US', + line_items: [ + { + key: '41327142600817', + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + presentment_title: 'The Collection Snowboard: Hydrogen', + presentment_variant_title: '', + product_id: 7234590408817, + quantity: 1, + requires_shipping: true, + sku: '', + tax_lines: [], + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + variant_id: 41327142600817, + variant_title: '', + variant_price: '600.00', + vendor: 'Hydrogen Vendor', + unit_price_measurement: { + measured_type: null, + quantity_value: null, + quantity_unit: null, + reference_value: null, + reference_unit: null, + }, + compare_at_price: null, + line_price: '600.00', + price: '600.00', + applied_discounts: [], + destination_location_id: null, + user_id: null, + rank: null, + origin_location_id: null, + properties: {}, + }, + ], + name: '#35550298931313', + abandoned_checkout_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ/recover?key=0385163be3875d3a2117e982d9cc3517&locale=en-US', + discount_codes: [], + tax_lines: [], + presentment_currency: 'USD', + source_name: 'web', + total_line_items_price: '600.00', + total_tax: '0.00', + total_discounts: '0.00', + subtotal_price: '600.00', + total_price: '600.00', + total_duties: '0.00', + device_id: null, + user_id: null, + location_id: null, + source_identifier: null, + source_url: null, + source: null, + closed_at: null, + query_parameters: { + topic: ['checkouts_create'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + query_parameters: { + topic: ['carts_update'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + integration: { + name: 'SHOPIFY', + }, + topic: 'checkouts_create', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + shopifyDetails: { + id: 35550298931313, + token: '84ad78572dae52a8cbea7d55371afe89', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + email: null, + gateway: null, + buyer_accepts_marketing: false, + buyer_accepts_sms_marketing: false, + sms_marketing_phone: null, + created_at: '2024-11-06T02:22:00+00:00', + updated_at: '2024-11-05T21:22:02-05:00', + landing_site: '/', + note: '', + note_attributes: [], + referring_site: '', + shipping_lines: [], + shipping_address: [], + taxes_included: false, + total_weight: 0, + currency: 'USD', + completed_at: null, + phone: null, + customer_locale: 'en-US', + line_items: [ + { + key: '41327142600817', + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + presentment_title: 'The Collection Snowboard: Hydrogen', + presentment_variant_title: '', + product_id: 7234590408817, + quantity: 1, + requires_shipping: true, + sku: '', + tax_lines: [], + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + variant_id: 41327142600817, + variant_title: '', + variant_price: '600.00', + vendor: 'Hydrogen Vendor', + unit_price_measurement: { + measured_type: null, + quantity_value: null, + quantity_unit: null, + reference_value: null, + reference_unit: null, + }, + compare_at_price: null, + line_price: '600.00', + price: '600.00', + applied_discounts: [], + destination_location_id: null, + user_id: null, + rank: null, + origin_location_id: null, + properties: {}, + }, + ], + name: '#35550298931313', + abandoned_checkout_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ/recover?key=0385163be3875d3a2117e982d9cc3517&locale=en-US', + discount_codes: [], + tax_lines: [], + presentment_currency: 'USD', + source_name: 'web', + total_line_items_price: '600.00', + total_tax: '0.00', + total_discounts: '0.00', + subtotal_price: '600.00', + total_price: '600.00', + total_duties: '0.00', + device_id: null, + user_id: null, + location_id: null, + source_identifier: null, + source_url: null, + source: null, + closed_at: null, + }, + }, + integrations: { + SHOPIFY: true, + }, + type: 'track', + event: 'Checkout Started', + properties: { + order_id: 35550298931313, + value: '600.00', + tax: '0.00', + currency: 'USD', + products: [ + { + product_id: 7234590408817, + price: '600.00', + brand: 'Hydrogen Vendor', + quantity: 1, + }, + ], + }, + timestamp: '2024-11-06T02:22:02.000Z', + traits: { + shippingAddress: [], + }, + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + }, + ], + }, + }, + ], + }, + }, + }, + { + id: 'c002', + name: 'shopify', + description: 'Track Call -> Checkout Updated event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + query_parameters: { + topic: ['checkouts_update'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + id: 35374569160817, + token: 'e89d4437003b6b8480f8bc7f8036a659', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + email: 'testuser101@gmail.com', + created_at: '2024-09-16T03:50:1500:00', + updated_at: '2024-09-17T03:29:02-04:00', + note: '', + note_attributes: [], + shipping_address: { + first_name: 'testuser', + address1: 'oakwood bridge', + phone: null, + city: 'KLF', + zip: '85003', + province: 'Arizona', + country: 'United States', + last_name: 'dummy', + address2: 'Hedgetown', + company: null, + latitude: null, + longitude: null, + name: 'testuser dummy', + country_code: 'US', + province_code: 'AZ', + }, + total_weight: 0, + currency: 'USD', + customer_locale: 'en-US', + line_items: [ + { + key: '41327143059569', + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + presentment_title: 'The Multi-location Snowboard', + presentment_variant_title: '', + product_id: 7234590638193, + quantity: 1, + requires_shipping: true, + sku: '', + tax_lines: [], + taxable: true, + title: 'The Multi-location Snowboard', + variant_id: 41327143059569, + variant_title: '', + variant_price: '729.95', + vendor: 'pixel-testing-rs', + unit_price_measurement: { + measured_type: null, + quantity_value: null, + quantity_unit: null, + reference_value: null, + reference_unit: null, + }, + compare_at_price: null, + line_price: '729.95', + price: '729.95', + applied_discounts: [], + destination_location_id: null, + user_id: null, + rank: null, + origin_location_id: null, + properties: {}, + }, + ], + name: '#35374569160817', + abandoned_checkout_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2/recover?key=8195f56ee0de230b3a0469cc692f3436', + presentment_currency: 'USD', + total_tax: '0.00', + total_discounts: '0.00', + subtotal_price: '729.95', + total_price: '736.85', + total_duties: '0.00', + customer: { + id: 7188389789809, + email: 'testuser101@gmail.com', + accepts_marketing: false, + created_at: null, + updated_at: null, + first_name: 'testuser', + last_name: 'dummy', + orders_count: 0, + state: 'disabled', + total_spent: '0.00', + last_order_id: null, + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + tags: '', + currency: 'USD', + accepts_marketing_updated_at: null, + admin_graphql_api_id: 'gid://shopify/Customer/7188389789809', + default_address: { + id: null, + customer_id: 7188389789809, + first_name: 'testuser', + last_name: 'dummy', + company: null, + address1: 'oakwood bridge', + address2: 'Hedgetown', + city: 'KLF', + province: 'Arizona', + country: 'United States', + zip: '85003', + phone: null, + name: 'testuser dummy', + province_code: 'AZ', + country_code: 'US', + country_name: 'United States', + default: true, + }, + last_order_name: null, + marketing_opt_in_level: null, + }, + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + context: { + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + integration: { + name: 'SHOPIFY', + }, + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + shopifyDetails: { + id: 35374569160817, + token: 'e89d4437003b6b8480f8bc7f8036a659', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + email: 'testuser101@gmail.com', + created_at: '2024-09-16T03:50:1500:00', + updated_at: '2024-09-17T03:29:02-04:00', + note: '', + note_attributes: [], + shipping_address: { + first_name: 'testuser', + address1: 'oakwood bridge', + phone: null, + city: 'KLF', + zip: '85003', + province: 'Arizona', + country: 'United States', + last_name: 'dummy', + address2: 'Hedgetown', + company: null, + latitude: null, + longitude: null, + name: 'testuser dummy', + country_code: 'US', + province_code: 'AZ', + }, + total_weight: 0, + currency: 'USD', + customer_locale: 'en-US', + line_items: [ + { + key: '41327143059569', + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + presentment_title: 'The Multi-location Snowboard', + presentment_variant_title: '', + product_id: 7234590638193, + quantity: 1, + requires_shipping: true, + sku: '', + tax_lines: [], + taxable: true, + title: 'The Multi-location Snowboard', + variant_id: 41327143059569, + variant_title: '', + variant_price: '729.95', + vendor: 'pixel-testing-rs', + unit_price_measurement: { + measured_type: null, + quantity_value: null, + quantity_unit: null, + reference_value: null, + reference_unit: null, + }, + compare_at_price: null, + line_price: '729.95', + price: '729.95', + applied_discounts: [], + destination_location_id: null, + user_id: null, + rank: null, + origin_location_id: null, + properties: {}, + }, + ], + name: '#35374569160817', + abandoned_checkout_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/checkouts/ac/Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2/recover?key=8195f56ee0de230b3a0469cc692f3436', + presentment_currency: 'USD', + total_tax: '0.00', + total_discounts: '0.00', + subtotal_price: '729.95', + total_price: '736.85', + total_duties: '0.00', + customer: { + id: 7188389789809, + email: 'testuser101@gmail.com', + accepts_marketing: false, + created_at: null, + updated_at: null, + first_name: 'testuser', + last_name: 'dummy', + orders_count: 0, + state: 'disabled', + total_spent: '0.00', + last_order_id: null, + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + tags: '', + currency: 'USD', + accepts_marketing_updated_at: null, + admin_graphql_api_id: 'gid://shopify/Customer/7188389789809', + default_address: { + id: null, + customer_id: 7188389789809, + first_name: 'testuser', + last_name: 'dummy', + company: null, + address1: 'oakwood bridge', + address2: 'Hedgetown', + city: 'KLF', + province: 'Arizona', + country: 'United States', + zip: '85003', + phone: null, + name: 'testuser dummy', + province_code: 'AZ', + country_code: 'US', + country_name: 'United States', + default: true, + }, + last_order_name: null, + marketing_opt_in_level: null, + }, + }, + topic: 'checkouts_update', + }, + event: 'Checkout Updated', + integrations: { + SHOPIFY: true, + }, + properties: { + currency: 'USD', + order_id: 35374569160817, + products: [ + { + brand: 'pixel-testing-rs', + price: '729.95', + product_id: 7234590638193, + quantity: 1, + }, + ], + tax: '0.00', + value: '736.85', + }, + timestamp: '2024-09-17T07:29:02.000Z', + traits: { + acceptsMarketing: false, + address: { + address1: 'oakwood bridge', + address2: 'Hedgetown', + city: 'KLF', + company: null, + country: 'United States', + country_code: 'US', + country_name: 'United States', + customer_id: 7188389789809, + default: true, + first_name: 'testuser', + id: null, + last_name: 'dummy', + name: 'testuser dummy', + phone: null, + province: 'Arizona', + province_code: 'AZ', + zip: '85003', + }, + adminGraphqlApiId: 'gid://shopify/Customer/7188389789809', + currency: 'USD', + email: 'testuser101@gmail.com', + firstName: 'testuser', + lastName: 'dummy', + orderCount: 0, + shippingAddress: { + address1: 'oakwood bridge', + address2: 'Hedgetown', + city: 'KLF', + company: null, + country: 'United States', + country_code: 'US', + first_name: 'testuser', + last_name: 'dummy', + latitude: null, + longitude: null, + name: 'testuser dummy', + phone: null, + province: 'Arizona', + province_code: 'AZ', + zip: '85003', + }, + state: 'disabled', + tags: '', + taxExempt: false, + totalSpent: '0.00', + verifiedEmail: true, + }, + type: 'track', + userId: '7188389789809', + }, + ], + }, + }, + ], + }, + }, + }, + { + id: 'c003', + name: 'shopify', + description: 'Track Call -> Order Updated event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + app_id: 580111, + browser_ip: '139.5.255.205', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + confirmation_number: 'DPPARQ8UJ', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + current_total_discounts: '0.00', + current_total_discounts_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + current_total_price: '600.00', + current_total_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + current_total_tax: '0.00', + current_total_tax_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + customer_locale: 'en-US', + discount_codes: [], + email: 'henry@wfls.com', + estimated_taxes: false, + merchant_of_record_app_id: null, + name: '#1017', + note: null, + note_attributes: [], + number: 17, + order_number: 1017, + order_status_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', + original_total_additional_fees_set: null, + original_total_duties_set: null, + payment_gateway_names: ['bogus'], + phone: null, + presentment_currency: 'USD', + source_identifier: '4d92cf60cc24a1bd95929e17ead9845f', + subtotal_price: '600.00', + subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + tax_lines: [], + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_discounts_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_line_items_price: '600.00', + total_line_items_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + total_price: '600.00', + total_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + total_shipping_price_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_tax: '0.00', + total_tax_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_weight: 0, + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + tags: '', + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + admin_graphql_api_id: 'gid://shopify/LineItem/14234727743601', + attributed_staffs: [], + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + fulfillment_status: null, + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + product_exists: true, + product_id: 7234590408817, + properties: [], + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + total_discount_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + tax_lines: [], + duties: [], + discount_allocations: [], + }, + ], + payment_terms: null, + refunds: [], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + query_parameters: { + topic: ['orders_updated'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + integration: { + name: 'SHOPIFY', + }, + topic: 'orders_updated', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + shopifyDetails: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + app_id: 580111, + browser_ip: '139.5.255.205', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + confirmation_number: 'DPPARQ8UJ', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + current_total_discounts: '0.00', + current_total_discounts_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + current_total_price: '600.00', + current_total_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + current_total_tax: '0.00', + current_total_tax_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + customer_locale: 'en-US', + discount_codes: [], + email: 'henry@wfls.com', + estimated_taxes: false, + merchant_of_record_app_id: null, + name: '#1017', + note: null, + note_attributes: [], + number: 17, + order_number: 1017, + order_status_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', + original_total_additional_fees_set: null, + original_total_duties_set: null, + payment_gateway_names: ['bogus'], + phone: null, + presentment_currency: 'USD', + source_identifier: '4d92cf60cc24a1bd95929e17ead9845f', + subtotal_price: '600.00', + subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + tax_lines: [], + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_discounts_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_line_items_price: '600.00', + total_line_items_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + total_price: '600.00', + total_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + total_shipping_price_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_tax: '0.00', + total_tax_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + total_weight: 0, + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + tags: '', + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + admin_graphql_api_id: 'gid://shopify/LineItem/14234727743601', + attributed_staffs: [], + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + fulfillment_status: null, + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + product_exists: true, + product_id: 7234590408817, + properties: [], + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + total_discount_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + tax_lines: [], + duties: [], + discount_allocations: [], + }, + ], + payment_terms: null, + refunds: [], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + }, + order_token: '676613a0027fc8240e16d67fdc9f5ac8', + }, + integrations: { + SHOPIFY: true, + }, + type: 'track', + event: 'Order Updated', + properties: { + order_id: 5778367414385, + value: '600.00', + tax: '0.00', + currency: 'USD', + products: [ + { + product_id: 7234590408817, + title: 'The Collection Snowboard: Hydrogen', + price: '600.00', + brand: 'Hydrogen Vendor', + quantity: 1, + }, + ], + }, + userId: '7358220173425', + traits: { + email: 'henry@wfls.com', + firstName: 'yodi', + lastName: 'waffles', + address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + state: 'disabled', + verifiedEmail: true, + taxExempt: false, + tags: '', + currency: 'USD', + taxExemptions: [], + adminGraphqlApiId: 'gid://shopify/Customer/7358220173425', + shippingAddress: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + billingAddress: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + }, + timestamp: '2024-11-06T02:54:50.000Z', + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + }, + ], + }, + }, + ], + }, + }, + }, + { + id: 'c004', + name: 'shopify', + description: 'Track Call -> Order Created event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + app_id: 580111, + browser_ip: '139.5.255.205', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + confirmation_number: 'DPPARQ8UJ', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_total_discounts: '0.00', + current_total_price: '600.00', + current_total_tax: '0.00', + email: 'henry@wfls.com', + name: '#1017', + order_number: 1017, + order_status_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', + phone: null, + presentment_currency: 'USD', + subtotal_price: '600.00', + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_line_items_price: '600.00', + total_outstanding: '0.00', + total_price: '600.00', + total_tax: '0.00', + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + phone: null, + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + }, + ], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + query_parameters: { + topic: ['orders_create'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnZkCHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + query_parameters: { + topic: ['carts_update'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + integration: { + name: 'SHOPIFY', + }, + topic: 'orders_create', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + shopifyDetails: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + app_id: 580111, + browser_ip: '139.5.255.205', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + confirmation_number: 'DPPARQ8UJ', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_total_discounts: '0.00', + current_total_price: '600.00', + current_total_tax: '0.00', + email: 'henry@wfls.com', + name: '#1017', + order_number: 1017, + order_status_url: + 'https://pixel-testing-rs.myshopify.com/59026964593/orders/676613a0027fc8240e16d67fdc9f5ac8/authenticate?key=a70bbe7ec8abcc46b77e4331e4df8c60', + phone: null, + presentment_currency: 'USD', + subtotal_price: '600.00', + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_line_items_price: '600.00', + total_outstanding: '0.00', + total_price: '600.00', + total_tax: '0.00', + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + phone: null, + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + }, + ], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + }, + }, + integrations: { + SHOPIFY: true, + }, + type: 'track', + event: 'Order Created', + properties: { + order_id: 5778367414385, + value: '600.00', + tax: '0.00', + currency: 'USD', + products: [ + { + product_id: 7234590408817, + title: 'The Collection Snowboard: Hydrogen', + price: '600.00', + brand: 'Hydrogen Vendor', + quantity: 1, + }, + ], + }, + userId: '7358220173425', + traits: { + email: 'henry@wfls.com', + firstName: 'yodi', + lastName: 'waffles', + address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + state: 'disabled', + currency: 'USD', + taxExemptions: [], + adminGraphqlApiId: 'gid://shopify/Customer/7358220173425', + shippingAddress: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + billingAddress: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + }, + timestamp: '2024-11-06T02:54:50.000Z', + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + }, + ], + }, + }, + ], + }, + }, + }, +].map((d1) => ({ ...d1, mockFns })); diff --git a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts new file mode 100644 index 0000000000..f04fd7e08e --- /dev/null +++ b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts @@ -0,0 +1,557 @@ +// This file contains the test scenarios for the server-side events from the Shopify GraphQL API for +// the v1 transformation flow +import { mockFns } from '../mocks'; +import { dummySourceConfig } from '../constants'; + +export const genericTrackTestScenarios = [ + { + id: 'c005', + name: 'shopify', + description: 'Track Call -> Cart Update event with no line items from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + query_parameters: { + topic: ['carts_update'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + id: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + line_items: [], + note: '', + updated_at: '2024-09-17T08:15:13.280Z', + created_at: '2024-09-16T03:50:15.478Z', + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + context: { + integration: { + name: 'SHOPIFY', + }, + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + shopifyDetails: { + created_at: '2024-09-16T03:50:15.478Z', + id: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + line_items: [], + note: '', + token: 'Z2NwLXVzLWVhc3QxOjAxSjdXRjdOQjY0NlFFNFdQVEg0MTRFM1E2', + updated_at: '2024-09-17T08:15:13.280Z', + }, + topic: 'carts_update', + }, + event: 'Cart Update', + integrations: { + SHOPIFY: true, + }, + properties: { + products: [], + }, + type: 'track', + }, + ], + }, + }, + ], + }, + }, + }, + { + id: 'c006', + name: 'shopify', + description: 'Track Call -> Unsupported event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 35550298931313, + query_parameters: { + topic: ['unsupported_event'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + outputToSource: { + body: 'T0s=', + contentType: 'text/plain', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + id: 'c007', + name: 'shopify', + description: 'Track Call -> generic event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_total_additional_fees_set: null, + current_total_discounts: '0.00', + current_total_duties_set: null, + current_total_price: '600.00', + current_total_tax: '0.00', + email: 'henry@wfls.com', + merchant_of_record_app_id: null, + name: '#1017', + note: null, + note_attributes: [], + order_number: 1017, + original_total_additional_fees_set: null, + original_total_duties_set: null, + payment_gateway_names: ['bogus'], + phone: null, + po_number: null, + presentment_currency: 'USD', + processed_at: '2024-11-05T21:54:48-05:00', + reference: '4d92cf60cc24a1bd95929e17ead9845f', + referring_site: '', + source_identifier: '4d92cf60cc24a1bd95929e17ead9845f', + subtotal_price: '600.00', + subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_line_items_price: '600.00', + total_outstanding: '0.00', + total_price: '600.00', + total_tax: '0.00', + total_weight: 0, + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + tags: '', + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + admin_graphql_api_id: 'gid://shopify/LineItem/14234727743601', + attributed_staffs: [], + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + fulfillment_status: null, + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + product_exists: true, + product_id: 7234590408817, + properties: [], + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + total_discount_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + tax_lines: [], + duties: [], + discount_allocations: [], + }, + ], + refunds: [], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + query_parameters: { + topic: ['orders_paid'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnZkCHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + integration: { + name: 'SHOPIFY', + }, + topic: 'orders_paid', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + shopifyDetails: { + id: 5778367414385, + admin_graphql_api_id: 'gid://shopify/Order/5778367414385', + cart_token: 'Z2NwLXVzLWVhc3QxOjAxSkJaTUVRSjgzNUJUN1BTNjEzRFdRUFFQ', + checkout_id: 35550298931313, + checkout_token: '84ad78572dae52a8cbea7d55371afe89', + contact_email: 'henry@wfls.com', + created_at: '2024-11-05T21:54:49-05:00', + currency: 'USD', + current_subtotal_price: '600.00', + current_total_additional_fees_set: null, + current_total_discounts: '0.00', + current_total_duties_set: null, + current_total_price: '600.00', + current_total_tax: '0.00', + email: 'henry@wfls.com', + merchant_of_record_app_id: null, + name: '#1017', + note: null, + note_attributes: [], + order_number: 1017, + original_total_additional_fees_set: null, + original_total_duties_set: null, + payment_gateway_names: ['bogus'], + phone: null, + po_number: null, + presentment_currency: 'USD', + processed_at: '2024-11-05T21:54:48-05:00', + reference: '4d92cf60cc24a1bd95929e17ead9845f', + referring_site: '', + source_identifier: '4d92cf60cc24a1bd95929e17ead9845f', + subtotal_price: '600.00', + subtotal_price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + token: '676613a0027fc8240e16d67fdc9f5ac8', + total_discounts: '0.00', + total_line_items_price: '600.00', + total_outstanding: '0.00', + total_price: '600.00', + total_tax: '0.00', + total_weight: 0, + updated_at: '2024-11-05T21:54:50-05:00', + user_id: null, + billing_address: { + first_name: 'yodi', + address1: 'Yuma Proving Ground', + phone: null, + city: 'Yuma Proving Ground', + zip: '85365', + province: 'Arizona', + country: 'United States', + last_name: 'waffles', + address2: 'suite 001', + company: null, + latitude: 33.0177811, + longitude: -114.2525392, + name: 'yodi waffles', + country_code: 'US', + province_code: 'AZ', + }, + customer: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + state: 'disabled', + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + phone: null, + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + tags: '', + currency: 'USD', + tax_exemptions: [], + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + line_items: [ + { + id: 14234727743601, + admin_graphql_api_id: 'gid://shopify/LineItem/14234727743601', + attributed_staffs: [], + current_quantity: 1, + fulfillable_quantity: 1, + fulfillment_service: 'manual', + fulfillment_status: null, + gift_card: false, + grams: 0, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + price_set: { + shop_money: { + amount: '600.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '600.00', + currency_code: 'USD', + }, + }, + product_exists: true, + product_id: 7234590408817, + properties: [], + quantity: 1, + requires_shipping: true, + sku: '', + taxable: true, + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + total_discount_set: { + shop_money: { + amount: '0.00', + currency_code: 'USD', + }, + presentment_money: { + amount: '0.00', + currency_code: 'USD', + }, + }, + variant_id: 41327142600817, + variant_inventory_management: 'shopify', + variant_title: null, + vendor: 'Hydrogen Vendor', + tax_lines: [], + duties: [], + discount_allocations: [], + }, + ], + refunds: [], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + phone: null, + city: 'Johnson City', + zip: '37604', + province: 'Tennessee', + country: 'United States', + last_name: 'waffles', + address2: '6', + company: null, + latitude: 36.3528845, + longitude: -82.4006335, + name: 'henry waffles', + country_code: 'US', + province_code: 'TN', + }, + }, + }, + integrations: { + SHOPIFY: true, + }, + type: 'track', + event: 'Order Paid', + properties: { + products: [ + { + product_id: 7234590408817, + title: 'The Collection Snowboard: Hydrogen', + price: '600.00', + brand: 'Hydrogen Vendor', + quantity: 1, + }, + ], + }, + traits: { + email: 'henry@wfls.com', + }, + anonymousId: '5d3e2cb6-4011-5c9c-b7ee-11bc1e905097', + }, + ], + }, + }, + ], + }, + }, + }, +].map((d2) => ({ ...d2, mockFns })); diff --git a/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts new file mode 100644 index 0000000000..b03f5635b6 --- /dev/null +++ b/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts @@ -0,0 +1,256 @@ +import { mockFns } from '../mocks'; +import { dummySourceConfig } from '../constants'; + +export const identityTestScenarios = [ + { + id: 'c008', + name: 'shopify', + description: 'Identify Call -> Customer update event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + orders_count: 0, + state: 'disabled', + total_spent: '0.00', + last_order_id: null, + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + tags: '', + last_order_name: null, + currency: 'USD', + phone: null, + addresses: [ + { + id: 8715246895217, + customer_id: 7358220173425, + first_name: 'yodi', + last_name: 'waffles', + company: null, + address1: 'Yuma Proving Ground', + address2: 'suite 001', + city: 'Yuma Proving Ground', + province: 'Arizona', + country: 'United States', + zip: '85365', + phone: null, + name: 'yodi waffles', + province_code: 'AZ', + country_code: 'US', + country_name: 'United States', + default: false, + }, + ], + tax_exemptions: [], + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + query_parameters: { + topic: ['customers_update'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + query_parameters: { + topic: ['carts_update'], + writeKey: ['2mw9SN679HngnXXXHT4oSVVBVmb'], + version: ['pixel'], + }, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + integration: { + name: 'SHOPIFY', + }, + topic: 'customers_update', + shopifyDetails: { + id: 7358220173425, + email: 'henry@wfls.com', + created_at: '2024-10-23T16:03:11-04:00', + updated_at: '2024-11-05T21:54:49-05:00', + first_name: 'yodi', + last_name: 'waffles', + orders_count: 0, + state: 'disabled', + total_spent: '0.00', + last_order_id: null, + note: null, + verified_email: true, + multipass_identifier: null, + tax_exempt: false, + tags: '', + last_order_name: null, + currency: 'USD', + phone: null, + addresses: [ + { + id: 8715246895217, + customer_id: 7358220173425, + first_name: 'yodi', + last_name: 'waffles', + company: null, + address1: 'Yuma Proving Ground', + address2: 'suite 001', + city: 'Yuma Proving Ground', + province: 'Arizona', + country: 'United States', + zip: '85365', + phone: null, + name: 'yodi waffles', + province_code: 'AZ', + country_code: 'US', + country_name: 'United States', + default: false, + }, + ], + tax_exemptions: [], + email_marketing_consent: { + state: 'not_subscribed', + opt_in_level: 'single_opt_in', + consent_updated_at: null, + }, + sms_marketing_consent: null, + admin_graphql_api_id: 'gid://shopify/Customer/7358220173425', + default_address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + }, + }, + integrations: { + SHOPIFY: true, + }, + type: 'identify', + userId: '7358220173425', + traits: { + email: 'henry@wfls.com', + firstName: 'yodi', + lastName: 'waffles', + addressList: [ + { + id: 8715246895217, + customer_id: 7358220173425, + first_name: 'yodi', + last_name: 'waffles', + company: null, + address1: 'Yuma Proving Ground', + address2: 'suite 001', + city: 'Yuma Proving Ground', + province: 'Arizona', + country: 'United States', + zip: '85365', + phone: null, + name: 'yodi waffles', + province_code: 'AZ', + country_code: 'US', + country_name: 'United States', + default: false, + }, + ], + address: { + id: 8715246862449, + customer_id: 7358220173425, + first_name: 'henry', + last_name: 'waffles', + company: null, + address1: 'Yuimaru Kitchen', + address2: '6', + city: 'Johnson City', + province: 'Tennessee', + country: 'United States', + zip: '37604', + phone: null, + name: 'henry waffles', + province_code: 'TN', + country_code: 'US', + country_name: 'United States', + default: true, + }, + orderCount: 0, + state: 'disabled', + totalSpent: '0.00', + verifiedEmail: true, + taxExempt: false, + tags: '', + currency: 'USD', + taxExemptions: [], + adminGraphqlApiId: 'gid://shopify/Customer/7358220173425', + }, + timestamp: '2024-11-06T02:54:49.000Z', + }, + ], + }, + }, + ], + }, + }, + }, +].map((d3) => ({ ...d3, mockFns })); From a80f87486dc93b423e4fe6efbee6f4cb8330ba02 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Mon, 18 Nov 2024 12:50:00 +0530 Subject: [PATCH 19/32] fix: adding uuid transformation for airship (#3884) * fix: adding uuid transformation for airship * fix: adding doc refs --- package-lock.json | 5 +++-- package.json | 2 +- src/v0/destinations/airship/transform.js | 4 ++++ src/v0/destinations/airship/utils.js | 12 ++++++++++++ .../destinations/airship/processor/data.ts | 6 +++--- 5 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 src/v0/destinations/airship/utils.js diff --git a/package-lock.json b/package-lock.json index 0422078f21..dd490e88c6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -73,7 +73,7 @@ "truncate-utf8-bytes": "^1.0.2", "ua-parser-js": "^1.0.37", "unset-value": "^2.0.1", - "uuid": "^9.0.0", + "uuid": "^9.0.1", "valid-url": "^1.0.9", "zod": "^3.22.4" }, @@ -21872,11 +21872,12 @@ }, "node_modules/uuid": { "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], - "license": "MIT", "bin": { "uuid": "dist/bin/uuid" } diff --git a/package.json b/package.json index 907c340cbf..75a618574c 100644 --- a/package.json +++ b/package.json @@ -118,7 +118,7 @@ "truncate-utf8-bytes": "^1.0.2", "ua-parser-js": "^1.0.37", "unset-value": "^2.0.1", - "uuid": "^9.0.0", + "uuid": "^9.0.1", "valid-url": "^1.0.9", "zod": "^3.22.4" }, diff --git a/src/v0/destinations/airship/transform.js b/src/v0/destinations/airship/transform.js index 091c9b7f39..dc8543fbc5 100644 --- a/src/v0/destinations/airship/transform.js +++ b/src/v0/destinations/airship/transform.js @@ -24,6 +24,7 @@ const { simpleProcessRouterDest, } = require('../../util'); const { JSON_MIME_TYPE } = require('../../util/constant'); +const { transformSessionId } = require('./utils'); const DEFAULT_ACCEPT_HEADER = 'application/vnd.urbanairship+json; version=3'; @@ -128,6 +129,9 @@ const trackResponseBuilder = async (message, { Config }) => { name = name.toLowerCase(); const payload = constructPayload(message, trackMapping); + if (isDefinedAndNotNullAndNotEmpty(payload.session_id)) { + payload.session_id = transformSessionId(payload.session_id); + } let properties = {}; properties = extractCustomFields(message, properties, ['properties'], AIRSHIP_TRACK_EXCLUSION); if (!isEmptyObject(properties)) { diff --git a/src/v0/destinations/airship/utils.js b/src/v0/destinations/airship/utils.js new file mode 100644 index 0000000000..0ef637245f --- /dev/null +++ b/src/v0/destinations/airship/utils.js @@ -0,0 +1,12 @@ +const { v5 } = require('uuid'); + +// ref : https://docs.airship.com/api/ua/#operation-api-custom-events-post +const transformSessionId = (rawSessionId) => { + const NAMESPACE = v5.DNS; + const uuidV5 = v5(rawSessionId, NAMESPACE); + return uuidV5; +}; + +module.exports = { + transformSessionId, +}; diff --git a/test/integrations/destinations/airship/processor/data.ts b/test/integrations/destinations/airship/processor/data.ts index 3a6c5394cb..a72495d23d 100644 --- a/test/integrations/destinations/airship/processor/data.ts +++ b/test/integrations/destinations/airship/processor/data.ts @@ -2296,7 +2296,7 @@ export const data = [ }, { name: 'airship', - description: 'Test 22', + description: 'Test 22 : session id gets converted to v5 uuid format', feature: 'processor', module: 'destination', version: 'v0', @@ -2321,7 +2321,7 @@ export const data = [ ip: '0.0.0.0', os: { name: '', version: '' }, screen: { density: 2 }, - sessionId: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + sessionId: '1731403898', }, type: 'track', messageId: '84e26acc-56a5-4835-8233-591137fca468', @@ -2365,7 +2365,7 @@ export const data = [ user: { named_user_id: 'testuserId1' }, body: { name: 'product_clicked', - session_id: '3049dc4c-5a95-4ccd-a3e7-d74a7e411f22', + session_id: 'd5627eac-795d-5005-9bb4-2c7c0af6cab0', }, }, JSON_ARRAY: {}, From 0aeaa391b025fc68de6e3d63a6721f067c5be318 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Mon, 18 Nov 2024 13:01:26 +0530 Subject: [PATCH 20/32] fix: revert gaec changes (#3885) --- .../transform.js | 35 +--- .../processor/data.ts | 190 ------------------ .../router/data.ts | 189 ----------------- 3 files changed, 5 insertions(+), 409 deletions(-) diff --git a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js index 497d4f294f..0badf49241 100644 --- a/src/v0/destinations/google_adwords_enhanced_conversions/transform.js +++ b/src/v0/destinations/google_adwords_enhanced_conversions/transform.js @@ -2,11 +2,7 @@ const get = require('get-value'); const { cloneDeep, isNumber } = require('lodash'); -const { - InstrumentationError, - ConfigurationError, - isDefinedAndNotNull, -} = require('@rudderstack/integrations-lib'); +const { InstrumentationError, ConfigurationError } = require('@rudderstack/integrations-lib'); const isString = require('lodash/isString'); const { constructPayload, @@ -15,7 +11,6 @@ const { removeHyphens, simpleProcessRouterDest, getAccessToken, - isDefined, } = require('../../util'); const { trackMapping, BASE_ENDPOINT } = require('./config'); @@ -43,15 +38,6 @@ const responseBuilder = async (metadata, message, { Config }, payload) => { const { event } = message; const { subAccount } = Config; let { customerId, loginCustomerId } = Config; - const { configData } = Config; - - if (isDefinedAndNotNull(configData)) { - const configDetails = JSON.parse(configData); - customerId = configDetails.customerId; - if (isDefined(configDetails.loginCustomerId)) { - loginCustomerId = configDetails.loginCustomerId; - } - } if (isNumber(customerId)) { customerId = customerId.toString(); @@ -84,29 +70,18 @@ const responseBuilder = async (metadata, message, { Config }, payload) => { response.headers['login-customer-id'] = filteredLoginCustomerId; } - if (loginCustomerId) { - const filteredLoginCustomerId = removeHyphens(loginCustomerId); - response.headers['login-customer-id'] = filteredLoginCustomerId; - } - return response; }; const processTrackEvent = async (metadata, message, destination) => { - let flag = 0; + let flag = false; const { Config } = destination; const { event } = message; const { listOfConversions } = Config; - if (listOfConversions && listOfConversions.length > 0) { - if (typeof listOfConversions[0] === 'string') { - if (listOfConversions.includes(event)) { - flag = 1; - } - } else if (listOfConversions.some((i) => i.conversions === event)) { - flag = 1; - } + if (listOfConversions.some((i) => i.conversions === event)) { + flag = true; } - if (event === undefined || event === '' || flag === 0) { + if (event === undefined || event === '' || !flag) { throw new ConfigurationError( `Conversion named "${event}" was not specified in the RudderStack destination configuration`, ); diff --git a/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts b/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts index fcdb6f15ca..1d20e887e9 100644 --- a/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts +++ b/test/integrations/destinations/google_adwords_enhanced_conversions/processor/data.ts @@ -1720,194 +1720,4 @@ export const data = [ }, }, }, - { - name: 'google_adwords_enhanced_conversions', - description: 'Success test with configDetails', - feature: 'processor', - module: 'destination', - version: 'v0', - input: { - request: { - body: [ - { - metadata: { - secret: { - access_token: 'abcd1234', - refresh_token: 'efgh5678', - developer_token: 'ijkl91011', - }, - }, - destination: { - Config: { - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', - listOfConversions: ['Page View', 'Product Added'], - authStatus: 'active', - configData: '{"customerId": "1234567890", "loginCustomerId": ""}', - }, - }, - message: { - channel: 'web', - context: { - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.0.0', - }, - traits: { - phone: '912382193', - firstName: 'John', - lastName: 'Gomes', - city: 'London', - state: 'UK', - countryCode: 'us', - streetAddress: '71 Cherry Court SOUTHAMPTON SO53 5PD UK', - }, - library: { - name: 'RudderLabs JavaScript SDK', - version: '1.0.0', - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - locale: 'en-US', - ip: '0.0.0.0', - os: { - name: '', - version: '', - }, - screen: { - density: 2, - }, - }, - event: 'Page View', - type: 'track', - messageId: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', - originalTimestamp: '2019-10-14T11:15:18.299Z', - anonymousId: '00000000000000000000000000', - userId: '12345', - properties: { - gclid: 'gclid1234', - conversionDateTime: '2022-01-01 12:32:45-08:00', - adjustedValue: '10', - currency: 'INR', - adjustmentDateTime: '2022-01-01 12:32:45-08:00', - partialFailure: true, - campaignId: '1', - templateId: '0', - order_id: 10000, - total: 1000, - products: [ - { - product_id: '507f1f77bcf86cd799439011', - sku: '45790-32', - name: 'Monopoly: 3rd Edition', - price: '19', - position: '1', - category: 'cars', - url: 'https://www.example.com/product/path', - image_url: 'https://www.example.com/product/path.jpg', - quantity: '2', - }, - { - product_id: '507f1f77bcf86cd7994390112', - sku: '45790-322', - name: 'Monopoly: 3rd Edition2', - price: '192', - quantity: 22, - position: '12', - category: 'Cars2', - url: 'https://www.example.com/product/path2', - image_url: 'https://www.example.com/product/path.jpg2', - }, - ], - }, - integrations: { - All: true, - }, - name: 'ApplicationLoaded', - sentAt: '2019-10-14T11:15:53.296Z', - }, - }, - ], - }, - }, - output: { - response: { - status: 200, - body: [ - { - output: { - version: '1', - type: 'REST', - method: 'POST', - endpoint: `https://googleads.googleapis.com/${API_VERSION}/customers/1234567890:uploadConversionAdjustments`, - headers: { - Authorization: 'Bearer abcd1234', - 'Content-Type': 'application/json', - 'developer-token': 'ijkl91011', - }, - params: { - event: 'Page View', - customerId: '1234567890', - }, - body: { - JSON: { - conversionAdjustments: [ - { - gclidDateTimePair: { - gclid: 'gclid1234', - conversionDateTime: '2022-01-01 12:32:45-08:00', - }, - restatementValue: { - adjustedValue: 10, - currencyCode: 'INR', - }, - orderId: '10000', - adjustmentDateTime: '2022-01-01 12:32:45-08:00', - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - userIdentifiers: [ - { - hashedPhoneNumber: - '04387707e6cbed8c4538c81cc570ed9252d579469f36c273839b26d784e4bdbe', - }, - { - addressInfo: { - hashedFirstName: - 'a8cfcd74832004951b4408cdb0a5dbcd8c7e52d43f7fe244bf720582e05241da', - hashedLastName: - '1c574b17eefa532b6d61c963550a82d2d3dfca4a7fb69e183374cfafd5328ee4', - state: 'UK', - city: 'London', - countryCode: 'us', - hashedStreetAddress: - '9a4d2e50828448f137f119a3ebdbbbab8d6731234a67595fdbfeb2a2315dd550', - }, - }, - ], - adjustmentType: 'ENHANCEMENT', - }, - ], - partialFailure: true, - }, - JSON_ARRAY: {}, - XML: {}, - FORM: {}, - }, - files: {}, - userId: '', - }, - metadata: { - secret: { - access_token: 'abcd1234', - refresh_token: 'efgh5678', - developer_token: 'ijkl91011', - }, - }, - statusCode: 200, - }, - ], - }, - }, - }, ]; diff --git a/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts b/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts index fe0acf7964..5ac05b5a53 100644 --- a/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts +++ b/test/integrations/destinations/google_adwords_enhanced_conversions/router/data.ts @@ -1,5 +1,3 @@ -import { API_VERSION } from '../../../../../src/v0/destinations/google_adwords_enhanced_conversions/config'; - const events = [ { metadata: { @@ -413,100 +411,6 @@ const events = [ sentAt: '2019-10-14T11:15:53.296Z', }, }, - { - metadata: { - secret: { - access_token: 'abcd1234', - refresh_token: 'efgh5678', - developer_token: 'ijkl91011', - }, - jobId: 6, - userId: 'u1', - }, - destination: { - Config: { - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', - configData: '{"customerId":"1234567890", "loginCustomerId":"65656565"}', - customerId: '1234567890', - subAccount: true, - listOfConversions: [{ conversions: 'Page View' }, { conversions: 'Product Added' }], - authStatus: 'active', - }, - }, - message: { - channel: 'web', - context: { - app: { - build: '1.0.0', - name: 'RudderLabs JavaScript SDK', - namespace: 'com.rudderlabs.javascript', - version: '1.0.0', - }, - traits: { - phone: '912382193', - firstName: 'John', - lastName: 'Gomes', - city: 'London', - state: 'UK', - streetAddress: '71 Cherry Court SOUTHAMPTON SO53 5PD UK', - }, - library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - locale: 'en-US', - ip: '0.0.0.0', - os: { name: '', version: '' }, - screen: { density: 2 }, - customerID: {}, - subaccountID: 11, - }, - event: 'Page View', - type: 'track', - messageId: '5e10d13a-bf9a-44bf-b884-43a9e591ea71', - originalTimestamp: '2019-10-14T11:15:18.299Z', - anonymousId: '00000000000000000000000000', - userId: '12345', - properties: { - gclid: 'gclid1234', - conversionDateTime: '2022-01-01 12:32:45-08:00', - adjustedValue: '10', - currency: 'INR', - adjustmentDateTime: '2022-01-01 12:32:45-08:00', - partialFailure: true, - campaignId: '1', - templateId: '0', - order_id: 10000, - total: 1000, - products: [ - { - product_id: '507f1f77bcf86cd799439011', - sku: '45790-32', - name: 'Monopoly: 3rd Edition', - price: '19', - position: '1', - category: 'cars', - url: 'https://www.example.com/product/path', - image_url: 'https://www.example.com/product/path.jpg', - quantity: '2', - }, - { - product_id: '507f1f77bcf86cd7994390112', - sku: '45790-322', - name: 'Monopoly: 3rd Edition2', - price: '192', - quantity: 22, - position: '12', - category: 'Cars2', - url: 'https://www.example.com/product/path2', - image_url: 'https://www.example.com/product/path.jpg2', - }, - ], - }, - integrations: { All: true }, - name: 'ApplicationLoaded', - sentAt: '2019-10-14T11:15:53.296Z', - }, - }, { metadata: { secret: { @@ -1012,99 +916,6 @@ export const data = [ module: 'destination', }, }, - { - batched: false, - batchedRequest: { - body: { - FORM: {}, - JSON: { - conversionAdjustments: [ - { - adjustmentDateTime: '2022-01-01 12:32:45-08:00', - adjustmentType: 'ENHANCEMENT', - gclidDateTimePair: { - conversionDateTime: '2022-01-01 12:32:45-08:00', - gclid: 'gclid1234', - }, - orderId: '10000', - restatementValue: { - adjustedValue: 10, - currencyCode: 'INR', - }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', - userIdentifiers: [ - { - hashedPhoneNumber: - '04387707e6cbed8c4538c81cc570ed9252d579469f36c273839b26d784e4bdbe', - }, - { - addressInfo: { - city: 'London', - hashedFirstName: - 'a8cfcd74832004951b4408cdb0a5dbcd8c7e52d43f7fe244bf720582e05241da', - hashedLastName: - '1c574b17eefa532b6d61c963550a82d2d3dfca4a7fb69e183374cfafd5328ee4', - hashedStreetAddress: - '9a4d2e50828448f137f119a3ebdbbbab8d6731234a67595fdbfeb2a2315dd550', - state: 'UK', - }, - }, - ], - }, - ], - partialFailure: true, - }, - JSON_ARRAY: {}, - XML: {}, - }, - endpoint: - 'https://googleads.googleapis.com/v17/customers/1234567890:uploadConversionAdjustments', - files: {}, - headers: { - Authorization: 'Bearer abcd1234', - 'Content-Type': 'application/json', - 'developer-token': 'ijkl91011', - 'login-customer-id': '65656565', - }, - method: 'POST', - params: { - customerId: '1234567890', - event: 'Page View', - }, - type: 'REST', - version: '1', - }, - destination: { - Config: { - authStatus: 'active', - configData: '{"customerId":"1234567890", "loginCustomerId":"65656565"}', - customerId: '1234567890', - listOfConversions: [ - { - conversions: 'Page View', - }, - { - conversions: 'Product Added', - }, - ], - rudderAccountId: '25u5whFH7gVTnCiAjn4ykoCLGoC', - subAccount: true, - }, - }, - metadata: [ - { - jobId: 6, - secret: { - access_token: 'abcd1234', - developer_token: 'ijkl91011', - refresh_token: 'efgh5678', - }, - userId: 'u1', - }, - ], - statusCode: 200, - }, { batched: false, destination: { From f3ff4092d455508dd3354ffb22d345fa97f4d1f2 Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Mon, 18 Nov 2024 13:52:34 +0530 Subject: [PATCH 21/32] feat: onboard linkedin audience destination (#3857) * feat: onboard linkedin audience destination * chore: resolve conflicts * chore: add test cases * chore: fix lint errors * refactor: linkedin audiences proc workflow --------- Co-authored-by: Dilip Kola --- .../destinations/linkedin_audience/config.ts | 10 + .../linkedin_audience/procWorkflow.yaml | 89 +++ .../linkedin_audience/rtWorkflow.yaml | 40 ++ .../destinations/linkedin_audience/utils.ts | 87 +++ src/features.ts | 1 + .../linkedin_audience/processor/business.ts | 520 ++++++++++++++++++ .../linkedin_audience/processor/data.ts | 3 + .../linkedin_audience/processor/validation.ts | 396 +++++++++++++ .../linkedin_audience/router/data.ts | 384 +++++++++++++ test/integrations/testUtils.ts | 24 + 10 files changed, 1554 insertions(+) create mode 100644 src/cdk/v2/destinations/linkedin_audience/config.ts create mode 100644 src/cdk/v2/destinations/linkedin_audience/procWorkflow.yaml create mode 100644 src/cdk/v2/destinations/linkedin_audience/rtWorkflow.yaml create mode 100644 src/cdk/v2/destinations/linkedin_audience/utils.ts create mode 100644 test/integrations/destinations/linkedin_audience/processor/business.ts create mode 100644 test/integrations/destinations/linkedin_audience/processor/data.ts create mode 100644 test/integrations/destinations/linkedin_audience/processor/validation.ts create mode 100644 test/integrations/destinations/linkedin_audience/router/data.ts diff --git a/src/cdk/v2/destinations/linkedin_audience/config.ts b/src/cdk/v2/destinations/linkedin_audience/config.ts new file mode 100644 index 0000000000..86ea94425a --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_audience/config.ts @@ -0,0 +1,10 @@ +export const SUPPORTED_EVENT_TYPE = 'record'; +export const ACTION_TYPES = ['insert', 'delete']; +export const BASE_ENDPOINT = 'https://api.linkedin.com/rest'; +export const USER_ENDPOINT = '/dmpSegments/audienceId/users'; +export const COMPANY_ENDPOINT = '/dmpSegments/audienceId/companies'; +export const FIELD_MAP = { + sha256Email: 'SHA256_EMAIL', + sha512Email: 'SHA512_EMAIL', + googleAid: 'GOOGLE_AID', +}; diff --git a/src/cdk/v2/destinations/linkedin_audience/procWorkflow.yaml b/src/cdk/v2/destinations/linkedin_audience/procWorkflow.yaml new file mode 100644 index 0000000000..f3f4ce0772 --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_audience/procWorkflow.yaml @@ -0,0 +1,89 @@ +bindings: + - path: ./config + exportAll: true + - path: ./utils + exportAll: true + - name: defaultRequestConfig + path: ../../../../v0/util + +steps: + - name: validateInput + description: Validate input, if all the required fields are available or not. + template: | + const config = .connection.config.destination; + const secret = .metadata.secret; + let messageType = .message.type; + $.assertConfig(config.audienceId, "Audience Id is not present. Aborting"); + $.assertConfig(secret.accessToken, "Access Token is not present. Aborting"); + $.assertConfig(config.audienceType, "audienceType is not present. Aborting"); + $.assert(messageType, "Message Type is not present. Aborting message."); + $.assert(messageType.toLowerCase() === $.SUPPORTED_EVENT_TYPE, `Event type ${.message.type.toLowerCase()} is not supported. Aborting message.`); + $.assert(.message.fields, "`fields` is not present. Aborting message."); + $.assert(.message.identifiers, "`identifiers` is not present inside properties. Aborting message."); + $.assert($.containsAll([.message.action], $.ACTION_TYPES), "Unsupported action type. Aborting message.") + + - name: getConfigs + description: This step fetches the configs from different places and combines them. + template: | + const config = .connection.config.destination; + { + audienceType: config.audienceType, + audienceId: config.audienceId, + accessToken: .metadata.secret.accessToken, + isHashRequired: config.isHashRequired, + } + + - name: prepareUserTypeBasePayload + condition: $.outputs.getConfigs.audienceType === 'user' + steps: + - name: prepareUserIds + description: Prepare user ids for user audience type + template: | + const identifiers = $.outputs.getConfigs.isHashRequired === true ? + $.hashIdentifiers(.message.identifiers) : + .message.identifiers; + $.prepareUserIds(identifiers) + + - name: preparePayload + description: Prepare base payload for user audiences + template: | + const payload = { + 'elements': [ + { + 'action': $.generateActionType(.message.action), + 'userIds': $.outputs.prepareUserTypeBasePayload.prepareUserIds, + ....message.fields + } + ] + } + payload; + + - name: prepareCompanyTypeBasePayload + description: Prepare base payload for company audiences + condition: $.outputs.getConfigs.audienceType === 'company' + template: | + const payload = { + 'elements': [ + { + 'action': $.generateActionType(.message.action), + ....message.identifiers, + ....message.fields + } + ] + } + payload; + + - name: buildResponseForProcessTransformation + description: build response depending upon batch size + template: | + const response = $.defaultRequestConfig(); + response.body.JSON = {...$.outputs.prepareUserTypeBasePayload, ...$.outputs.prepareCompanyTypeBasePayload}; + response.endpoint = $.generateEndpoint($.outputs.getConfigs.audienceType, $.outputs.getConfigs.audienceId); + response.headers = { + "Authorization": "Bearer " + $.outputs.getConfigs.accessToken, + "Content-Type": "application/json", + "X-RestLi-Method": "BATCH_CREATE", + "X-Restli-Protocol-Version": "2.0.0", + "LinkedIn-Version": "202409" + }; + response; diff --git a/src/cdk/v2/destinations/linkedin_audience/rtWorkflow.yaml b/src/cdk/v2/destinations/linkedin_audience/rtWorkflow.yaml new file mode 100644 index 0000000000..fe16ab786a --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_audience/rtWorkflow.yaml @@ -0,0 +1,40 @@ +bindings: + - path: ./utils + - name: handleRtTfSingleEventError + path: ../../../../v0/util/index + +steps: + - name: validateInput + template: | + $.assert(Array.isArray(^) && ^.length > 0, "Invalid event array") + + - name: transform + externalWorkflow: + path: ./procWorkflow.yaml + bindings: + - name: batchMode + value: true + loopOverInput: true + + - name: successfulEvents + template: | + $.outputs.transform#idx.output.({ + "message": .[], + "destination": ^ [idx].destination, + "metadata": ^ [idx].metadata + })[] + + - name: failedEvents + template: | + $.outputs.transform#idx.error.( + $.handleRtTfSingleEventError(^[idx], .originalError ?? ., {}) + )[] + + - name: batchSuccessfulEvents + description: Batches the successfulEvents + template: | + $.batchResponseBuilder($.outputs.successfulEvents); + + - name: finalPayload + template: | + [...$.outputs.batchSuccessfulEvents, ...$.outputs.failedEvents] diff --git a/src/cdk/v2/destinations/linkedin_audience/utils.ts b/src/cdk/v2/destinations/linkedin_audience/utils.ts new file mode 100644 index 0000000000..12f5a0572b --- /dev/null +++ b/src/cdk/v2/destinations/linkedin_audience/utils.ts @@ -0,0 +1,87 @@ +import lodash from 'lodash'; +import { hashToSha256 } from '@rudderstack/integrations-lib'; +import { createHash } from 'crypto'; +import { BASE_ENDPOINT, COMPANY_ENDPOINT, FIELD_MAP, USER_ENDPOINT } from './config'; + +export function hashIdentifiers(identifiers: string[]): Record { + const hashedIdentifiers = {}; + Object.keys(identifiers).forEach((key) => { + if (key === 'sha256Email') { + hashedIdentifiers[key] = hashToSha256(identifiers[key]); + } else if (key === 'sha512Email') { + hashedIdentifiers[key] = createHash('sha512').update(identifiers[key]).digest('hex'); + } else { + hashedIdentifiers[key] = identifiers[key]; + } + }); + return hashedIdentifiers; +} + +export function prepareUserIds( + identifiers: Record, +): { idType: string; idValue: string }[] { + const userIds: { idType: string; idValue: string }[] = []; + Object.keys(identifiers).forEach((key) => { + userIds.push({ idType: FIELD_MAP[key], idValue: identifiers[key] }); + }); + return userIds; +} + +export function generateEndpoint(audienceType: string, audienceId: string) { + if (audienceType === 'user') { + return BASE_ENDPOINT + USER_ENDPOINT.replace('audienceId', audienceId); + } + return BASE_ENDPOINT + COMPANY_ENDPOINT.replace('audienceId', audienceId); +} + +export function batchResponseBuilder(successfulEvents) { + const chunkOnActionType = lodash.groupBy( + successfulEvents, + (event) => event.message[0].body.JSON.elements[0].action, + ); + const result: any = []; + Object.keys(chunkOnActionType).forEach((actionType) => { + const firstEvent = chunkOnActionType[actionType][0]; + const { method, endpoint, headers, type, version } = firstEvent.message[0]; + const batchEvent = { + batchedRequest: { + body: { + JSON: { elements: firstEvent.message[0].body.JSON.elements }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + version, + type, + method, + endpoint, + headers, + params: {}, + files: {}, + }, + metadata: [firstEvent.metadata], + batched: true, + statusCode: 200, + destination: firstEvent.destination, + }; + firstEvent.metadata = [firstEvent.metadata]; + chunkOnActionType[actionType].forEach((element, index) => { + if (index !== 0) { + batchEvent.batchedRequest.body.JSON.elements.push(element.message[0].body.JSON.elements[0]); + batchEvent.metadata.push(element.metadata); + } + }); + result.push(batchEvent); + }); + return result; +} + +export const generateActionType = (actionType: string): string => { + if (actionType === 'insert') { + return 'ADD'; + } + if (actionType === 'delete') { + return 'REMOVE'; + } + return actionType; +}; diff --git a/src/features.ts b/src/features.ts index 9f60d44483..4ff419a7fe 100644 --- a/src/features.ts +++ b/src/features.ts @@ -92,6 +92,7 @@ const defaultFeaturesConfig: FeaturesConfig = { HTTP: true, AMAZON_AUDIENCE: true, INTERCOM_V2: true, + LINKEDIN_AUDIENCE: true, }, regulations: [ 'BRAZE', diff --git a/test/integrations/destinations/linkedin_audience/processor/business.ts b/test/integrations/destinations/linkedin_audience/processor/business.ts new file mode 100644 index 0000000000..28cb6a9a97 --- /dev/null +++ b/test/integrations/destinations/linkedin_audience/processor/business.ts @@ -0,0 +1,520 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, generateRecordPayload } from '../../../testUtils'; + +export const businessTestData: ProcessorTestData[] = [ + { + id: 'linkedin_audience-business-test-1', + name: 'linkedin_audience', + description: 'Record call : non string values provided as email', + scenario: 'Business', + successCriteria: 'should fail with 400 status code and error message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 12345, + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'The "string" argument must be of type string. Received type number (12345): Workflow: procWorkflow, Step: prepareUserTypeBasePayload, ChildStep: prepareUserIds, OriginalError: The "string" argument must be of type string. Received type number (12345)', + metadata: generateMetadata(1), + statTags: { + destType: 'LINKEDIN_AUDIENCE', + destinationId: 'default-destinationId', + errorCategory: 'transformation', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + statusCode: 500, + }, + ], + }, + }, + }, + { + id: 'linkedin_audience-business-test-2', + name: 'linkedin_audience', + description: 'Record call : Valid event without any field mappings', + scenario: 'Business', + successCriteria: 'should pass with 200 status code and transformed message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: {}, + identifiers: { + sha256Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: generateMetadata(1), + output: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/32589526/users', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + id: 'linkedin_audience-business-test-2', + name: 'linkedin_audience', + description: 'Record call : customer provided hashed value and isHashRequired is false', + scenario: 'Business', + successCriteria: 'should pass with 200 status code and transformed message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: {}, + identifiers: { + sha256Email: '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + sha512Email: + '631372c5eafe80f3fe1b5d067f6a1870f1f04a0f0c0d9298eeaa20b9e54224da9588e3164d2ec6e2a5545a5299ed7df563e4a60315e6782dfa7db4de6b1c5326', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: false, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: generateMetadata(1), + output: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + }, + { + idType: 'SHA512_EMAIL', + idValue: + '631372c5eafe80f3fe1b5d067f6a1870f1f04a0f0c0d9298eeaa20b9e54224da9588e3164d2ec6e2a5545a5299ed7df563e4a60315e6782dfa7db4de6b1c5326', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/32589526/users', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + id: 'linkedin_audience-business-test-2', + name: 'linkedin_audience', + description: 'Record call : event with company audience details', + scenario: 'Business', + successCriteria: 'should pass with 200 status code and transformed message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: { + city: 'Dhaka', + state: 'Dhaka', + industries: 'Information Technology', + postalCode: '123456', + }, + identifiers: { + companyName: 'Rudderstack', + organizationUrn: 'urn:li:organization:456', + companyWebsiteDomain: 'rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'company', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'city', + to: 'city', + }, + { + from: 'state', + to: 'state', + }, + { + from: 'domain', + to: 'industries', + }, + { + from: 'psCode', + to: 'postalCode', + }, + ], + identifierMappings: [ + { + from: 'name', + to: 'companyName', + }, + { + from: 'urn', + to: 'organizationUrn', + }, + { + from: 'Website Domain', + to: 'companyWebsiteDomain', + }, + ], + isHashRequired: false, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: generateMetadata(1), + output: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + city: 'Dhaka', + companyName: 'Rudderstack', + companyWebsiteDomain: 'rudderstack.com', + industries: 'Information Technology', + organizationUrn: 'urn:li:organization:456', + postalCode: '123456', + state: 'Dhaka', + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/32589526/companies', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/linkedin_audience/processor/data.ts b/test/integrations/destinations/linkedin_audience/processor/data.ts new file mode 100644 index 0000000000..233a9dcf86 --- /dev/null +++ b/test/integrations/destinations/linkedin_audience/processor/data.ts @@ -0,0 +1,3 @@ +import { businessTestData } from './business'; +import { validationTestData } from './validation'; +export const data = [...validationTestData, ...businessTestData]; diff --git a/test/integrations/destinations/linkedin_audience/processor/validation.ts b/test/integrations/destinations/linkedin_audience/processor/validation.ts new file mode 100644 index 0000000000..3ad37b2f4d --- /dev/null +++ b/test/integrations/destinations/linkedin_audience/processor/validation.ts @@ -0,0 +1,396 @@ +import { ProcessorTestData } from '../../../testTypes'; +import { generateMetadata, generateRecordPayload } from '../../../testUtils'; + +export const validationTestData: ProcessorTestData[] = [ + { + id: 'linkedin_audience-validation-test-1', + name: 'linkedin_audience', + description: 'Record call : event is valid with all required elements', + scenario: 'Validation', + successCriteria: 'should pass with 200 status code and transformed message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 'random@rudderstack.com', + sha512Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: generateMetadata(1), + output: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + company: 'Rudderlabs', + country: 'Dhaka', + firstName: 'Test', + lastName: 'User', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + }, + { + idType: 'SHA512_EMAIL', + idValue: + '631372c5eafe80f3fe1b5d067f6a1870f1f04a0f0c0d9298eeaa20b9e54224da9588e3164d2ec6e2a5545a5299ed7df563e4a60315e6782dfa7db4de6b1c5326', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/32589526/users', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + id: 'linkedin_audience-validation-test-2', + name: 'linkedin_audience', + description: 'Record call : event is not valid with all required elements', + scenario: 'Validation', + successCriteria: 'should fail with 400 status code and error message', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 'random@rudderstack.com', + sha512Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Audience Id is not present. Aborting: Workflow: procWorkflow, Step: validateInput, ChildStep: undefined, OriginalError: Audience Id is not present. Aborting', + metadata: generateMetadata(1), + statTags: { + destType: 'LINKEDIN_AUDIENCE', + destinationId: 'default-destinationId', + errorCategory: 'dataValidation', + errorType: 'configuration', + feature: 'processor', + implementation: 'cdkV2', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + statusCode: 400, + }, + ], + }, + }, + }, + { + id: 'linkedin_audience-validation-test-3', + name: 'linkedin_audience', + description: 'Record call : isHashRequired is not provided', + scenario: 'Validation', + successCriteria: + 'should succeed with 200 status code and transformed message with provided values of identifiers', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 'random@rudderstack.com', + sha512Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 1234, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + }, + source: {}, + }, + }, + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + metadata: generateMetadata(1), + output: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + company: 'Rudderlabs', + country: 'Dhaka', + firstName: 'Test', + lastName: 'User', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: 'random@rudderstack.com', + }, + { + idType: 'SHA512_EMAIL', + idValue: 'random@rudderstack.com', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/1234/users', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + userId: '', + version: '1', + }, + statusCode: 200, + }, + ], + }, + }, + }, +]; diff --git a/test/integrations/destinations/linkedin_audience/router/data.ts b/test/integrations/destinations/linkedin_audience/router/data.ts new file mode 100644 index 0000000000..c76d3e84c6 --- /dev/null +++ b/test/integrations/destinations/linkedin_audience/router/data.ts @@ -0,0 +1,384 @@ +import { generateMetadata, generateRecordPayload } from '../../../testUtils'; + +export const data = [ + { + name: 'linkedin_audience', + description: 'Test 0', + feature: 'router', + module: 'destination', + version: 'v0', + input: { + request: { + body: { + input: [ + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 'random@rudderstack.com', + sha512Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(1), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + { + message: generateRecordPayload({ + fields: {}, + identifiers: { + sha256Email: 'random@rudderstack.com', + }, + action: 'insert', + }), + metadata: generateMetadata(2), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + { + message: generateRecordPayload({ + fields: { + firstName: 'Test', + lastName: 'User', + country: 'Dhaka', + company: 'Rudderlabs', + }, + identifiers: { + sha256Email: 12345, + }, + action: 'insert', + }), + metadata: generateMetadata(3), + destination: { + ID: '123', + Name: 'Linkedin Audience', + DestinationDefinition: { + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + DisplayName: 'Linkedin Audience', + Config: { + cdkV2Enabled: true, + }, + }, + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + Enabled: true, + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + Transformations: [], + }, + connection: { + sourceId: 'randomSourceId', + destinationId: 'randomDestinationId', + enabled: true, + config: { + destination: { + accountId: 512315509, + audienceId: 32589526, + audienceType: 'user', + createAudience: 'no', + eventType: 'record', + fieldMappings: [ + { + from: 'name', + to: 'firstName', + }, + { + from: 'name', + to: 'lastName', + }, + ], + identifierMappings: [ + { + from: 'email', + to: 'sha256Email', + }, + { + from: 'email', + to: 'sha512Email', + }, + ], + isHashRequired: true, + }, + source: {}, + }, + }, + }, + ], + destType: 'linkedin_audience', + }, + }, + }, + output: { + response: { + status: 200, + body: { + output: [ + { + batched: true, + batchedRequest: { + body: { + FORM: {}, + JSON: { + elements: [ + { + action: 'ADD', + company: 'Rudderlabs', + country: 'Dhaka', + firstName: 'Test', + lastName: 'User', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + }, + { + idType: 'SHA512_EMAIL', + idValue: + '631372c5eafe80f3fe1b5d067f6a1870f1f04a0f0c0d9298eeaa20b9e54224da9588e3164d2ec6e2a5545a5299ed7df563e4a60315e6782dfa7db4de6b1c5326', + }, + ], + }, + { + action: 'ADD', + userIds: [ + { + idType: 'SHA256_EMAIL', + idValue: + '52ac4b9ef8f745e007c19fac81ddb0a3f50b20029f6699ca1406225fc217f392', + }, + ], + }, + ], + }, + JSON_ARRAY: {}, + XML: {}, + }, + endpoint: 'https://api.linkedin.com/rest/dmpSegments/32589526/users', + files: {}, + headers: { + Authorization: 'Bearer default-accessToken', + 'Content-Type': 'application/json', + 'LinkedIn-Version': '202409', + 'X-RestLi-Method': 'BATCH_CREATE', + 'X-Restli-Protocol-Version': '2.0.0', + }, + method: 'POST', + params: {}, + type: 'REST', + version: '1', + }, + destination: { + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + DisplayName: 'Linkedin Audience', + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + }, + Enabled: true, + ID: '123', + Name: 'Linkedin Audience', + Transformations: [], + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + }, + metadata: [ + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + jobId: 1, + secret: { + accessToken: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + }, + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + jobId: 2, + secret: { + accessToken: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + }, + ], + statusCode: 200, + }, + { + batched: false, + destination: { + Config: { + connectionMode: 'cloud', + rudderAccountId: '2nmIV6FMXvyyqRM9Ifj8V92yElu', + }, + DestinationDefinition: { + Config: { + cdkV2Enabled: true, + }, + DisplayName: 'Linkedin Audience', + ID: '2njmJIfG6JH3guvFHSjLQNiIYh5', + Name: 'LINKEDIN_AUDIENCE', + }, + Enabled: true, + ID: '123', + Name: 'Linkedin Audience', + Transformations: [], + WorkspaceID: '2lepjs3uWK6ac2WLukJjOrbcTfC', + }, + error: 'The "string" argument must be of type string. Received type number (12345)', + metadata: [ + { + attemptNum: 1, + destinationId: 'default-destinationId', + dontBatch: false, + jobId: 3, + secret: { + accessToken: 'default-accessToken', + }, + sourceId: 'default-sourceId', + userId: 'default-userId', + workspaceId: 'default-workspaceId', + }, + ], + statTags: { + destType: 'LINKEDIN_AUDIENCE', + destinationId: 'default-destinationId', + errorCategory: 'transformation', + feature: 'router', + implementation: 'cdkV2', + module: 'destination', + workspaceId: 'default-workspaceId', + }, + statusCode: 500, + }, + ], + }, + }, + }, + }, +]; diff --git a/test/integrations/testUtils.ts b/test/integrations/testUtils.ts index 4eda20a901..7e6e6b9acb 100644 --- a/test/integrations/testUtils.ts +++ b/test/integrations/testUtils.ts @@ -237,6 +237,30 @@ export const generateTrackPayload: any = (parametersOverride: any) => { return removeUndefinedAndNullValues(payload); }; +export const generateRecordPayload: any = (parametersOverride: any) => { + const payload = { + type: 'record', + action: parametersOverride.action || 'insert', + fields: parametersOverride.fields || {}, + channel: 'sources', + context: { + sources: { + job_id: 'randomJobId', + version: 'local', + job_run_id: 'jobRunId', + task_run_id: 'taskRunId', + }, + }, + recordId: '3', + rudderId: 'randomRudderId', + messageId: 'randomMessageId', + receivedAt: '2024-11-08T10:30:41.618+05:30', + request_ip: '[::1]', + identifiers: parametersOverride.identifiers || {}, + }; + return removeUndefinedAndNullValues(payload); +}; + export const generateSimplifiedTrackPayload: any = (parametersOverride: any) => { return removeUndefinedAndNullValues({ type: 'track', From d57f48e989d18d469bea0de94293bc685300945b Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Mon, 18 Nov 2024 13:58:49 +0530 Subject: [PATCH 22/32] fix: handling invalid timestamp for adjust source (#3866) * fix: handling invalid timestamp for adjust source * fix: small change in logic * fix: unnecessary line removal * fix: unnecessary line removal * fix: review comment addressed * fix: review comment addressed * chore: add edge testcase - update error messages * fix: testcase error message --------- Co-authored-by: Sai Sankeerth --- src/v0/sources/adjust/transform.js | 4 +- src/v0/sources/adjust/utils.js | 27 ++++++++++++ src/v0/sources/adjust/utils.test.js | 37 +++++++++++++++++ test/integrations/sources/adjust/data.ts | 53 ++++++++++++++++++++++++ 4 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 src/v0/sources/adjust/utils.js create mode 100644 src/v0/sources/adjust/utils.test.js diff --git a/src/v0/sources/adjust/transform.js b/src/v0/sources/adjust/transform.js index 9da90751b7..f68e87d476 100644 --- a/src/v0/sources/adjust/transform.js +++ b/src/v0/sources/adjust/transform.js @@ -7,6 +7,7 @@ const Message = require('../message'); const { CommonUtils } = require('../../../util/common'); const { excludedFieldList } = require('./config'); const { extractCustomFields, generateUUID } = require('../../util'); +const { convertToISODate } = require('./utils'); // ref : https://help.adjust.com/en/article/global-callbacks#general-recommended-placeholders // import mapping json using JSON.parse to preserve object key order @@ -43,11 +44,10 @@ const processEvent = (inputEvent) => { message.properties = { ...message.properties, ...customProperties }; if (formattedPayload.created_at) { - const ts = new Date(formattedPayload.created_at * 1000).toISOString(); + const ts = convertToISODate(formattedPayload.created_at); message.setProperty('originalTimestamp', ts); message.setProperty('timestamp', ts); } - // adjust does not has the concept of user but we need to set some random anonymousId in order to make the server accept the message message.anonymousId = generateUUID(); return message; diff --git a/src/v0/sources/adjust/utils.js b/src/v0/sources/adjust/utils.js new file mode 100644 index 0000000000..73ec696e34 --- /dev/null +++ b/src/v0/sources/adjust/utils.js @@ -0,0 +1,27 @@ +const { TransformationError } = require('@rudderstack/integrations-lib'); + +const convertToISODate = (rawTimestamp) => { + if (typeof rawTimestamp !== 'number' && typeof rawTimestamp !== 'string') { + throw new TransformationError( + `Invalid timestamp type: expected number or string, received ${typeof rawTimestamp}`, + ); + } + + const createdAt = Number(rawTimestamp); + + if (Number.isNaN(createdAt)) { + throw new TransformationError(`Failed to parse timestamp: "${rawTimestamp}"`); + } + + const date = new Date(createdAt * 1000); + + if (Number.isNaN(date.getTime())) { + throw new TransformationError(`Failed to create valid date for timestamp "${rawTimestamp}"`); + } + + return date.toISOString(); +}; + +module.exports = { + convertToISODate, +}; diff --git a/src/v0/sources/adjust/utils.test.js b/src/v0/sources/adjust/utils.test.js new file mode 100644 index 0000000000..f5a0caa832 --- /dev/null +++ b/src/v0/sources/adjust/utils.test.js @@ -0,0 +1,37 @@ +const { convertToISODate } = require('./utils'); +const { TransformationError } = require('@rudderstack/integrations-lib'); + +describe('convertToISODate', () => { + // Converts valid numeric timestamp to ISO date string + it('should return ISO date string when given a valid numeric timestamp', () => { + const timestamp = 1633072800; // Example timestamp for 2021-10-01T00:00:00.000Z + const result = convertToISODate(timestamp); + expect(result).toBe('2021-10-01T07:20:00.000Z'); + }); + + // Throws error for non-numeric string input + it('should throw TransformationError when given a non-numeric string', () => { + const invalidTimestamp = 'invalid'; + expect(() => convertToISODate(invalidTimestamp)).toThrow(TransformationError); + }); + + // Converts valid numeric string timestamp to ISO date string + it('should convert valid numeric string timestamp to ISO date string', () => { + const rawTimestamp = '1633072800'; // Corresponds to 2021-10-01T00:00:00.000Z + const result = convertToISODate(rawTimestamp); + expect(result).toBe('2021-10-01T07:20:00.000Z'); + }); + + // Throws error for non-number and non-string input + it('should throw error for non-number and non-string input', () => { + expect(() => convertToISODate({})).toThrow(TransformationError); + expect(() => convertToISODate([])).toThrow(TransformationError); + expect(() => convertToISODate(null)).toThrow(TransformationError); + expect(() => convertToISODate(undefined)).toThrow(TransformationError); + }); + + it('should throw error for timestamp that results in invalid date when multiplied', () => { + const hugeTimestamp = 999999999999999; // This will become invalid when multiplied by 1000 + expect(() => convertToISODate(hugeTimestamp)).toThrow(TransformationError); + }); +}); diff --git a/test/integrations/sources/adjust/data.ts b/test/integrations/sources/adjust/data.ts index e57feb45d4..107bb444c4 100644 --- a/test/integrations/sources/adjust/data.ts +++ b/test/integrations/sources/adjust/data.ts @@ -125,4 +125,57 @@ export const data = [ defaultMockFns(); }, }, + { + name: 'adjust', + description: 'Simple track call with wrong created at', + module: 'source', + version: 'v0', + skipGo: 'FIXME', + input: { + request: { + body: [ + { + id: 'adjust', + query_parameters: { + gps_adid: ['38400000-8cf0-11bd-b23e-10b96e40000d'], + adid: ['18546f6171f67e29d1cb983322ad1329'], + tracker_token: ['abc'], + custom: ['custom'], + tracker_name: ['dummy'], + created_at: ['test'], + event_name: ['Click'], + }, + updated_at: '2023-02-10T12:16:07.251Z', + created_at: 'test', + }, + ], + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + error: 'Failed to parse timestamp: "test"', + statTags: { + destinationId: 'Non determinable', + errorCategory: 'transformation', + implementation: 'native', + module: 'source', + workspaceId: 'Non determinable', + }, + statusCode: 400, + }, + ], + }, + }, + mockFns: () => { + defaultMockFns(); + }, + }, ]; From 85202781de3464bd46fe910159d2b143cd4209e8 Mon Sep 17 00:00:00 2001 From: Sudip Paul <67197965+ItsSudip@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:03:33 +0530 Subject: [PATCH 23/32] feat: update pinterest_tag single product events with new mapping (#3858) * feat: update pinterest_tag single product events with new mapping * chore: use toString function from integrations-lib * chore: update integrations-lib library --- package-lock.json | 9 +++-- package.json | 2 +- .../pinterest_tag/procWorkflow.yaml | 40 +++++++++++-------- .../data_scenarios/cdk_v2/failure.json | 8 ++++ .../data_scenarios/cdk_v2/success.json | 12 ++++++ .../proc/batch_input_multiplex.json | 12 ++++++ .../proc/multiplex_partial_failure.json | 8 ++++ .../destination/proc/multiplex_success.json | 8 ++++ .../destination/router/failure_test.json | 8 ++++ .../pinterest_tag/processor/data.ts | 11 +++-- .../destinations/pinterest_tag/router/data.ts | 2 +- .../destinations/pinterest_tag/step/data.ts | 7 ++-- 12 files changed, 94 insertions(+), 33 deletions(-) diff --git a/package-lock.json b/package-lock.json index dd490e88c6..d10997f143 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "@koa/router": "^12.0.0", "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", - "@rudderstack/integrations-lib": "^0.2.12", + "@rudderstack/integrations-lib": "^0.2.13", "@rudderstack/json-template-engine": "^0.18.0", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", @@ -6602,9 +6602,10 @@ } }, "node_modules/@rudderstack/integrations-lib": { - "version": "0.2.12", - "resolved": "https://registry.npmjs.org/@rudderstack/integrations-lib/-/integrations-lib-0.2.12.tgz", - "integrity": "sha512-xy+T9SHFkSeVDd4svGOyrTtIGljZ/l4qUh5o5EQWk3dTStzaV9mKnbXLsG62kEO3aTmCVg+VYr4OPwZY2+6rxQ==", + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/@rudderstack/integrations-lib/-/integrations-lib-0.2.13.tgz", + "integrity": "sha512-MBI+OQpnYAuOzRlbGCnUX6oVfQsYA7daZ8z07WmqQYQtWFOfd2yFbaxKclu+R/a8W7+jBo4gvbW+ScEW6h+Mgg==", + "license": "MIT", "dependencies": { "axios": "^1.4.0", "axios-mock-adapter": "^1.22.0", diff --git a/package.json b/package.json index 75a618574c..92ac9f0fb9 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "@koa/router": "^12.0.0", "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", - "@rudderstack/integrations-lib": "^0.2.12", + "@rudderstack/integrations-lib": "^0.2.13", "@rudderstack/json-template-engine": "^0.18.0", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", diff --git a/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml b/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml index 64d391c888..aebb7b0667 100644 --- a/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml +++ b/src/cdk/v2/destinations/pinterest_tag/procWorkflow.yaml @@ -11,6 +11,8 @@ bindings: path: ../../../../v0/util - name: validateEventName path: ../../../../v0/util + - path: '@rudderstack/integrations-lib' + steps: - name: checkIfProcessed condition: .message.statusCode @@ -67,9 +69,9 @@ steps: "event_id": $.getOneByPaths(., ^.destination.Config.deduplicationKey) ?? .messageId, "app_id": ^.destination.Config.appId, "advertiser_id": ^.destination.Config.advertiserId, - "partner_name": .properties.partnerName, - "device_carrier": .context.network.carrier, - "wifi": .context.network.wifi + "partner_name": .properties.partnerName ? $.convertToString(.properties.partnerName) : undefined, + "device_carrier": .properties.partnerName ? $.convertToString(.context.network.carrier) : undefined, + "wifi": .context.network.wifi ? Boolean(.context.network.wifi) : undefined }); $.outputs.apiVersion === {{$.API_VERSION.v5}} ? commonFields = commonFields{~["advertiser_id"]}; $.removeUndefinedValues(commonFields) @@ -107,7 +109,7 @@ steps: "client_user_agent": .context.userAgent, "external_id": {{{{$.getGenericPaths("userId")}}}}, "click_id": .properties.clickId, - "partner_id": .traits.partnerId ?? .context.traits.partnerId + "partner_id": .traits.partnerId ?? .context.traits.partnerId ? $.convertToString(.traits.partnerId ?? .context.traits.partnerId) : undefined }); !.destination.Config.sendExternalId ? userFields = userFields{~["external_id"]} : null; userFields = $.removeUndefinedAndNullAndEmptyValues(userFields); @@ -127,17 +129,17 @@ steps: template: | const customFields = .message.().({ "currency": .properties.currency, - "value": .properties.value !== undefined ? String(.properties.value) : - .properties.total !== undefined ? String(.properties.total) : - .properties.revenue !== undefined ? String(.properties.revenue) : undefined, + "value": .properties.value !== undefined ? $.convertToString(.properties.value) : + .properties.total !== undefined ? $.convertToString(.properties.total) : + .properties.revenue !== undefined ? $.convertToString(.properties.revenue) : undefined, "num_items": .properties.numOfItems && Number(.properties.numOfItems), "order_id": .properties.order_id, "search_string": .properties.query, "opt_out_type": .properties.optOutType, - "content_name": .properties.contentName, - "content_category": .properties.contentCategory, - "content_brand": .properties.contentBrand, - "np": .properties.np + "content_name": .properties.contentName ? $.convertToString(.properties.contentName) : undefined, + "content_category": .properties.contentCategory ? $.convertToString(.properties.contentCategory) : undefined, + "content_brand": .properties.contentBrand ? $.convertToString(.properties.contentBrand) : undefined, + "np": .properties.np ? $.convertToString(.properties.np) : undefined }); $.removeUndefinedValues(customFields) @@ -151,11 +153,11 @@ steps: "content_ids": products.(.product_id ?? .sku ?? .id)[], "contents": .message.properties@prop.products.({ "quantity": Number(.quantity ?? prop.quantity ?? 1), - "item_price": String(.price ?? prop.price), - "item_name": String(.name), - "id": .product_id ?? .sku, - "item_category": .category, - "item_brand": .brand + "item_price": $.convertToString(.price ?? prop.price), + "item_name": $.convertToString(.name), + "id": .product_id ?? .sku ? $.convertToString(.product_id ?? .sku) : undefined, + "item_category": .category ? $.convertToString(.category) : undefined, + "item_brand": .brand ? $.convertToString(.brand) : undefined })[] } else: @@ -167,7 +169,11 @@ steps: "content_ids": (props.product_id ?? props.sku ?? props.id)[], "contents": { "quantity": Number(props.quantity) || 1, - "item_price": String(props.price) + "item_price": props.price ? $.convertToString(props.price), + "item_name": props.name ? $.convertToString(props.name), + "id": props.product_id ?? props.sku ? $.convertToString(props.product_id ?? props.sku) : undefined, + "item_category": props.category ? $.convertToString(props.category) : undefined, + "item_brand": props.brand ? $.convertToString(props.brand) : undefined }[] }; - name: combineAllEcomFields diff --git a/test/apitests/data_scenarios/cdk_v2/failure.json b/test/apitests/data_scenarios/cdk_v2/failure.json index 1635a3f0db..154d24481d 100644 --- a/test/apitests/data_scenarios/cdk_v2/failure.json +++ b/test/apitests/data_scenarios/cdk_v2/failure.json @@ -556,6 +556,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -679,6 +683,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/cdk_v2/success.json b/test/apitests/data_scenarios/cdk_v2/success.json index ced7433a28..88f430dd7c 100644 --- a/test/apitests/data_scenarios/cdk_v2/success.json +++ b/test/apitests/data_scenarios/cdk_v2/success.json @@ -556,6 +556,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -634,6 +638,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -712,6 +720,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json b/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json index 3ce7c15091..3deb7d4b8b 100644 --- a/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json +++ b/test/apitests/data_scenarios/destination/proc/batch_input_multiplex.json @@ -388,6 +388,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -466,6 +470,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -544,6 +552,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json b/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json index 0e467c26d0..a2652855d5 100644 --- a/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json +++ b/test/apitests/data_scenarios/destination/proc/multiplex_partial_failure.json @@ -388,6 +388,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -466,6 +470,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/proc/multiplex_success.json b/test/apitests/data_scenarios/destination/proc/multiplex_success.json index 66b6c870a9..ba4d5266f3 100644 --- a/test/apitests/data_scenarios/destination/proc/multiplex_success.json +++ b/test/apitests/data_scenarios/destination/proc/multiplex_success.json @@ -207,6 +207,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -285,6 +289,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/apitests/data_scenarios/destination/router/failure_test.json b/test/apitests/data_scenarios/destination/router/failure_test.json index 9e36da50cb..197456f66a 100644 --- a/test/apitests/data_scenarios/destination/router/failure_test.json +++ b/test/apitests/data_scenarios/destination/router/failure_test.json @@ -754,6 +754,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } @@ -781,6 +785,10 @@ "content_ids": ["123"], "contents": [ { + "id": "123", + "item_brand": "Gamepro", + "item_category": "Games", + "item_name": "Game", "quantity": 11, "item_price": "13.49" } diff --git a/test/integrations/destinations/pinterest_tag/processor/data.ts b/test/integrations/destinations/pinterest_tag/processor/data.ts index b856d247d7..4982444346 100644 --- a/test/integrations/destinations/pinterest_tag/processor/data.ts +++ b/test/integrations/destinations/pinterest_tag/processor/data.ts @@ -482,7 +482,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 2, content_ids: ['123'], - contents: [{ quantity: 2, item_price: '25' }], + contents: [{ id: '123', quantity: 2, item_price: '25' }], }, }, JSON_ARRAY: {}, @@ -2405,7 +2405,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 0, content_ids: ['1234'], - contents: [{ quantity: 1, item_price: 'undefined' }], + contents: [{ id: '1234', quantity: 1 }], }, }, JSON_ARRAY: {}, @@ -2666,7 +2666,7 @@ export const data = [ advertiser_id: '123456', app_id: '429047995', custom_data: { - contents: [{ item_price: 'undefined', quantity: 1 }], + contents: [{ quantity: 1 }], currency: 'USD', num_items: 0, order_id: '50314b8e9bcf000000000000', @@ -3486,7 +3486,7 @@ export const data = [ timestamp: '2020-08-14T05:30:30.118Z', properties: { tax: 2, - total: 27.5, + total: [27.5, 123], coupon: 'hasbros', revenue: 48, currency: 'USD', @@ -3562,11 +3562,10 @@ export const data = [ contents: [ { quantity: 1, - item_price: 'undefined', }, ], currency: 'USD', - value: '27.5', + value: '[27.5,123]', order_id: '50314b8e9bcf000000000000', }, event_name: 'custom event', diff --git a/test/integrations/destinations/pinterest_tag/router/data.ts b/test/integrations/destinations/pinterest_tag/router/data.ts index c9ab29a45a..4049f7663a 100644 --- a/test/integrations/destinations/pinterest_tag/router/data.ts +++ b/test/integrations/destinations/pinterest_tag/router/data.ts @@ -815,7 +815,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 2, content_ids: ['123'], - contents: [{ quantity: 2, item_price: '25' }], + contents: [{ id: '123', quantity: 2, item_price: '25' }], }, }, { diff --git a/test/integrations/destinations/pinterest_tag/step/data.ts b/test/integrations/destinations/pinterest_tag/step/data.ts index b607e3c9fa..71f12c735c 100644 --- a/test/integrations/destinations/pinterest_tag/step/data.ts +++ b/test/integrations/destinations/pinterest_tag/step/data.ts @@ -468,7 +468,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 2, content_ids: ['123'], - contents: [{ quantity: 2, item_price: '25' }], + contents: [{ id: '123', quantity: 2, item_price: '25' }], }, }, JSON_ARRAY: {}, @@ -2420,7 +2420,7 @@ export const data = [ order_id: '50314b8e9bcf000000000000', num_items: 0, content_ids: ['1234'], - contents: [{ quantity: 1, item_price: 'undefined' }], + contents: [{ id: '1234', quantity: 1 }], }, }, JSON_ARRAY: {}, @@ -2685,7 +2685,7 @@ export const data = [ advertiser_id: '123456', app_id: '429047995', custom_data: { - contents: [{ item_price: 'undefined', quantity: 1 }], + contents: [{ quantity: 1 }], currency: 'USD', num_items: 0, order_id: '50314b8e9bcf000000000000', @@ -3606,7 +3606,6 @@ export const data = [ contents: [ { quantity: 1, - item_price: 'undefined', }, ], }, From 74fab0ebe9c4e197a5b66129ca999e6e362ac7a8 Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Date: Mon, 18 Nov 2024 14:20:45 +0530 Subject: [PATCH 24/32] chore: test case cleanup (#3888) --- test/integrations/destinations/active_campaign/router/data.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integrations/destinations/active_campaign/router/data.ts b/test/integrations/destinations/active_campaign/router/data.ts index f65a65d9bc..a73140c161 100644 --- a/test/integrations/destinations/active_campaign/router/data.ts +++ b/test/integrations/destinations/active_campaign/router/data.ts @@ -258,7 +258,7 @@ export const data = [ path: 'path', title: 'title', search: 'search', - tab_url: 'https://simple-tenet.github.io/rudderstack-sample-site/', + tab_url: 'https://abc.com/sample-site/', referrer: 'referrer', initial_referrer: '$direct', referring_domain: '', @@ -288,7 +288,7 @@ export const data = [ path: 'path', title: 'title', search: 'search', - tab_url: 'https://simple-tenet.github.io/rudderstack-sample-site/', + tab_url: 'https://abc.com/sample-site/', referrer: 'referrer', initial_referrer: '$direct', referring_domain: '', From 79e597907eee126b4187e4534b2aa2253d1431da Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:49:18 +0530 Subject: [PATCH 25/32] fix: adding logger for undefined source event (#3879) * fix: adding logger for undefined source event * fix: review comment addressed * chore: enrich metadata, update error msg in logger * chore: code review updates --------- Co-authored-by: Sai Sankeerth --- src/controllers/source.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/controllers/source.ts b/src/controllers/source.ts index 8b6d2d70f8..3d9fa4f4a4 100644 --- a/src/controllers/source.ts +++ b/src/controllers/source.ts @@ -11,6 +11,11 @@ export class SourceController { const requestMetadata = MiscService.getRequestMetadata(ctx); const events = ctx.request.body as object[]; const { version, source }: { version: string; source: string } = ctx.params; + const enrichedMetadata = { + ...requestMetadata, + source, + version, + }; const integrationService = ServiceSelector.getNativeSourceService(); try { @@ -28,6 +33,7 @@ export class SourceController { ); ctx.body = resplist; } catch (err: any) { + logger.error(err?.message || 'error in source transformation', enrichedMetadata); const metaTO = integrationService.getTags(); const resp = SourcePostTransformationService.handleFailureEventsSource(err, metaTO); ctx.body = [resp]; From f3b0300323722680737b99c60595986bacd07f7d Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Mon, 18 Nov 2024 09:46:29 +0000 Subject: [PATCH 26/32] chore(release): 1.85.0 --- CHANGELOG.md | 24 ++++++++++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3c5528ff6..260da5bd73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,30 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.85.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.84.0...v1.85.0) (2024-11-18) + + +### Features + +* added support to eu/us2 datacenter for gainsight px destination ([#3871](https://github.com/rudderlabs/rudder-transformer/issues/3871)) ([12ac3de](https://github.com/rudderlabs/rudder-transformer/commit/12ac3de6e7cc91a6cd52c33bc342f74bbaa8a631)) +* iterable EUDC ([#3828](https://github.com/rudderlabs/rudder-transformer/issues/3828)) ([1c134f8](https://github.com/rudderlabs/rudder-transformer/commit/1c134f84601aaea78581078137cb9955de576f9e)) +* iterable EUDC deleteUsers ([#3881](https://github.com/rudderlabs/rudder-transformer/issues/3881)) ([becb4fa](https://github.com/rudderlabs/rudder-transformer/commit/becb4fa54e9093ed69779f54c36864cb9d28d321)) +* moved userSchema to connection config in GARL vdmv2 ([#3870](https://github.com/rudderlabs/rudder-transformer/issues/3870)) ([640a11e](https://github.com/rudderlabs/rudder-transformer/commit/640a11eb3dca5735fed3ad9ad5bd058974b069d6)) +* now getting consent related fields from connection config from retl for GARL ([#3877](https://github.com/rudderlabs/rudder-transformer/issues/3877)) ([51bbc02](https://github.com/rudderlabs/rudder-transformer/commit/51bbc02d5b00ce1b8fe8c91b4a7041e926bae9bd)) +* onboard linkedin audience destination ([#3857](https://github.com/rudderlabs/rudder-transformer/issues/3857)) ([f3ff409](https://github.com/rudderlabs/rudder-transformer/commit/f3ff4092d455508dd3354ffb22d345fa97f4d1f2)) +* onboarding intercom v2 retl support ([#3843](https://github.com/rudderlabs/rudder-transformer/issues/3843)) ([3d7db73](https://github.com/rudderlabs/rudder-transformer/commit/3d7db7366e30df31c37cc473e344da82b49ed885)) +* sources v2 spec support along with adapters ([04c0694](https://github.com/rudderlabs/rudder-transformer/commit/04c069486bdd3c101906fa6c621e983090fcab25)) +* sources v2 spec support along with adapters ([#3810](https://github.com/rudderlabs/rudder-transformer/issues/3810)) ([c51cfbb](https://github.com/rudderlabs/rudder-transformer/commit/c51cfbb4664a8531dce23b2d06fe40997f95697e)) +* update pinterest_tag single product events with new mapping ([#3858](https://github.com/rudderlabs/rudder-transformer/issues/3858)) ([8520278](https://github.com/rudderlabs/rudder-transformer/commit/85202781de3464bd46fe910159d2b143cd4209e8)) + + +### Bug Fixes + +* adding logger for undefined source event ([#3879](https://github.com/rudderlabs/rudder-transformer/issues/3879)) ([79e5979](https://github.com/rudderlabs/rudder-transformer/commit/79e597907eee126b4187e4534b2aa2253d1431da)) +* adding uuid transformation for airship ([#3884](https://github.com/rudderlabs/rudder-transformer/issues/3884)) ([a80f874](https://github.com/rudderlabs/rudder-transformer/commit/a80f87486dc93b423e4fe6efbee6f4cb8330ba02)) +* handling invalid timestamp for adjust source ([#3866](https://github.com/rudderlabs/rudder-transformer/issues/3866)) ([d57f48e](https://github.com/rudderlabs/rudder-transformer/commit/d57f48e989d18d469bea0de94293bc685300945b)) +* revert gaec changes ([#3885](https://github.com/rudderlabs/rudder-transformer/issues/3885)) ([0aeaa39](https://github.com/rudderlabs/rudder-transformer/commit/0aeaa391b025fc68de6e3d63a6721f067c5be318)) + ## [1.84.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.83.2...v1.84.0) (2024-11-11) diff --git a/package-lock.json b/package-lock.json index d10997f143..804bed84ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.84.0", + "version": "1.85.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.84.0", + "version": "1.85.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index 92ac9f0fb9..db80c9b159 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.84.0", + "version": "1.85.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From e6077b285035e3842509c62df919d288ed28f5df Mon Sep 17 00:00:00 2001 From: Vinay Teki Date: Wed, 20 Nov 2024 15:59:06 +0530 Subject: [PATCH 27/32] chore: github action for unchanged commits fixed --- .github/workflows/verify.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 4bd66285bd..604ff1eaee 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -35,8 +35,8 @@ jobs: - name: Filter JS/TS Files run: | - echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' > changed_files.txt - if [ ! -s changed_files.txt ]; then + changed_files=$(echo "${{ steps.files.outputs.added_modified }}" | tr ' ' '\n' | grep -E '\.(js|ts|jsx|tsx)$' || true) + if [ -z "$changed_files" ]; then echo "No JS/TS files to format or lint." exit 0 fi From ca71a318e4d8d098116fe539964b699254f58617 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Thu, 21 Nov 2024 14:49:15 +0530 Subject: [PATCH 28/32] fix: braze subscription batch size (#3897) --- src/v0/destinations/braze/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v0/destinations/braze/config.js b/src/v0/destinations/braze/config.js index 2bbade2754..8ccabbdb0a 100644 --- a/src/v0/destinations/braze/config.js +++ b/src/v0/destinations/braze/config.js @@ -36,7 +36,7 @@ const IDENTIFY_BRAZE_MAX_REQ_COUNT = 50; // https://www.braze.com/docs/api/endpoints/user_data/post_user_delete/ const ALIAS_BRAZE_MAX_REQ_COUNT = 50; -const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 50; +const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 25; const DEL_MAX_BATCH_SIZE = 50; const DESTINATION = 'braze'; From bb0b9dc1e5a56e8141c6cb56e89835ba61ee7761 Mon Sep 17 00:00:00 2001 From: shrouti1507 <60211312+shrouti1507@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:25:48 +0530 Subject: [PATCH 29/32] fix: stringifying session ID for airship (#3896) * fix: stringifying session ID for airship * fix: test case numbering fix * fix: adding more test cases * fix: review comment addressed * fix: edit error message * chore: refactored code * chore: handled stringifcation in convertToUuid * fix: adding test cases * fix: small edit * fix: code review addressed --------- Co-authored-by: krishnachaitanya --- .../airship/data/airshipTrackConfig.json | 5 +- src/v0/destinations/airship/transform.js | 12 +- src/v0/destinations/airship/utils.js | 12 - src/v0/util/index.js | 25 ++ src/v0/util/index.test.js | 64 +++++ .../destinations/airship/processor/data.ts | 263 +++++++++++++++++- 6 files changed, 366 insertions(+), 15 deletions(-) delete mode 100644 src/v0/destinations/airship/utils.js diff --git a/src/v0/destinations/airship/data/airshipTrackConfig.json b/src/v0/destinations/airship/data/airshipTrackConfig.json index 1f280f756b..4b09fdb943 100644 --- a/src/v0/destinations/airship/data/airshipTrackConfig.json +++ b/src/v0/destinations/airship/data/airshipTrackConfig.json @@ -22,7 +22,10 @@ { "destKey": "session_id", "sourceKeys": ["properties.sessionId", "context.sessionId"], - "required": false + "required": false, + "metadata": { + "type": "toString" + } }, { "destKey": "transaction", diff --git a/src/v0/destinations/airship/transform.js b/src/v0/destinations/airship/transform.js index dc8543fbc5..fcf18daa7e 100644 --- a/src/v0/destinations/airship/transform.js +++ b/src/v0/destinations/airship/transform.js @@ -22,12 +22,20 @@ const { extractCustomFields, isEmptyObject, simpleProcessRouterDest, + convertToUuid, } = require('../../util'); const { JSON_MIME_TYPE } = require('../../util/constant'); -const { transformSessionId } = require('./utils'); const DEFAULT_ACCEPT_HEADER = 'application/vnd.urbanairship+json; version=3'; +const transformSessionId = (rawSessionId) => { + try { + return convertToUuid(rawSessionId); + } catch (error) { + throw new InstrumentationError(`Failed to transform session ID: ${error.message}`); + } +}; + const identifyResponseBuilder = (message, { Config }) => { const tagPayload = constructPayload(message, identifyMapping); const { apiKey, dataCenter } = Config; @@ -129,6 +137,8 @@ const trackResponseBuilder = async (message, { Config }) => { name = name.toLowerCase(); const payload = constructPayload(message, trackMapping); + + // ref : https://docs.airship.com/api/ua/#operation-api-custom-events-post if (isDefinedAndNotNullAndNotEmpty(payload.session_id)) { payload.session_id = transformSessionId(payload.session_id); } diff --git a/src/v0/destinations/airship/utils.js b/src/v0/destinations/airship/utils.js deleted file mode 100644 index 0ef637245f..0000000000 --- a/src/v0/destinations/airship/utils.js +++ /dev/null @@ -1,12 +0,0 @@ -const { v5 } = require('uuid'); - -// ref : https://docs.airship.com/api/ua/#operation-api-custom-events-post -const transformSessionId = (rawSessionId) => { - const NAMESPACE = v5.DNS; - const uuidV5 = v5(rawSessionId, NAMESPACE); - return uuidV5; -}; - -module.exports = { - transformSessionId, -}; diff --git a/src/v0/util/index.js b/src/v0/util/index.js index 1676498fdb..ad08be448e 100644 --- a/src/v0/util/index.js +++ b/src/v0/util/index.js @@ -16,6 +16,7 @@ const uaParser = require('ua-parser-js'); const moment = require('moment-timezone'); const sha256 = require('sha256'); const crypto = require('crypto'); +const { v5 } = require('uuid'); const { InstrumentationError, BaseError, @@ -2330,6 +2331,29 @@ const isEventSentByVDMV1Flow = (event) => event?.message?.context?.mappedToDesti const isEventSentByVDMV2Flow = (event) => event?.connection?.config?.destination?.schemaVersion === VDM_V2_SCHEMA_VERSION; + +const convertToUuid = (input) => { + const NAMESPACE = v5.DNS; + + if (!isDefinedAndNotNull(input)) { + throw new InstrumentationError('Input is undefined or null.'); + } + + try { + // Stringify and trim the input + const trimmedInput = String(input).trim(); + + // Check for empty input after trimming + if (!trimmedInput) { + throw new InstrumentationError('Input is empty or invalid.'); + } + // Generate and return UUID + return v5(trimmedInput, NAMESPACE); + } catch (error) { + const errorMessage = `Failed to transform input to uuid: ${error.message}`; + throw new InstrumentationError(errorMessage); + } +}; // ======================================================================== // EXPORTS // ======================================================================== @@ -2456,4 +2480,5 @@ module.exports = { getRelativePathFromURL, removeEmptyKey, isAxiosError, + convertToUuid, }; diff --git a/src/v0/util/index.test.js b/src/v0/util/index.test.js index 0b05b6f2d6..cfdfefddee 100644 --- a/src/v0/util/index.test.js +++ b/src/v0/util/index.test.js @@ -2,6 +2,7 @@ const { InstrumentationError } = require('@rudderstack/integrations-lib'); const utilities = require('.'); const { getFuncTestData } = require('../../../test/testHelper'); const { FilteredEventsError } = require('./errorTypes'); +const { v5 } = require('uuid'); const { hasCircularReference, flattenJson, @@ -11,6 +12,7 @@ const { groupRouterTransformEvents, isAxiosError, removeHyphens, + convertToUuid, } = require('./index'); const exp = require('constants'); @@ -985,3 +987,65 @@ describe('removeHyphens', () => { }); }); }); + +describe('convertToUuid', () => { + const NAMESPACE = v5.DNS; + + test('should generate UUID for valid string input', () => { + const input = 'testInput'; + const expectedUuid = '7ba1e88f-acf9-5528-9c1c-0c897ed80e1e'; + const result = convertToUuid(input); + expect(result).toBe(expectedUuid); + }); + + test('should generate UUID for valid numeric input', () => { + const input = 123456; + const expectedUuid = 'a52b2702-9bcf-5701-852a-2f4edc640fe1'; + const result = convertToUuid(input); + expect(result).toBe(expectedUuid); + }); + + test('should trim spaces and generate UUID', () => { + const input = ' testInput '; + const expectedUuid = '7ba1e88f-acf9-5528-9c1c-0c897ed80e1e'; + const result = convertToUuid(input); + expect(result).toBe(expectedUuid); + }); + + test('should throw an error for empty input', () => { + const input = ''; + expect(() => convertToUuid(input)).toThrow(InstrumentationError); + expect(() => convertToUuid(input)).toThrow('Input is empty or invalid.'); + }); + + test('to throw an error for null input', () => { + const input = null; + expect(() => convertToUuid(input)).toThrow(InstrumentationError); + expect(() => convertToUuid(input)).toThrow('Input is undefined or null'); + }); + + test('to throw an error for undefined input', () => { + const input = undefined; + expect(() => convertToUuid(input)).toThrow(InstrumentationError); + expect(() => convertToUuid(input)).toThrow('Input is undefined or null'); + }); + + test('should throw an error for input that is whitespace only', () => { + const input = ' '; + expect(() => convertToUuid(input)).toThrow(InstrumentationError); + expect(() => convertToUuid(input)).toThrow('Input is empty or invalid.'); + }); + + test('should handle long string input gracefully', () => { + const input = 'a'.repeat(1000); + const expectedUuid = v5(input, NAMESPACE); + const result = convertToUuid(input); + expect(result).toBe(expectedUuid); + }); + + test('any invalid input if stringified does not throw error', () => { + const input = {}; + const result = convertToUuid(input); + expect(result).toBe('672ca00c-37f4-5d71-b8c3-6ae0848080ec'); + }); +}); diff --git a/test/integrations/destinations/airship/processor/data.ts b/test/integrations/destinations/airship/processor/data.ts index a72495d23d..3c6d827ce6 100644 --- a/test/integrations/destinations/airship/processor/data.ts +++ b/test/integrations/destinations/airship/processor/data.ts @@ -2296,7 +2296,7 @@ export const data = [ }, { name: 'airship', - description: 'Test 22 : session id gets converted to v5 uuid format', + description: 'Test 22 : session id from Web SDK gets converted to v5 uuid format', feature: 'processor', module: 'destination', version: 'v0', @@ -2381,6 +2381,267 @@ export const data = [ }, }, }, + { + name: 'airship', + description: 'Test 23 : session id from mobile SDK gets converted to v5 uuid format', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + traits: { email: 'testone@gmail.com', firstName: 'test', lastName: 'one' }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { name: '', version: '' }, + screen: { density: 2 }, + sessionId: 1731403898, + }, + type: 'track', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + anonymousId: '123456', + event: 'Product Clicked', + userId: 'testuserId1', + properties: {}, + integrations: { All: true }, + }, + destination: { + Config: { + apiKey: 'dummyApiKey', + appKey: 'ffdf', + dataCenter: false, + }, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://go.urbanairship.com/api/custom-events', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/vnd.urbanairship+json; version=3', + 'X-UA-Appkey': 'ffdf', + Authorization: 'Bearer dummyApiKey', + }, + params: {}, + body: { + JSON: { + user: { named_user_id: 'testuserId1' }, + body: { + name: 'product_clicked', + session_id: 'd5627eac-795d-5005-9bb4-2c7c0af6cab0', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'airship', + description: 'Test 24 : session id null gets ignored', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + traits: { email: 'testone@gmail.com', firstName: 'test', lastName: 'one' }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { name: '', version: '' }, + screen: { density: 2 }, + }, + type: 'track', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + anonymousId: '123456', + event: 'Product Clicked', + userId: 'testuserId1', + properties: { + sessionId: null, + }, + integrations: { All: true }, + }, + destination: { + Config: { + apiKey: 'dummyApiKey', + appKey: 'ffdf', + dataCenter: false, + }, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://go.urbanairship.com/api/custom-events', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/vnd.urbanairship+json; version=3', + 'X-UA-Appkey': 'ffdf', + Authorization: 'Bearer dummyApiKey', + }, + params: {}, + body: { + JSON: { + user: { named_user_id: 'testuserId1' }, + body: { + name: 'product_clicked', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, + { + name: 'airship', + description: 'Test 24 : session id undefined gets ignored', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + message: { + channel: 'web', + context: { + app: { + build: '1.0.0', + name: 'RudderLabs JavaScript SDK', + namespace: 'com.rudderlabs.javascript', + version: '1.0.0', + }, + traits: { email: 'testone@gmail.com', firstName: 'test', lastName: 'one' }, + library: { name: 'RudderLabs JavaScript SDK', version: '1.0.0' }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36', + locale: 'en-US', + ip: '0.0.0.0', + os: { name: '', version: '' }, + screen: { density: 2 }, + }, + type: 'track', + messageId: '84e26acc-56a5-4835-8233-591137fca468', + anonymousId: '123456', + event: 'Product Clicked', + userId: 'testuserId1', + properties: { + sessionId: undefined, + }, + integrations: { All: true }, + }, + destination: { + Config: { + apiKey: 'dummyApiKey', + appKey: 'ffdf', + dataCenter: false, + }, + }, + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: { + version: '1', + type: 'REST', + method: 'POST', + endpoint: 'https://go.urbanairship.com/api/custom-events', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/vnd.urbanairship+json; version=3', + 'X-UA-Appkey': 'ffdf', + Authorization: 'Bearer dummyApiKey', + }, + params: {}, + body: { + JSON: { + user: { named_user_id: 'testuserId1' }, + body: { + name: 'product_clicked', + }, + }, + JSON_ARRAY: {}, + XML: {}, + FORM: {}, + }, + files: {}, + userId: '', + }, + statusCode: 200, + }, + ], + }, + }, + }, ].map((tc) => ({ ...tc, mockFns: (_) => { From cf28f14eb261f873ca32e5bdca98c13e825a8ed1 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Thu, 21 Nov 2024 10:08:47 +0000 Subject: [PATCH 30/32] chore(release): 1.85.1 --- CHANGELOG.md | 8 ++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 260da5bd73..bb1779fd3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [1.85.1](https://github.com/rudderlabs/rudder-transformer/compare/v1.85.0...v1.85.1) (2024-11-21) + + +### Bug Fixes + +* braze subscription batch size ([#3897](https://github.com/rudderlabs/rudder-transformer/issues/3897)) ([ca71a31](https://github.com/rudderlabs/rudder-transformer/commit/ca71a318e4d8d098116fe539964b699254f58617)) +* stringifying session ID for airship ([#3896](https://github.com/rudderlabs/rudder-transformer/issues/3896)) ([bb0b9dc](https://github.com/rudderlabs/rudder-transformer/commit/bb0b9dc1e5a56e8141c6cb56e89835ba61ee7761)) + ## [1.85.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.84.0...v1.85.0) (2024-11-18) diff --git a/package-lock.json b/package-lock.json index 804bed84ce..40033e278b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.85.0", + "version": "1.85.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.85.0", + "version": "1.85.1", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", diff --git a/package.json b/package.json index db80c9b159..e0b17a4f47 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.85.0", + "version": "1.85.1", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { From 864a9aaf5f7373b2d7b1a1290a3c69511355be09 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Thu, 21 Nov 2024 15:59:12 +0530 Subject: [PATCH 31/32] fix: test (#3899) --- src/v0/destinations/braze/braze.util.test.js | 21 +++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js index 71052f8d77..0ee0ec2ebf 100644 --- a/src/v0/destinations/braze/braze.util.test.js +++ b/src/v0/destinations/braze/braze.util.test.js @@ -972,7 +972,7 @@ describe('processBatch', () => { // Assert that the response is as expected expect(result.length).toBe(1); // One successful batched request and one failure response - expect(result[0].batchedRequest.length).toBe(6); // Two batched requests + expect(result[0].batchedRequest.length).toBe(8); // Two batched requests expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events @@ -981,10 +981,12 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.attributes.length).toBe(25); // Second batch contains remaining 25 attributes expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(25); // Second batch contains remaining 25 events expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(25); // Second batch contains remaining 25 purchases - expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group - expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); // First batch contains 25 subscription group - expect(result[0].batchedRequest[4].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates - expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates + expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(25); // First batch contains 50 subscription group + expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); + expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // First batch contains 50 subscription group + expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group + expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates + expect(result[0].batchedRequest[7].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates }); test('processBatch handles more than 75 attributes, events, and purchases with non uniform distribution', () => { @@ -1093,7 +1095,7 @@ describe('processBatch', () => { // Assert that the response is as expected expect(result.length).toBe(1); // One successful batched request and one failure response expect(result[0].metadata.length).toBe(490); // Check the total length is same as input jobs (120 + 160 + 100 + 70 +40) - expect(result[0].batchedRequest.length).toBe(6); // Two batched requests + expect(result[0].batchedRequest.length).toBe(7); // Two batched requests expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events @@ -1103,9 +1105,10 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(45); // Second batch contains remaining 45 events expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(75); // Second batch contains remaining 75 purchases expect(result[0].batchedRequest[2].body.JSON.purchases.length).toBe(10); // Third batch contains remaining 10 purchases - expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group - expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(20); // First batch contains 20 subscription group - expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(40); // First batch contains 50 merge_updates + expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group + expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group + expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(20); // Third batch contains 20 subscription group + expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(40); // First batch contains 50 merge_updates }); test('check success and failure scenarios both for processBatch', () => { From 49a6f189d15322e37b0e4efcd573b4b29fa8d147 Mon Sep 17 00:00:00 2001 From: AASHISH MALIK Date: Thu, 21 Nov 2024 17:22:04 +0530 Subject: [PATCH 32/32] chore: braze batch (#3900) --- src/v0/destinations/braze/braze.util.test.js | 21 +++++++++----------- src/v0/destinations/braze/config.js | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/v0/destinations/braze/braze.util.test.js b/src/v0/destinations/braze/braze.util.test.js index 0ee0ec2ebf..985f2434d5 100644 --- a/src/v0/destinations/braze/braze.util.test.js +++ b/src/v0/destinations/braze/braze.util.test.js @@ -972,7 +972,7 @@ describe('processBatch', () => { // Assert that the response is as expected expect(result.length).toBe(1); // One successful batched request and one failure response - expect(result[0].batchedRequest.length).toBe(8); // Two batched requests + expect(result[0].batchedRequest.length).toBe(6); // Two batched requests expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events @@ -981,12 +981,10 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.attributes.length).toBe(25); // Second batch contains remaining 25 attributes expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(25); // Second batch contains remaining 25 events expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(25); // Second batch contains remaining 25 purchases - expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(25); // First batch contains 50 subscription group - expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); - expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // First batch contains 50 subscription group - expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group - expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates - expect(result[0].batchedRequest[7].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates + expect(result[0].batchedRequest[2].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group + expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); + expect(result[0].batchedRequest[4].body.JSON.merge_updates.length).toBe(50); // First batch contains 50 merge_updates + expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(50); // First batch contains 25 merge_updates }); test('processBatch handles more than 75 attributes, events, and purchases with non uniform distribution', () => { @@ -1095,7 +1093,7 @@ describe('processBatch', () => { // Assert that the response is as expected expect(result.length).toBe(1); // One successful batched request and one failure response expect(result[0].metadata.length).toBe(490); // Check the total length is same as input jobs (120 + 160 + 100 + 70 +40) - expect(result[0].batchedRequest.length).toBe(7); // Two batched requests + expect(result[0].batchedRequest.length).toBe(6); // Two batched requests expect(result[0].batchedRequest[0].body.JSON.partner).toBe('RudderStack'); // Verify partner name expect(result[0].batchedRequest[0].body.JSON.attributes.length).toBe(75); // First batch contains 75 attributes expect(result[0].batchedRequest[0].body.JSON.events.length).toBe(75); // First batch contains 75 events @@ -1105,10 +1103,9 @@ describe('processBatch', () => { expect(result[0].batchedRequest[1].body.JSON.events.length).toBe(45); // Second batch contains remaining 45 events expect(result[0].batchedRequest[1].body.JSON.purchases.length).toBe(75); // Second batch contains remaining 75 purchases expect(result[0].batchedRequest[2].body.JSON.purchases.length).toBe(10); // Third batch contains remaining 10 purchases - expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(25); // First batch contains 25 subscription group - expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(25); // Second batch contains 25 subscription group - expect(result[0].batchedRequest[5].body.JSON.subscription_groups.length).toBe(20); // Third batch contains 20 subscription group - expect(result[0].batchedRequest[6].body.JSON.merge_updates.length).toBe(40); // First batch contains 50 merge_updates + expect(result[0].batchedRequest[3].body.JSON.subscription_groups.length).toBe(50); // First batch contains 50 subscription group + expect(result[0].batchedRequest[4].body.JSON.subscription_groups.length).toBe(20); // Second batch contains 20 subscription group + expect(result[0].batchedRequest[5].body.JSON.merge_updates.length).toBe(40); // First batch contains 40 merge_updates }); test('check success and failure scenarios both for processBatch', () => { diff --git a/src/v0/destinations/braze/config.js b/src/v0/destinations/braze/config.js index 8ccabbdb0a..2bbade2754 100644 --- a/src/v0/destinations/braze/config.js +++ b/src/v0/destinations/braze/config.js @@ -36,7 +36,7 @@ const IDENTIFY_BRAZE_MAX_REQ_COUNT = 50; // https://www.braze.com/docs/api/endpoints/user_data/post_user_delete/ const ALIAS_BRAZE_MAX_REQ_COUNT = 50; -const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 25; +const SUBSCRIPTION_BRAZE_MAX_REQ_COUNT = 50; const DEL_MAX_BATCH_SIZE = 50; const DESTINATION = 'braze';