diff --git a/lib/cli.js b/lib/cli.js index d79d73cd3..df3af46aa 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -87,37 +87,50 @@ yargs(hideBin(process.argv)) describe: 'type or type:key or type:i:id or type:n:name to deploy; if not provided, all metadata will be deploy', }) + .option('keySuffix', { + type: 'string', + alias: 'ks', + group: 'Options for deploy:', + describe: + 'allows you to add a suffix to the key of the metadata to be deployed. Only works together with changeKeyField', + }) .option('changeKeyField', { type: 'string', + alias: 'ckf', group: 'Options for deploy:', describe: 'enables updating the key of the deployed metadata with the value in provided field (e.g. c__newKey). Can be used to sync name and key fields.', }) .option('changeKeyValue', { type: 'string', + alias: 'ckv', group: 'Options for deploy:', describe: 'allows updating the key of the metadata to the provided value. Only available if a single type and key is deployed', }) .option('fromRetrieve', { type: 'boolean', + alias: 'fr', group: 'Options for deploy:', describe: 'optionally deploy from retrieve folder', }) .option('refresh', { type: 'boolean', + alias: 'r', group: 'Options for deploy:', describe: 'optional for asset-message: runs refresh command for related triggeredSends after deploy', }) .option('execute', { type: 'boolean', + alias: 'e', group: 'Options for deploy:', describe: 'optional: executes item after deploy; this will run the item once immediately', }) .option('schedule', { type: 'boolean', + alias: 's', group: 'Options for deploy:', describe: 'optionally start existing schedule instead of running item once immediately (only works for automations)', @@ -127,7 +140,6 @@ yargs(hideBin(process.argv)) describe: "optionally ensure that updates to shared DataExtensions become visible in child BU's data designer (SF Known issue W-11031095)", }); - // TODO: add option --metadata }, handler: (argv) => { Mcdev.setOptions(argv); @@ -644,14 +656,23 @@ yargs(hideBin(process.argv)) describe: 'filter metadata components (can include % as wildcard or _ for a single character)', }) + .option('keySuffix', { + type: 'string', + alias: 'ks', + group: 'Options for fixKeys:', + describe: + 'allows you to add a suffix to the key of the metadata to be deployed.', + }) .option('execute', { type: 'boolean', + alias: 'e', group: 'Options for fixKeys:', describe: 'optional: executes item after deploy; this will run the item once immediately', }) .option('schedule', { type: 'boolean', + alias: 's', group: 'Options for fixKeys:', describe: 'optionally start existing schedule instead of running item once immediately (only works for automations)', diff --git a/lib/index.js b/lib/index.js index 87171109c..d2ac822e6 100644 --- a/lib/index.js +++ b/lib/index.js @@ -77,6 +77,7 @@ class Mcdev { static setOptions(argv) { const knownOptions = [ 'api', + 'keySuffix', 'changeKeyField', 'changeKeyValue', 'commitHistory', diff --git a/lib/metadataTypes/MetadataType.js b/lib/metadataTypes/MetadataType.js index 0159a1a84..2e9b2349e 100644 --- a/lib/metadataTypes/MetadataType.js +++ b/lib/metadataTypes/MetadataType.js @@ -738,12 +738,17 @@ class MetadataType { ` - --changeKeyField is set to the same value as the keyField for ${this.definition.type}. Skipping change.` ); } else if (metadataMap[metadataKey][Util.OPTIONS.changeKeyField]) { - // NOTE: trim twice while getting the newKey value to remove leading spaces before limiting the length - const newKey = (metadataMap[metadataKey][Util.OPTIONS.changeKeyField] + '') - .trim() - .slice(0, maxKeyLength) - .trim(); - if (metadataMap[metadataKey][Util.OPTIONS.changeKeyField] + '' > maxKeyLength) { + Util.OPTIONS.keySuffix = Util.OPTIONS.keySuffix + ? Util.OPTIONS.keySuffix.trim() + : ''; + const newKey = this.getNewKey(metadataMap[metadataKey], maxKeyLength); + + if ( + metadataMap[metadataKey][Util.OPTIONS.changeKeyField] + + '' + + Util.OPTIONS.keySuffix > + maxKeyLength + ) { Util.logger.warn( `${this.definition.type} ${this.definition.keyField} may not exceed ${maxKeyLength} characters. Truncated the value in field ${Util.OPTIONS.changeKeyField} to ${newKey}` ); @@ -2166,30 +2171,31 @@ class MetadataType { Util.logger.info( `Searching for ${this.definition.type} keys among downloaded items that need fixing:` ); + Util.OPTIONS.keySuffix = Util.OPTIONS.keySuffix ? Util.OPTIONS.keySuffix.trim() : ''; + const maxKeyLength = this.definition.maxKeyLength || 36; + for (const item of Object.values(metadataMap)) { - if (item[this.definition.nameField].length > this.definition.maxKeyLength) { + if ( + (item[this.definition.nameField].endsWith(Util.OPTIONS.keySuffix) && + item[this.definition.nameField].length > maxKeyLength) || + (!item[this.definition.nameField].endsWith(Util.OPTIONS.keySuffix) && + item[this.definition.nameField].length + Util.OPTIONS.keySuffix.length > + maxKeyLength) + ) { Util.logger.warn( - `Name of the item ${ - item[this.definition.keyField] - } is too long for a key. Consider renaming your item. Key will be equal first ${ - this.definition.maxKeyLength - } characters of the name` - ); - item[this.definition.nameField] = item[this.definition.nameField].slice( - 0, - this.definition.maxKeyLength + `Name of the item ${item[this.definition.keyField]} (${ + item[this.definition.nameField] + }) is too long for a key${Util.OPTIONS.keySuffix.length ? ' (including the suffix ' + Util.OPTIONS.keySuffix + ')' : ''}. Consider renaming your item. Key will be equal first ${maxKeyLength} characters of the name` ); } - - if ( - item[this.definition.nameField] != item[this.definition.keyField] && - !this.definition.keyIsFixed - ) { - keysForDeploy.push(item[this.definition.keyField]); + const newKey = this.getNewKey(item, maxKeyLength); + if (newKey != item[this.definition.keyField] && !this.definition.keyIsFixed) { + // add key but make sure to turn it into string or else numeric keys will be filtered later + keysForDeploy.push(item[this.definition.keyField] + ''); Util.logger.info( ` - added ${this.definition.type} to fixKey queue: ${ item[this.definition.keyField] - }` + } >> ${newKey}` ); } else { Util.logger.info( @@ -2205,6 +2211,29 @@ class MetadataType { } return keysForDeploy; } + /** + * helper for getKeysForFixing and createOrUpdate + * + * @param {MetadataTypeItem} metadataItem - + * @param {number} maxKeyLength - + * @returns {string} newKey + */ + static getNewKey(metadataItem, maxKeyLength) { + let newKey; + newKey = (metadataItem[this.definition.nameField] + '') + .trim() + .slice(0, maxKeyLength) + .trim(); + if (Util.OPTIONS.keySuffix.length && !newKey.endsWith(Util.OPTIONS.keySuffix)) { + newKey = + (metadataItem[this.definition.nameField] + '') + .trim() + .slice(0, maxKeyLength - Util.OPTIONS.keySuffix.length) + .trim() + Util.OPTIONS.keySuffix; + } + + return newKey; + } } MetadataType.definition = { diff --git a/lib/metadataTypes/definitions/DeliveryProfile.definition.js b/lib/metadataTypes/definitions/DeliveryProfile.definition.js index fefbe0059..c0b8fafb3 100644 --- a/lib/metadataTypes/definitions/DeliveryProfile.definition.js +++ b/lib/metadataTypes/definitions/DeliveryProfile.definition.js @@ -5,6 +5,8 @@ export default { hasExtended: false, idField: 'id', keyField: 'key', + keyIsFixed: false, + maxKeyLength: 36, // confirmed max length nameField: 'name', createdDateField: 'createdDate', createdNameField: null, diff --git a/lib/metadataTypes/definitions/SendClassification.definition.js b/lib/metadataTypes/definitions/SendClassification.definition.js index 11f0c87f7..2e67e4168 100644 --- a/lib/metadataTypes/definitions/SendClassification.definition.js +++ b/lib/metadataTypes/definitions/SendClassification.definition.js @@ -5,8 +5,9 @@ export default { filter: {}, hasExtended: false, idField: 'ObjectID', - keyIsFixed: null, keyField: 'CustomerKey', + keyIsFixed: false, + maxKeyLength: 36, // confirmed max length nameField: 'Name', createdDateField: 'CreatedDate', createdNameField: null, diff --git a/lib/metadataTypes/definitions/SenderProfile.definition.js b/lib/metadataTypes/definitions/SenderProfile.definition.js index ea98f6d9a..2fdc023fc 100644 --- a/lib/metadataTypes/definitions/SenderProfile.definition.js +++ b/lib/metadataTypes/definitions/SenderProfile.definition.js @@ -4,8 +4,9 @@ export default { filter: {}, hasExtended: false, idField: 'ObjectID', - keyIsFixed: false, keyField: 'CustomerKey', + keyIsFixed: false, + maxKeyLength: 36, // confirmed max length nameField: 'Name', createdDateField: 'CreatedDate', createdNameField: null, diff --git a/test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat_fixKeysSuffix/get-response.json b/test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat_fixKeysSuffix/get-response.json new file mode 100644 index 000000000..b3cbc33ee --- /dev/null +++ b/test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat_fixKeysSuffix/get-response.json @@ -0,0 +1,17 @@ +{ + "queryDefinitionId": "549f0568-607c-4940-afef-437965094dat_fixKeysSuffix", + "name": "testExisting_query_fixedKeys", + "key": "testExisting_query_fixKeysSuffix", + "description": "bla bla", + "queryText": "Select\n SubscriberKey as testField, Trim(last_name) AS name\nfrom\n _Subscribers\nwhere\n country in ('test')\n", + "targetName": "testExisting_dataExtension", + "targetKey": "testExisting_dataExtension", + "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7", + "targetDescription": "", + "createdDate": "2022-04-26T15:21:16.453", + "modifiedDate": "2022-04-26T16:02:44.01", + "targetUpdateTypeId": 0, + "targetUpdateTypeName": "Overwrite", + "categoryId": 999, + "isFrozen": false +} diff --git a/test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat_fixKeysSuffix/patch-response.json b/test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat_fixKeysSuffix/patch-response.json new file mode 100644 index 000000000..fae619936 --- /dev/null +++ b/test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat_fixKeysSuffix/patch-response.json @@ -0,0 +1,18 @@ +{ + "queryDefinitionId": "549f0568-607c-4940-afef-437965094dat_fixKeys", + "name": "testExisting_query_fixedKeys", + "key": "testExisting_query_fixedKeys_DEV", + "description": "updated on deploy", + "queryText": "SELECT\n SubscriberKey as testField\nFROM\n _Subscribers\nWHERE\n country IN ('test')\n", + "targetName": "testExisting_dataExtension", + "targetKey": "testExisting_dataExtension", + "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7", + "targetDescription": "", + "createdDate": "2022-04-26T15:21:16.453", + "modifiedDate": "2022-04-26T16:04:15.88", + "targetUpdateTypeId": 0, + "targetUpdateTypeName": "Overwrite", + "validatedQueryText": "SET NOCOUNT ON; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;\r\n\r\nINSERT INTO C518001158.[testDataExtension] ([testField])\r\nSELECT querydef.[testField]\r\nFROM (SELECT SubscriberKey as testField FROM C518001158._Subscribers ) AS querydef \r\nSELECT @rcInsert = @@ROWCOUNT;;\r\n", + "categoryId": 999, + "isFrozen": false +} diff --git a/test/resources/9999999/automation/v1/queries/get-response.json b/test/resources/9999999/automation/v1/queries/get-response.json index d07088bac..3da91d772 100644 --- a/test/resources/9999999/automation/v1/queries/get-response.json +++ b/test/resources/9999999/automation/v1/queries/get-response.json @@ -53,6 +53,23 @@ "targetUpdateTypeName": "Overwrite", "categoryId": 999, "isFrozen": false + }, + { + "queryDefinitionId": "549f0568-607c-4940-afef-437965094dat_fixKeysSuffix", + "name": "testExisting_query_fixedKeys", + "key": "testExisting_query_fixKeysSuffix", + "description": "bla bla", + "queryText": "Select\n SubscriberKey as testField, Trim(last_name) AS name\nfrom\n _Subscribers\nwhere\n country in ('test')\n", + "targetName": "testExisting_dataExtension", + "targetKey": "testExisting_dataExtension", + "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7", + "targetDescription": "", + "createdDate": "2022-04-26T15:21:16.453", + "modifiedDate": "2022-04-26T16:02:44.01", + "targetUpdateTypeId": 0, + "targetUpdateTypeName": "Overwrite", + "categoryId": 999, + "isFrozen": false } ] } diff --git a/test/resources/9999999/query/patch_fixKeysSuffix-expected.json b/test/resources/9999999/query/patch_fixKeysSuffix-expected.json new file mode 100644 index 000000000..7a7c4bee2 --- /dev/null +++ b/test/resources/9999999/query/patch_fixKeysSuffix-expected.json @@ -0,0 +1,11 @@ +{ + "name": "testExisting_query_fixedKeys", + "key": "testExisting_query_fixedKeys_DEV", + "description": "updated on deploy", + "targetKey": "testExisting_dataExtension", + "createdDate": "2022-04-26T15:21:16.453", + "modifiedDate": "2022-04-26T16:04:15.88", + "targetUpdateTypeName": "Overwrite", + "isFrozen": false, + "r__folder_Path": "Query" +} diff --git a/test/resources/9999999/query/patch_fixKeysSuffix-expected.sql b/test/resources/9999999/query/patch_fixKeysSuffix-expected.sql new file mode 100644 index 000000000..2a32f5fad --- /dev/null +++ b/test/resources/9999999/query/patch_fixKeysSuffix-expected.sql @@ -0,0 +1,6 @@ +SELECT + SubscriberKey AS testField +FROM + _Subscribers +WHERE + country IN ('test') diff --git a/test/resources/9999999/queryDefinition/retrieve-CustomerKey=testExisting_query_fixKeysSuffixANDStatus=Active-response.xml b/test/resources/9999999/queryDefinition/retrieve-CustomerKey=testExisting_query_fixKeysSuffixANDStatus=Active-response.xml new file mode 100644 index 000000000..af515b27e --- /dev/null +++ b/test/resources/9999999/queryDefinition/retrieve-CustomerKey=testExisting_query_fixKeysSuffixANDStatus=Active-response.xml @@ -0,0 +1,30 @@ + + + + RetrieveResponse + urn:uuid:7ef0345e-b559-4fc4-8986-47e54e1a8a58 + urn:uuid:b2e814a6-517c-4882-9bbb-238bfce951ce + http://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymous + + + 2023-04-11T16:33:48Z + 2023-04-11T16:38:48Z + + + + + + OK + e8eb2988-2f43-4243-a6b0-6ab6b841a6ab + + + 549f0568-607c-4940-afef-437965094dat_fixKeysSuffix + + + + diff --git a/test/type.query.test.js b/test/type.query.test.js index 826c3314b..f001b7d55 100644 --- a/test/type.query.test.js +++ b/test/type.query.test.js @@ -28,8 +28,8 @@ describe('type: query', () => { const result = cache.getCache(); assert.equal( result.query ? Object.keys(result.query).length : 0, - 3, - 'only three queries expected' + 4, + 'only 4 queries expected' ); // normal test assert.deepEqual( @@ -93,8 +93,8 @@ describe('type: query', () => { const result = cache.getCache(); assert.equal( result.query ? Object.keys(result.query).length : 0, - 3, - 'three queries in cache expected' + 4, + '4 queries in cache expected' ); assert.deepEqual( await testUtils.getActualJson('testExisting_query', 'query'), @@ -124,8 +124,8 @@ describe('type: query', () => { const result = cache.getCache(); assert.equal( result.query ? Object.keys(result.query).length : 0, - 3, - 'three queries in cache expected' + 4, + '4 queries in cache expected' ); expect(file(testUtils.getActualFile('testExisting_query', 'query', 'sql'))).to.not @@ -166,8 +166,8 @@ describe('type: query', () => { const result = cache.getCache(); assert.equal( result.query ? Object.keys(result.query).length : 0, - 4, - 'four queries expected in cache' + 5, + '5 queries expected in cache' ); // confirm created item assert.deepEqual( @@ -332,6 +332,55 @@ describe('type: query', () => { return; }); + it('Should change the key during update with --changeKeyField and --keySuffix', async () => { + // WHEN + await handler.retrieve( + 'testInstance/testBU', + ['query'], + ['testExisting_query_fixKeysSuffix'] + ); + handler.setOptions({ changeKeyField: 'name', keySuffix: '_DEV', fromRetrieve: true }); + const deployed = await handler.deploy( + 'testInstance/testBU', + ['query'], + ['testExisting_query_fixKeysSuffix'] + ); + // THEN + assert.equal( + process.exitCode, + 0, + 'deploy --changeKeyValue --keySuffix should not have thrown an error' + ); + assert.equal( + Object.keys(deployed['testInstance/testBU'].query).length, + 1, + 'returned number of keys does not correspond to number of expected fixed keys' + ); + assert.equal( + Object.keys(deployed['testInstance/testBU'].query)[0], + 'testExisting_query_fixedKeys_DEV', + 'returned keys do not correspond to expected fixed keys' + ); + // confirm updated item + assert.deepEqual( + await testUtils.getActualJson('testExisting_query_fixedKeys_DEV', 'query'), + await testUtils.getExpectedJson('9999999', 'query', 'patch_fixKeysSuffix'), + 'returned metadata was not equal expected for update query' + ); + expect( + file(testUtils.getActualFile('testExisting_query_fixedKeys_DEV', 'query', 'sql')) + ).to.equal( + file(testUtils.getExpectedFile('9999999', 'query', 'patch_fixKeysSuffix', 'sql')) + ); + // check number of API calls + assert.equal( + testUtils.getAPIHistoryLength(), + 14, + 'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests' + ); + return; + }); + it('Should run fixKeys but not find fixable keys and hence stop', async () => { // WHEN handler.setOptions({ skipInteraction: { fixKeysReretrieve: false } }); @@ -400,6 +449,48 @@ describe('type: query', () => { return; }); + it('Should fixKeys by key with --keySuffix WITHOUT re-retrieving dependent types', async () => { + // WHEN + handler.setOptions({ + keySuffix: '_DEV', + skipInteraction: { fixKeysReretrieve: false }, + }); + const resultFixKeys = await handler.fixKeys('testInstance/testBU', 'query', [ + 'testExisting_query_fixKeysSuffix', + 'testExisting_query', + ]); + assert.equal( + resultFixKeys['testInstance/testBU'].length, + 1, + 'returned number of keys does not correspond to number of expected fixed keys' + ); + assert.equal( + resultFixKeys['testInstance/testBU'][0], + 'testExisting_query_fixedKeys_DEV', + 'returned keys do not correspond to expected fixed keys' + ); + // THEN + assert.equal(process.exitCode, 0, 'fixKeys should not have thrown an error'); + // confirm updated item + assert.deepEqual( + await testUtils.getActualJson('testExisting_query_fixedKeys_DEV', 'query'), + await testUtils.getExpectedJson('9999999', 'query', 'patch_fixKeysSuffix'), + 'returned metadata was not equal expected for update query' + ); + expect( + file(testUtils.getActualFile('testExisting_query_fixedKeys_DEV', 'query', 'sql')) + ).to.equal( + file(testUtils.getExpectedFile('9999999', 'query', 'patch_fixKeysSuffix', 'sql')) + ); + // check number of API calls + assert.equal( + testUtils.getAPIHistoryLength(), + 16, + 'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests' + ); + return; + }); + it('Should fixKeys by key WITHOUT re-retrieving dependent types and then --execute', async () => { // WHEN handler.setOptions({ skipInteraction: { fixKeysReretrieve: false }, execute: true }); @@ -528,7 +619,7 @@ describe('type: query', () => { const resultFixKeys = await handler.fixKeys('testInstance/testBU', 'query'); assert.equal( resultFixKeys['testInstance/testBU'].length, - 1, + 2, 'returned number of keys does not correspond to number of expected fixed keys' ); assert.equal( @@ -550,7 +641,7 @@ describe('type: query', () => { // check number of API calls assert.equal( testUtils.getAPIHistoryLength(), - 13, + 14, 'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests' ); return;