Feature/1302 + Bugfix/1298 automatically add suffix to key

lib/cli.js

@@ -87,37 +87,50 @@ yargs(hideBin(process.argv))
                     describe:
                         'type or type:key or type:i:id or type:n:name to deploy; if not provided, all metadata will be deploy',
                 })
+                .option('keySuffix', {
+                    type: 'string',
+                    alias: 'ks',
+                    group: 'Options for deploy:',
+                    describe:
+                        'allows you to add a suffix to the key of the metadata to be deployed. Only works together with changeKeyField',
+                })
                 .option('changeKeyField', {
                     type: 'string',
+                    alias: 'ckf',
                     group: 'Options for deploy:',
                     describe:
                         'enables updating the key of the deployed metadata with the value in provided field (e.g. c__newKey). Can be used to sync name and key fields.',
                 })
                 .option('changeKeyValue', {
                     type: 'string',
+                    alias: 'ckv',
                     group: 'Options for deploy:',
                     describe:
                         'allows updating the key of the metadata to the provided value. Only available if a single type and key is deployed',
                 })
                 .option('fromRetrieve', {
                     type: 'boolean',
+                    alias: 'fr',
                     group: 'Options for deploy:',
                     describe: 'optionally deploy from retrieve folder',
                 })
                 .option('refresh', {
                     type: 'boolean',
+                    alias: 'r',
                     group: 'Options for deploy:',
                     describe:
                         'optional for asset-message: runs refresh command for related triggeredSends after deploy',
                 })
                 .option('execute', {
                     type: 'boolean',
+                    alias: 'e',
                     group: 'Options for deploy:',
                     describe:
                         'optional: executes item after deploy; this will run the item once immediately',
                 })
                 .option('schedule', {
                     type: 'boolean',
+                    alias: 's',
                     group: 'Options for deploy:',
                     describe:
                         'optionally start existing schedule instead of running item once immediately (only works for automations)',
@@ -127,7 +140,6 @@ yargs(hideBin(process.argv))
                     describe:
                         "optionally ensure that updates to shared DataExtensions become visible in child BU's data designer (SF Known issue W-11031095)",
                 });
-            // TODO: add option --metadata
         },
         handler: (argv) => {
             Mcdev.setOptions(argv);
@@ -644,14 +656,23 @@ yargs(hideBin(process.argv))
                     describe:
                         'filter metadata components (can include % as wildcard or _ for a single character)',
                 })
+                .option('keySuffix', {
+                    type: 'string',
+                    alias: 'ks',
+                    group: 'Options for fixKeys:',
+                    describe:
+                        'allows you to add a suffix to the key of the metadata to be deployed.',
+                })
                 .option('execute', {
                     type: 'boolean',
+                    alias: 'e',
                     group: 'Options for fixKeys:',
                     describe:
                         'optional: executes item after deploy; this will run the item once immediately',
                 })
                 .option('schedule', {
                     type: 'boolean',
+                    alias: 's',
                     group: 'Options for fixKeys:',
                     describe:
                         'optionally start existing schedule instead of running item once immediately (only works for automations)',

lib/index.js

@@ -77,6 +77,7 @@ class Mcdev {
     static setOptions(argv) {
         const knownOptions = [
             'api',
+            'keySuffix',
             'changeKeyField',
             'changeKeyValue',
             'commitHistory',

lib/metadataTypes/MetadataType.js

@@ -738,12 +738,17 @@ class MetadataType {
                         ` - --changeKeyField is set to the same value as the keyField for ${this.definition.type}. Skipping change.`
                     );
                 } else if (metadataMap[metadataKey][Util.OPTIONS.changeKeyField]) {
-                    // NOTE: trim twice while getting the newKey value to remove leading spaces before limiting the length
-                    const newKey = (metadataMap[metadataKey][Util.OPTIONS.changeKeyField] + '')
-                        .trim()
-                        .slice(0, maxKeyLength)
-                        .trim();
-                    if (metadataMap[metadataKey][Util.OPTIONS.changeKeyField] + '' > maxKeyLength) {
+                    Util.OPTIONS.keySuffix = Util.OPTIONS.keySuffix
+                        ? Util.OPTIONS.keySuffix.trim()
+                        : '';
+                    const newKey = this.getNewKey(metadataMap[metadataKey], maxKeyLength);
+
+                    if (
+                        metadataMap[metadataKey][Util.OPTIONS.changeKeyField] +
+                            '' +
+                            Util.OPTIONS.keySuffix >
+                        maxKeyLength
+                    ) {
                         Util.logger.warn(
                             `${this.definition.type} ${this.definition.keyField} may not exceed ${maxKeyLength} characters. Truncated the value in field ${Util.OPTIONS.changeKeyField} to ${newKey}`
                         );
@@ -2166,30 +2171,31 @@ class MetadataType {
             Util.logger.info(
                 `Searching for ${this.definition.type} keys among downloaded items that need fixing:`
             );
+            Util.OPTIONS.keySuffix = Util.OPTIONS.keySuffix ? Util.OPTIONS.keySuffix.trim() : '';
+            const maxKeyLength = this.definition.maxKeyLength || 36;
+
             for (const item of Object.values(metadataMap)) {
-                if (item[this.definition.nameField].length > this.definition.maxKeyLength) {
+                if (
+                    (item[this.definition.nameField].endsWith(Util.OPTIONS.keySuffix) &&
+                        item[this.definition.nameField].length > maxKeyLength) ||
+                    (!item[this.definition.nameField].endsWith(Util.OPTIONS.keySuffix) &&
+                        item[this.definition.nameField].length + Util.OPTIONS.keySuffix.length >
+                            maxKeyLength)
+                ) {
                     Util.logger.warn(
-                        `Name of the item ${
-                            item[this.definition.keyField]
-                        } is too long for a key. Consider renaming your item. Key will be equal first ${
-                            this.definition.maxKeyLength
-                        } characters of the name`
-                    );
-                    item[this.definition.nameField] = item[this.definition.nameField].slice(
-                        0,
-                        this.definition.maxKeyLength
+                        `Name of the item ${item[this.definition.keyField]} (${
+                            item[this.definition.nameField]
+                        }) is too long for a key${Util.OPTIONS.keySuffix.length ? ' (including the suffix ' + Util.OPTIONS.keySuffix + ')' : ''}. Consider renaming your item. Key will be equal first ${maxKeyLength} characters of the name`
                     );
                 }
-
-                if (
-                    item[this.definition.nameField] != item[this.definition.keyField] &&
-                    !this.definition.keyIsFixed
-                ) {
-                    keysForDeploy.push(item[this.definition.keyField]);
+                const newKey = this.getNewKey(item, maxKeyLength);
+                if (newKey != item[this.definition.keyField] && !this.definition.keyIsFixed) {
+                    // add key but make sure to turn it into string or else numeric keys will be filtered later
+                    keysForDeploy.push(item[this.definition.keyField] + '');
                     Util.logger.info(
                         ` - added ${this.definition.type} to fixKey queue: ${
                             item[this.definition.keyField]
-                        }`
+                        } >> ${newKey}`
                     );
                 } else {
                     Util.logger.info(
@@ -2205,6 +2211,29 @@ class MetadataType {
         }
         return keysForDeploy;
     }
+    /**
+     * helper for getKeysForFixing and createOrUpdate
+     *
+     * @param {MetadataTypeItem} metadataItem -
+     * @param {number} maxKeyLength -
+     * @returns {string} newKey
+     */
+    static getNewKey(metadataItem, maxKeyLength) {
+        let newKey;
+        newKey = (metadataItem[this.definition.nameField] + '')
+            .trim()
+            .slice(0, maxKeyLength)
+            .trim();
+        if (Util.OPTIONS.keySuffix.length && !newKey.endsWith(Util.OPTIONS.keySuffix)) {
+            newKey =
+                (metadataItem[this.definition.nameField] + '')
+                    .trim()
+                    .slice(0, maxKeyLength - Util.OPTIONS.keySuffix.length)
+                    .trim() + Util.OPTIONS.keySuffix;
+        }
+
+        return newKey;
+    }
 }
 
 MetadataType.definition = {

lib/metadataTypes/definitions/DeliveryProfile.definition.js

@@ -5,6 +5,8 @@ export default {
     hasExtended: false,
     idField: 'id',
     keyField: 'key',
+    keyIsFixed: false,
+    maxKeyLength: 36, // confirmed max length
     nameField: 'name',
     createdDateField: 'createdDate',
     createdNameField: null,

lib/metadataTypes/definitions/SendClassification.definition.js

@@ -5,8 +5,9 @@ export default {
     filter: {},
     hasExtended: false,
     idField: 'ObjectID',
-    keyIsFixed: null,
     keyField: 'CustomerKey',
+    keyIsFixed: false,
+    maxKeyLength: 36, // confirmed max length
     nameField: 'Name',
     createdDateField: 'CreatedDate',
     createdNameField: null,

lib/metadataTypes/definitions/SenderProfile.definition.js

@@ -4,8 +4,9 @@ export default {
     filter: {},
     hasExtended: false,
     idField: 'ObjectID',
-    keyIsFixed: false,
     keyField: 'CustomerKey',
+    keyIsFixed: false,
+    maxKeyLength: 36, // confirmed max length
     nameField: 'Name',
     createdDateField: 'CreatedDate',
     createdNameField: null,

test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat_fixKeysSuffix/get-response.json

@@ -0,0 +1,17 @@
+{
+    "queryDefinitionId": "549f0568-607c-4940-afef-437965094dat_fixKeysSuffix",
+    "name": "testExisting_query_fixedKeys",
+    "key": "testExisting_query_fixKeysSuffix",
+    "description": "bla bla",
+    "queryText": "Select\n    SubscriberKey as testField, Trim(last_name) AS name\nfrom\n    _Subscribers\nwhere\n    country in ('test')\n",
+    "targetName": "testExisting_dataExtension",
+    "targetKey": "testExisting_dataExtension",
+    "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7",
+    "targetDescription": "",
+    "createdDate": "2022-04-26T15:21:16.453",
+    "modifiedDate": "2022-04-26T16:02:44.01",
+    "targetUpdateTypeId": 0,
+    "targetUpdateTypeName": "Overwrite",
+    "categoryId": 999,
+    "isFrozen": false
+}

test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat_fixKeysSuffix/patch-response.json

@@ -0,0 +1,18 @@
+{
+    "queryDefinitionId": "549f0568-607c-4940-afef-437965094dat_fixKeys",
+    "name": "testExisting_query_fixedKeys",
+    "key": "testExisting_query_fixedKeys_DEV",
+    "description": "updated on deploy",
+    "queryText": "SELECT\n    SubscriberKey as testField\nFROM\n    _Subscribers\nWHERE\n    country IN ('test')\n",
+    "targetName": "testExisting_dataExtension",
+    "targetKey": "testExisting_dataExtension",
+    "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7",
+    "targetDescription": "",
+    "createdDate": "2022-04-26T15:21:16.453",
+    "modifiedDate": "2022-04-26T16:04:15.88",
+    "targetUpdateTypeId": 0,
+    "targetUpdateTypeName": "Overwrite",
+    "validatedQueryText": "SET NOCOUNT ON; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;\r\n\r\nINSERT INTO C518001158.[testDataExtension] ([testField])\r\nSELECT querydef.[testField]\r\nFROM (SELECT SubscriberKey as testField FROM C518001158._Subscribers ) AS querydef \r\nSELECT @rcInsert = @@ROWCOUNT;;\r\n",
+    "categoryId": 999,
+    "isFrozen": false
+}

test/resources/9999999/automation/v1/queries/get-response.json

@@ -53,6 +53,23 @@
             "targetUpdateTypeName": "Overwrite",
             "categoryId": 999,
             "isFrozen": false
+        },
+        {
+            "queryDefinitionId": "549f0568-607c-4940-afef-437965094dat_fixKeysSuffix",
+            "name": "testExisting_query_fixedKeys",
+            "key": "testExisting_query_fixKeysSuffix",
+            "description": "bla bla",
+            "queryText": "Select\n    SubscriberKey as testField, Trim(last_name) AS name\nfrom\n    _Subscribers\nwhere\n    country in ('test')\n",
+            "targetName": "testExisting_dataExtension",
+            "targetKey": "testExisting_dataExtension",
+            "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7",
+            "targetDescription": "",
+            "createdDate": "2022-04-26T15:21:16.453",
+            "modifiedDate": "2022-04-26T16:02:44.01",
+            "targetUpdateTypeId": 0,
+            "targetUpdateTypeName": "Overwrite",
+            "categoryId": 999,
+            "isFrozen": false
         }
     ]
 }

test/resources/9999999/query/patch_fixKeysSuffix-expected.json

@@ -0,0 +1,11 @@
+{
+    "name": "testExisting_query_fixedKeys",
+    "key": "testExisting_query_fixedKeys_DEV",
+    "description": "updated on deploy",
+    "targetKey": "testExisting_dataExtension",
+    "createdDate": "2022-04-26T15:21:16.453",
+    "modifiedDate": "2022-04-26T16:04:15.88",
+    "targetUpdateTypeName": "Overwrite",
+    "isFrozen": false,
+    "r__folder_Path": "Query"
+}

test/resources/9999999/query/patch_fixKeysSuffix-expected.sql

@@ -0,0 +1,6 @@
+SELECT
+    SubscriberKey AS testField
+FROM
+    _Subscribers
+WHERE
+    country IN ('test')

test/resources/9999999/queryDefinition/retrieve-CustomerKey=testExisting_query_fixKeysSuffixANDStatus=Active-response.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?>
+<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+    xmlns:wsa="http://schemas.xmlsoap.org/ws/2004/08/addressing"
+    xmlns:wsse="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd"
+    xmlns:wsu="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd">
+    <soap:Header>
+        <wsa:Action>RetrieveResponse</wsa:Action>
+        <wsa:MessageID>urn:uuid:7ef0345e-b559-4fc4-8986-47e54e1a8a58</wsa:MessageID>
+        <wsa:RelatesTo>urn:uuid:b2e814a6-517c-4882-9bbb-238bfce951ce</wsa:RelatesTo>
+        <wsa:To>http://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymous</wsa:To>
+        <wsse:Security>
+            <wsu:Timestamp wsu:Id="Timestamp-dfc4ae59-8642-4432-b505-554669b47186">
+                <wsu:Created>2023-04-11T16:33:48Z</wsu:Created>
+                <wsu:Expires>2023-04-11T16:38:48Z</wsu:Expires>
+            </wsu:Timestamp>
+        </wsse:Security>
+    </soap:Header>
+    <soap:Body>
+        <RetrieveResponseMsg xmlns="http://exacttarget.com/wsdl/partnerAPI">
+            <OverallStatus>OK</OverallStatus>
+            <RequestID>e8eb2988-2f43-4243-a6b0-6ab6b841a6ab</RequestID>
+            <Results xsi:type="QueryDefinition">
+                <PartnerKey xsi:nil="true" />
+                <ObjectID>549f0568-607c-4940-afef-437965094dat_fixKeysSuffix</ObjectID>
+            </Results>
+        </RetrieveResponseMsg>
+    </soap:Body>
+</soap:Envelope>