Merge pull request #833 from Accenture/bugfix/801-query-targetkey-not…

…-updated-with-de-change

Bugfix/801 query targetkey not updated on DE change

docs/dist/documentation.md

@@ -485,7 +485,7 @@ main class
     * [.joinProject()](#Mcdev.joinProject) ⇒ <code>Promise.&lt;void&gt;</code>
     * [.findBUs(credentialsName)](#Mcdev.findBUs) ⇒ <code>Promise.&lt;void&gt;</code>
     * [.document(businessUnit, type)](#Mcdev.document) ⇒ <code>Promise.&lt;void&gt;</code>
-    * [.deleteByKey(businessUnit, type, customerKey)](#Mcdev.deleteByKey) ⇒ <code>Promise.&lt;void&gt;</code>
+    * [.deleteByKey(businessUnit, type, customerKey)](#Mcdev.deleteByKey) ⇒ <code>Promise.&lt;boolean&gt;</code>
     * [.refresh(businessUnit, type, [keyArr])](#Mcdev.refresh) ⇒ <code>Promise.&lt;void&gt;</code>
     * [.badKeys(businessUnit)](#Mcdev.badKeys) ⇒ <code>Promise.&lt;void&gt;</code>
     * [.retrieveAsTemplate(businessUnit, selectedType, name, market)](#Mcdev.retrieveAsTemplate) ⇒ <code>Promise.&lt;TYPE.MultiMetadataTypeList&gt;</code>
@@ -635,11 +635,11 @@ Creates docs for supported metadata types in Markdown and/or HTML format
 
 <a name="Mcdev.deleteByKey"></a>
 
-### Mcdev.deleteByKey(businessUnit, type, customerKey) ⇒ <code>Promise.&lt;void&gt;</code>
+### Mcdev.deleteByKey(businessUnit, type, customerKey) ⇒ <code>Promise.&lt;boolean&gt;</code>
 deletes metadata from MC instance by key
 
 **Kind**: static method of [<code>Mcdev</code>](#Mcdev)  
-**Returns**: <code>Promise.&lt;void&gt;</code> - -  
+**Returns**: <code>Promise.&lt;boolean&gt;</code> - true if successful, false otherwise  
 
 | Param | Type | Description |
 | --- | --- | --- |
@@ -4015,7 +4015,6 @@ Query MetadataType
     * [.applyTemplateValues(code, templateVariables)](#Query.applyTemplateValues) ⇒ <code>string</code>
     * [.buildDefinitionForNested(templateDir, targetDir, metadata, templateVariables, templateName)](#Query.buildDefinitionForNested) ⇒ <code>Promise.&lt;Array.&lt;Array.&lt;string&gt;&gt;&gt;</code>
     * [.buildTemplateForNested(templateDir, targetDir, metadata, templateVariables, templateName)](#Query.buildTemplateForNested) ⇒ <code>Promise.&lt;Array.&lt;Array.&lt;string&gt;&gt;&gt;</code>
-    * [.parseMetadata(metadata)](#Query.parseMetadata) ⇒ <code>TYPE.CodeExtractItem</code>
     * [.getFilesToCommit(keyArr)](#Query.getFilesToCommit) ⇒ <code>Array.&lt;string&gt;</code>
     * [.checkForErrors(ex)](#Query.checkForErrors) ⇒ <code>Array.&lt;string&gt;</code> \| <code>void</code>
     * [.deleteByKey(customerKey)](#Query.deleteByKey) ⇒ <code>boolean</code>
@@ -4158,18 +4157,6 @@ handles extracted code if any are found for complex types
 ```js
 queries are saved as 1 json and 1 sql file. both files need to be run through templating
 ```
-<a name="Query.parseMetadata"></a>
-
-### Query.parseMetadata(metadata) ⇒ <code>TYPE.CodeExtractItem</code>
-parses retrieved Metadata before saving
-
-**Kind**: static method of [<code>Query</code>](#Query)  
-**Returns**: <code>TYPE.CodeExtractItem</code> - a single item with code parts extracted  
-
-| Param | Type | Description |
-| --- | --- | --- |
-| metadata | <code>TYPE.QueryItem</code> | a single query activity definition |
-
 <a name="Query.getFilesToCommit"></a>
 
 ### Query.getFilesToCommit(keyArr) ⇒ <code>Array.&lt;string&gt;</code>

lib/index.js

@@ -424,7 +424,7 @@ class Mcdev {
      * @param {string} businessUnit references credentials from properties.json
      * @param {string} type supported metadata type
      * @param {string} customerKey Identifier of metadata
-     * @returns {Promise.<void>} -
+     * @returns {Promise.<boolean>} true if successful, false otherwise
      */
     static async deleteByKey(businessUnit, type, customerKey) {
         Util.logger.info('mcdev:: delete');
@@ -449,7 +449,7 @@ class Mcdev {
             try {
                 MetadataTypeInfo[type].properties = properties;
                 MetadataTypeInfo[type].buObject = buObject;
-                await MetadataTypeInfo[type].deleteByKey(customerKey);
+                return await MetadataTypeInfo[type].deleteByKey(customerKey);
             } catch (ex) {
                 Util.logger.errorStack(ex, ` - Deleting ${type} failed`);
             }

lib/metadataTypes/Query.js

@@ -109,7 +109,58 @@ class Query extends MetadataType {
      * @returns {TYPE.CodeExtractItem} Array with one metadata object and one query string
      */
     static postRetrieveTasks(metadata) {
-        return this.parseMetadata(metadata);
+        // folder
+        super.setFolderPath(metadata);
+
+        // extract SQL
+        const codeArr = [
+            {
+                subFolder: null,
+                fileName: metadata[this.definition.keyField],
+                fileExt: 'sql',
+                content: metadata.queryText,
+            },
+        ];
+        delete metadata.queryText;
+
+        try {
+            if (metadata.targetId) {
+                // overwrite targetKey via targetId (it's not updated on name/key change of the DE)
+                const targetKey = cache.searchForField(
+                    'dataExtension',
+                    metadata.targetId,
+                    'ObjectID',
+                    'CustomerKey'
+                );
+                if (targetKey !== metadata.targetKey) {
+                    Util.logger.debug(
+                        ` - ${this.definition.type} ${metadata[this.definition.nameField]} (${
+                            metadata[this.definition.keyField]
+                        }): Replacing targetKey value in saved JSON '${
+                            metadata.targetKey
+                        }' --> '${targetKey}'. Acquired new value from looking up the DE's ObjectID in targetId.`
+                    );
+                    metadata.targetKey = targetKey;
+                }
+            } else {
+                // if no targetId is set, at least check if the targetKey points to an existing DE (no override needed)
+                cache.searchForField(
+                    'dataExtension',
+                    metadata.targetKey,
+                    'CustomerKey',
+                    'CustomerKey'
+                );
+            }
+        } catch (ex) {
+            Util.logger.warn(
+                ` - ${this.definition.type} ${metadata[this.definition.nameField]} (${
+                    metadata[this.definition.keyField]
+                }): ${ex.message}.`
+            );
+        }
+        delete metadata.targetId;
+
+        return { json: metadata, codeArr: codeArr, subFolder: null };
     }
 
     /**
@@ -142,20 +193,32 @@ class Query extends MetadataType {
      * @returns {Promise.<TYPE.QueryItem>} Promise
      */
     static async preDeployTasks(metadata, deployDir) {
+        // folder
+        super.setFolderId(metadata);
+
+        // reinject queryText
         metadata.queryText = await File.readFilteredFilename(
             deployDir + '/' + this.definition.type,
             metadata[this.definition.keyField] + '.' + this.definition.type + '-meta',
             'sql'
         );
+
+        // dataExtension
         metadata.targetKey = cache.searchForField(
             'dataExtension',
             metadata.targetKey,
             'CustomerKey',
             'CustomerKey'
         );
-        // folder
-        super.setFolderId(metadata);
+        // we've seen queries without this ID set - crucial in case the DE ever gets renamed to ensure the query keeps working
+        metadata.targetId = cache.searchForField(
+            'dataExtension',
+            metadata.targetKey,
+            'CustomerKey',
+            'ObjectID'
+        );
 
+        // set ID for Append / Overwrite/ Update action
         metadata.targetUpdateTypeId =
             this.definition.targetUpdateTypeMapping[metadata.targetUpdateTypeName];
         return metadata;
@@ -303,29 +366,6 @@ class Query extends MetadataType {
         return nestedFilePaths;
     }
 
-    /**
-     * parses retrieved Metadata before saving
-     *
-     * @param {TYPE.QueryItem} metadata a single query activity definition
-     * @returns {TYPE.CodeExtractItem} a single item with code parts extracted
-     */
-    static parseMetadata(metadata) {
-        // folder
-        super.setFolderPath(metadata);
-
-        // extract SQL
-        const codeArr = [
-            {
-                subFolder: null,
-                fileName: metadata[this.definition.keyField],
-                fileExt: 'sql',
-                content: metadata.queryText,
-            },
-        ];
-        delete metadata.queryText;
-
-        return { json: metadata, codeArr: codeArr, subFolder: null };
-    }
     /**
      * should return only the json for all but asset, query and script that are saved as multiple files
      * additionally, the documentation for dataExtension and automation should be returned

lib/metadataTypes/definitions/Query.definition.js

@@ -86,8 +86,8 @@ module.exports = {
             template: false,
         },
         targetId: {
-            isCreateable: false,
-            isUpdateable: false,
+            isCreateable: true,
+            isUpdateable: true,
             retrieving: false,
             template: false,
         },

test/mockRoot/deploy/testInstance/testBU/query/testNewQuery.query-meta.sql

@@ -1,4 +1,4 @@
 SELECT
     SubscriberKey as testField
 FROM
-    _Subscribers 
+    _Subscribers

test/query.test.js

@@ -18,13 +18,48 @@ describe('query', () => {
     });
 
     describe('Retrieve ================', () => {
-        it('Should retrieve a query', async () => {
+        it('Should retrieve all queries', async () => {
             // WHEN
             await handler.retrieve('testInstance/testBU', ['query']);
             // THEN
             assert.equal(!!process.exitCode, false, 'retrieve should not have thrown an error');
             // get results from cache
             const result = cache.getCache();
+            assert.equal(
+                result.query ? Object.keys(result.query).length : 0,
+                2,
+                'only two queries expected'
+            );
+            // normal test
+            assert.deepEqual(
+                await testUtils.getActualJson('testExistingQuery', 'query'),
+                await testUtils.getExpectedJson('9999999', 'query', 'get'),
+                'returned metadata with correct key was not equal expected'
+            );
+            expect(file(testUtils.getActualFile('testExistingQuery', 'query', 'sql'))).to.equal(
+                file(testUtils.getExpectedFile('9999999', 'query', 'get', 'sql'))
+            );
+            // check if targetKey was overwritten
+            assert.deepEqual(
+                await testUtils.getActualJson('testExistingQuery2', 'query'),
+                await testUtils.getExpectedJson('9999999', 'query', 'get2'),
+                'returned metadata with wrong key was not equal expected'
+            );
+
+            assert.equal(
+                testUtils.getAPIHistoryLength(),
+                6,
+                'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
+            );
+            return;
+        });
+        it('Should retrieve one specific query', async () => {
+            // WHEN
+            await handler.retrieve('testInstance/testBU', ['query'], ['testExistingQuery']);
+            // THEN
+            assert.equal(!!process.exitCode, false, 'retrieve should not have thrown an error');
+            // get results from cache
+            const result = cache.getCache();
             assert.equal(
                 result.query ? Object.keys(result.query).length : 0,
                 1,
@@ -40,7 +75,7 @@ describe('query', () => {
             );
             assert.equal(
                 testUtils.getAPIHistoryLength(),
-                6,
+                7,
                 'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
             );
             return;
@@ -59,8 +94,8 @@ describe('query', () => {
             const result = cache.getCache();
             assert.equal(
                 result.query ? Object.keys(result.query).length : 0,
-                2,
-                'two querys expected'
+                3,
+                'three queries expected'
             );
             // confirm created item
             assert.deepEqual(
@@ -205,4 +240,38 @@ describe('query', () => {
             return;
         });
     });
+    describe('Delete ================', () => {
+        it('Should delete the item', async () => {
+            // WHEN
+            const result = await handler.deleteByKey('testInstance/testBU', 'query', [
+                'testExistingQuery',
+            ]);
+            // THEN
+
+            assert.equal(result, true, 'should have deleted the item');
+            return;
+        });
+    });
+    describe('CI/CD ================', () => {
+        it('Should return a list of files based on their type and key', async () => {
+            // WHEN
+            const fileList = await handler.getFilesToCommit('testInstance/testBU', 'query', [
+                'testExistingQuery',
+            ]);
+            // THEN
+            assert.equal(fileList.length, 2, 'expected only 2 file paths');
+
+            assert.equal(
+                fileList[0].split('\\').join('/'),
+                'retrieve/testInstance/testBU/query/testExistingQuery.query-meta.json',
+                'wrong JSON path'
+            );
+            assert.equal(
+                fileList[1].split('\\').join('/'),
+                'retrieve/testInstance/testBU/query/testExistingQuery.query-meta.sql',
+                'wrong JSON path'
+            );
+            return;
+        });
+    });
 });

test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat/delete-response.json

@@ -0,0 +1 @@
+"OK"

test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat/get-response.json

@@ -0,0 +1,17 @@
+{
+    "queryDefinitionId": "549f0568-607c-4940-afef-437965094dat",
+    "name": "testExistingQuery",
+    "key": "testExistingQuery",
+    "description": "bla bla",
+    "queryText": "SELECT\n    SubscriberKey as testField\nFROM\n    _Subscribers\nWHERE\n    country IN ('test')\n",
+    "targetName": "childBU_dataextension_test",
+    "targetKey": "childBU_dataextension_test",
+    "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7",
+    "targetDescription": "",
+    "createdDate": "2022-04-26T15:21:16.453",
+    "modifiedDate": "2022-04-26T16:02:44.01",
+    "targetUpdateTypeId": 0,
+    "targetUpdateTypeName": "Overwrite",
+    "categoryId": 999,
+    "isFrozen": false
+}

test/resources/9999999/automation/v1/queries/549f0568-607c-4940-afef-437965094dat/patch-response.json

@@ -6,7 +6,7 @@
     "queryText": "SELECT\n    SubscriberKey as testField\nFROM\n    _Subscribers\nWHERE\n    country IN ('test')\n",
     "targetName": "childBU_dataextension_test",
     "targetKey": "childBU_dataextension_test",
-    "targetId": "30400c03-0ec4-ec11-b83c-48df37d1de8a",
+    "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7",
     "targetDescription": "",
     "createdDate": "2022-04-26T15:21:16.453",
     "modifiedDate": "2022-04-26T16:04:15.88",

test/resources/9999999/automation/v1/queries/get-response.json

@@ -1,5 +1,5 @@
 {
-    "count": 1,
+    "count": 2,
     "page": 1,
     "pageSize": 50,
     "items": [
@@ -11,7 +11,24 @@
             "queryText": "SELECT\n    SubscriberKey as testField\nFROM\n    _Subscribers\nWHERE\n    country IN ('test')\n",
             "targetName": "childBU_dataextension_test",
             "targetKey": "childBU_dataextension_test",
-            "targetId": "30400c03-0ec4-ec11-b83c-48df37d1de8a",
+            "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7",
+            "targetDescription": "",
+            "createdDate": "2022-04-26T15:21:16.453",
+            "modifiedDate": "2022-04-26T16:02:44.01",
+            "targetUpdateTypeId": 0,
+            "targetUpdateTypeName": "Overwrite",
+            "categoryId": 999,
+            "isFrozen": false
+        },
+        {
+            "queryDefinitionId": "abcde-607c-4940-afef-437965094dat",
+            "name": "testExistingQuery2",
+            "key": "testExistingQuery2",
+            "description": "bla bla",
+            "queryText": "SELECT\n    SubscriberKey as testField\nFROM\n    _Subscribers\nWHERE\n    country IN ('test')\n",
+            "targetName": "childBU_dataextension_test",
+            "targetKey": "childBU_dataextension_test-WRONG",
+            "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7",
             "targetDescription": "",
             "createdDate": "2022-04-26T15:21:16.453",
             "modifiedDate": "2022-04-26T16:02:44.01",

test/resources/9999999/automation/v1/queries/post-response.json

@@ -3,10 +3,10 @@
     "name": "testNewQuery",
     "key": "testNewQuery",
     "description": "created on deploy",
-    "queryText": "SELECT\n    SubscriberKey as testField\nFROM\n    _Subscribers ",
+    "queryText": "SELECT\n    SubscriberKey as testField\nFROM\n    _Subscribers\n",
     "targetName": "childBU_dataextension_test",
     "targetKey": "childBU_dataextension_test",
-    "targetId": "30400c03-0ec4-ec11-b83c-48df37d1de8a",
+    "targetId": "21711373-72c1-ec11-b83b-48df37d1deb7",
     "targetDescription": "",
     "createdDate": "2022-04-26T15:21:16.453",
     "modifiedDate": "2022-04-26T16:04:15.88",